diff --git a/TensorFlow2x/Accuracy_Validation/ResNet50_Official/README.md b/TensorFlow2x/Accuracy_Validation/ResNet50_Official/README.md index 01893c491e4285cf1479cebee3e49b0c16aa2e14..338d62547ef8dec2f4a3515ab9aebef19b0a3387 100644 --- a/TensorFlow2x/Accuracy_Validation/ResNet50_Official/README.md +++ b/TensorFlow2x/Accuracy_Validation/ResNet50_Official/README.md @@ -1,6 +1,6 @@ # 介绍 -该测试用例可用于ResNet50等网络的性能测试及精度验证。 + 该测试用例可用于ResNet50等网络的性能测试及精度验证。 # 单卡测试（单精度） @@ -21,11 +21,35 @@ # 多卡测试 （单精度） ## 运行 + # sed指令只需要执行一次，添加支持多卡运行的代码 + sed -i '99 r configfile' official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py + export PYTHONPATH=/path/to/tensorflow/model:$PYTHONPATH - mpirun -np $np --hostfile hostfile --bind-to none scripts-run/single_process.sh + mpirun -np $np --hostfile hostfile -mca btl self,tcp --bind-to none scripts-run/single_process.sh + +# 多卡测试 (混合精度) + +## 运行 + # sed指令只需要执行一次，添加支持多卡运行的代码 + sed -i '99 r configfile' official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py + + 修改scripts-run/single_process.sh中的--dtype=fp16 + export PYTHONPATH=/path/to/tensorflow/model:$PYTHONPATH + mpirun -np $np --hostfile hostfile -mca btl self,tcp --bind-to none scripts-run/single_process.sh + ## 测试说明 -多卡测试时需要修改部分代码，具体可参考https://tensorflow.google.cn/guide/migrate/multi_worker_cpu_gpu_training?hl=en + 多卡测试时需要修改部分代码，具体可参考https://tensorflow.google.cn/guide/migrate/multi_worker_cpu_gpu_training?hl=en + + hostfile格式参考： + node1 slots=4 + node2 slots=4 + + nodefile格式参考(与hostfile文件内保持一致，用于构造TF_CONFIG)： + node1 + node2 + + 上面多卡测试方法默认每节点4卡 # 参考 [https://github.com/tensorflow/models/tree/v2.7.0](https://github.com/tensorflow/models/tree/v2.7.0/official/vision/image_classification/resnet) diff --git a/TensorFlow2x/Accuracy_Validation/ResNet50_Official/configfile b/TensorFlow2x/Accuracy_Validation/ResNet50_Official/configfile new file mode 100644 index 0000000000000000000000000000000000000000..60faf7e2da838b509da1c31b00cbbffd47acdcb5 --- /dev/null +++ b/TensorFlow2x/Accuracy_Validation/ResNet50_Official/configfile @@ -0,0 +1,22 @@ + + worker = [] + port_number =40000 + + filePath='./nodefile' + with open(filePath, 'r') as f: + nodename = f.read().splitlines() + + worker_nodes = nodename + num_index=flags_obj.task_index + + for node in worker_nodes: + for index in range(4): + worker_sockets = ":".join([node, str(port_number + index )]) + worker.append(worker_sockets) + os.environ['TF_CONFIG'] = json.dumps({ + 'cluster': { + 'worker': worker + }, + 'task': {'type': 'worker', 'index': num_index} + }) + diff --git a/TensorFlow2x/Accuracy_Validation/ResNet50_Official/scripts-run/.single_process.sh.swp b/TensorFlow2x/Accuracy_Validation/ResNet50_Official/scripts-run/.single_process.sh.swp new file mode 100644 index 0000000000000000000000000000000000000000..0dfa2d1964764accd9aa7be466ee03417f0e9752 Binary files /dev/null and b/TensorFlow2x/Accuracy_Validation/ResNet50_Official/scripts-run/.single_process.sh.swp differ diff --git a/TensorFlow2x/Accuracy_Validation/ResNet50_Official/scripts-run/single_process.sh b/TensorFlow2x/Accuracy_Validation/ResNet50_Official/scripts-run/single_process.sh index c0f203d6c444cecc8f2200fdfa33d734cca1e958..2049e3cb2789d7b57d59a703badd6ab40c357038 100644 --- a/TensorFlow2x/Accuracy_Validation/ResNet50_Official/scripts-run/single_process.sh +++ b/TensorFlow2x/Accuracy_Validation/ResNet50_Official/scripts-run/single_process.sh @@ -3,7 +3,7 @@ lrank=$OMPI_COMM_WORLD_LOCAL_RANK drank=$OMPI_COMM_WORLD_RANK -APP="python3 ../official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py --num_gpus=1 --skip_eval=true --batch_size=128 --train_epochs=90 --use_synthetic_data=false --distribution_strategy=multi_worker_mirrored --all_reduce_alg=nccl --data_dir=${data_dir} --task_index=${drank} " +APP="python3 ../official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py --num_gpus=1 --skip_eval=true --batch_size=512 --train_epochs=90 --use_synthetic_data=false --distribution_strategy=multi_worker_mirrored --all_reduce_alg=nccl --dtype=fp32 --data_dir=${data_dir} --task_index=${drank} " case ${lrank} in [0]) export HIP_VISIBLE_DEVICES=0 diff --git a/TensorFlow2x/ComputeVision/Classification/README.md b/TensorFlow2x/ComputeVision/Classification/README.md old mode 100644 new mode 100755 index 0314702e7b1f66b5b21045e3524fee8770695d20..6193cdf29d1b584221ca1e8f7f2b6d81fff28f4b --- a/TensorFlow2x/ComputeVision/Classification/README.md +++ b/TensorFlow2x/ComputeVision/Classification/README.md @@ -2,26 +2,42 @@ 该测试用例用于TensorFlow分类模型性能测试，使用的数据集是imagenet。 -* 该脚本支持horovod等分布式通信库方式 - # 运行 ## 单卡 - - python3 ./benchmarks-master/scripts/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_format=NCHW --batch_size=128 --model=resnet50 --optimizer=momentum --variable_update=parameter_server --print_training_accuracy=true --nodistortions --num_gpus=1 --num_epochs=90 --weight_decay=1e-4 --data_dir=$data_dir_path --use_fp16=False --data_name=imagenet --train_dir=$save_checkpoint_path + + export PYTHONPATH=/path/to/tensorflow/model:$PYTHONPATH + export HIP_VISIBLE_DEVICES=0 + python3 models-master/official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py --data_dir=/path/to/{ImageNet-tensorflow_data_dir} --model_dir=/path/to/{model_save_dir} --batch_size=128 --num_gpus=1 --use_synthetic_data=false ## 单机多卡 - - python3 ./benchmarks-master/scripts/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_format=NCHW --batch_size=128 --model=resnet50 --optimizer=momentum --variable_update=parameter_server --print_training_accuracy=true --nodistortions --num_gpus=4 --num_epochs=90 --weight_decay=1e-4 --data_dir=$data_dir_path --use_fp16=False --data_name=imagenet --train_dir=$save_checkpoint_path + + export PYTHONPATH=/path/to/tensorflow/model:$PYTHONPATH + export HIP_VISIBLE_DEVICES=0,1,2,3 + python3 models-master/official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py --data_dir=/path/to/{ImageNet-tensorflow_data_dir} --model_dir=/path/to/{model_save_dir} --batch_size=512 --num_gpus=4 --use_synthetic_data=false ## 分布式多卡 + + # sed指令只需要执行一次，添加支持多卡运行的代码 + sed -i '99 r configfile' models-master/official/vision/image_classification/resnet/resnet_ctl_imagenet_main.py + + export PYTHONPATH=/path/to/tensorflow/model:$PYTHONPATH + mpirun -np ${num_gpu} --hostfile hostfile -mca btl self,tcp --bind-to none scripts-run/single_process.sh + +### 测试说明 + + 多卡测试时需要修改部分代码，具体可参考https://tensorflow.google.cn/guide/migrate/multi_worker_cpu_gpu_training?hl=en + + hostfile格式参考： + node1 slots=4 + node2 slots=4 - mpirun -np ${num_gpu} --hostfile hostfile --bind-to none scripts-run/single_process.sh + nodefile格式参考(与hostfile文件内保持一致，用于构造TF_CONFIG)： + node1 + node2 -hostfile格式参考： + 上面分布式多卡测试方法默认每节点4卡 - node1 slots=4 - node2 slots=4 # 参考资料 [https://github.com/tensorflow/benchmarks/tree/master/scripts/tf_cnn_benchmarks](https://github.com/tensorflow/benchmarks/tree/master/scripts/tf_cnn_benchmarks) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/.gitignore b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/.gitignore deleted file mode 100644 index 00eb5794b2bac6f9328f9c59b9bf2d15f8c58fb7..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/.gitignore +++ /dev/null @@ -1,95 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class -*.sw[op] - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*,cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# IPython Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# dotenv -.env - -# virtualenv -venv/ -ENV/ - -# Spyder project settings -.spyderproject - -# Rope project settings -.ropeproject - -# PyCharm -.idea/ - -# For mac -.DS_Store diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/LICENSE b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/LICENSE deleted file mode 100644 index 8dada3edaf50dbc082c9a125058f25def75e625a..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/README.md b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/README.md deleted file mode 100644 index 39b36585999d4f196f1d30950ee19df2ece25f2b..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/README.md +++ /dev/null @@ -1,482 +0,0 @@ -Table of Contents -================= - -* [Table of Contents](#table-of-contents) -* [Introduction](#introduction) -* [Executing tests](#executing-tests) - * [PerfZero on private GCE instance.](#perfzero-on-private-gce-instance) - * [Step one: Create GCE Instance](#step-one-create-gce-instance) - * [Step two: Build docker on GCE instance](#step-two-build-docker-on-gce-instance) - * [Step three: Start and "enter" the docker instance](#step-three-start-and-enter-the-docker-instance) - * [Step four: Run tests](#step-four-run-tests) - * [Step five: Delete the instance when done](#step-five-delete-the-instance-when-done) - * [PerfZero on local workstation or any server](#perfzero-on-local-workstation-or-any-server) - * [PerfZero without docker](#perfzero-without-docker) -* [Creating tests](#creating-tests) -* [Deep dive into individual tools](#deep-dive-into-individual-tools) - * [Build docker image](#build-docker-image) - * [Run benchmark](#run-benchmark) - * [Instructions for managing Google Cloud Platform computing instance](#instructions-for-managing-google-cloud-platform-computing-instance) - * [Understand the benchmark execution output](#understand-the-benchmark-execution-output) - * [Json formatted benchmark summary](#json-formatted-benchmark-summary) - * [Profiling](#profiling) - * [Visualize in TensorBoard](#visualize-in-tensorboard) - * [Visualize system metric values over time](#visualize-system-metric-values-over-time) -* [PerfZero development](#perfzero-development) - -# Introduction - -PerfZero is a benchmark framework for TensorFlow. It intends to address the -following use-cases: - -1) For user who wants to execute TensorFlow test to debug performance -regression. - -PerfZero makes it easy to execute the pre-defined test by consolidating the -docker image build, GPU driver installation, TensorFlow installation, benchmark -library checkout, data download, system statistics collection, benchmark metrics -collection, profiler data collection and so on into 2 to 3 commands. This allows -developer to focus on investigating the issue rather than setting up the test -environment. - -2) For user who wants to track the performance change of TensorFlow for a -variety of setup (e.g. GPU model, cudnn version, TensorFlow version) - -The developer can setup periodic job to execute these benchmark methods using -PerfZero. PerfZero will collect the information needed to identify the -benchmark (e.g. GPU model, TensorFlow version, dependent library git hash), get -benchmark execution result (e.g. wall time, accuracy, succeeded or not), -summarize the result in a easy-to-read json string and upload the result to -bigquery table. Using the data in the bigquery table, user can then visualize -the performance change in a dashboard, compare performance between different -setup in a table, and trigger alert on when there is performance regression. - - -# Executing tests - -There are multiple ways to use PerfZero to execute a test. Listed from highest -to lowest abstraction. - -* [PerfZero on private GCE instance](#perfzero-on-private-gce-instance) -* [PerfZero on local workstation or any server](#perfzero-on-local-workstation-or-any-server) -* [PerfZero without docker](#perfzero-without-docker) - -## PerfZero on private GCE instance. - -There are many variations on this approach, to get you started quickly the steps -below detail setting up an 8xV100 setup with local nvme drives where training -data would be stored. The only con to this setup is that it cannot be stopped -and can only be deleted due to the local nvme drives. - -### Step one: Create GCE Instance - -Creates an 8xV100 instance with 4 nvme drives. Output of the command will -provide the command to run to SSH to the machine. To set the project, zone, and -other features read -[cloud_manager tool details](https://github.com/tensorflow/benchmarks/tree/master/perfzero#instructions-for-managing-google-cloud-platform-computing-instance). - -```bash -python perfzero/lib/cloud_manager.py create --accelerator_count 8 --nvme_count 4 -``` - -### Step two: Build docker on GCE instance - -After logging into the instance run the following command to create a docker -instance with the latest nightly TF 2.0 build. For more options read the -[build docker image section](https://github.com/tensorflow/benchmarks/tree/master/perfzero#build-docker-image) - -```bash -python3 perfzero/lib/setup.py --dockerfile_path=docker/Dockerfile_ubuntu_1804_tf_v2 -``` - -For all options for building the docker image, including controlling the version -of TensorFlow installed, checkout the public -[README for PerfZero](https://github.com/tensorflow/benchmarks/tree/master/perfzero#build-docker-image). - -### Step three: Start and "enter" the docker instance - -```bash -nvidia-docker run -it --rm -v $(pwd):/workspace -v /data:/data perfzero/tensorflow bash -``` - -### Step four: Run tests - -The command below pulls GitHub official/models, downloads the cifar-10 dataset -from our internal Google Cloud storage bucket, and executes a ResNet56 benchmark -with TensorFlow 2.0 nightly build. For info on the args read the -[run benchmark section](https://github.com/tensorflow/benchmarks/tree/master/perfzero#run-benchmark). - -```bash -python3 /workspace/perfzero/lib/benchmark.py \ ---git_repos="https://github.com/tensorflow/models.git;benchmark" \ ---python_path=models \ ---data_downloads="gs://tf-perf-imagenet-uswest1/tensorflow/cifar10_data/cifar-10-batches-bin" \ ---benchmark_methods=official.benchmark.keras_cifar_benchmark.Resnet56KerasBenchmarkReal.benchmark_1_gpu_no_dist_strat -``` - -For all options that can be used when executing a test checkout the public -[README for PerfZero](https://github.com/tensorflow/benchmarks/tree/master/perfzero#run-benchmark). - -### Step five: Delete the instance when done - -```bash -python perfzero/lib/cloud_manager.py delete -``` - -## PerfZero on local workstation or any server - -This approach is the same as PerfZero on personal GCE instance, just jump to -Step two: Build docker on GCE instance. - -If the workstation does not have access to the PerfZero GCS bucket and does not -need access, e.g. data is already copied locally via another method, passing -`--gcloud_key_file_url=""` will skip attempting to download the key. - -A quick test that does not require accessing GCS for data is: - -```bash -python3 /workspace/perfzero/lib/benchmark.py \ ---git_repos="https://github.com/tensorflow/models.git;benchmark" \ ---python_path=models \ ---gcloud_key_file_url="" \ ---benchmark_methods=official.benchmark.keras_cifar_benchmark.Resnet56KerasBenchmarkSynth.benchmark_1_gpu_no_dist_strat -``` - -## PerfZero without docker - -PerfZero is not dependent on Docker. Docker is used to handle dependencies and -create a consistent environment. Most tests do not require much beyond -TensorFlow and PerfZero mostly depends on Google Cloud, but only for downloading -data and upload results if desired. While this approach works, we do not -maintain a clear list of the required libraries. The Docker build files are a -good starting point beyond. - -Once the requirements are met the command below can be executed which will pull -GitHub official/models, downloads the cifar-10 dataset from our internal Google -Cloud storage bucket, and executes a ResNet56 benchmark with TensorFlow 2.0 -nightly build. - -```bash -python3 /workspace/perfzero/lib/benchmark.py \ ---git_repos="https://github.com/tensorflow/models.git;benchmark" \ ---python_path=models \ ---data_downloads="gs://tf-perf-imagenet-uswest1/tensorflow/cifar10_data/cifar-10-batches-bin" \ ---benchmark_methods=official.r1.resnet.estimator_benchmark.Resnet50EstimatorBenchmarkReal.benchmark_graph_1_gpu -``` - -# Creating tests - -Here are the instructions that developers of benchmark method needs to follow in -order to run benchmark method in PerfZero. See -[estimator_benchmark.py](https://github.com/tensorflow/models/blob/master/official/r1/resnet/estimator_benchmark.py) -for example test code that supports PerfZero. - -1) The benchmark class should extend the TensorFlow python class -[tensorflow.test.Benchmark](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/platform/benchmark.py). The benchmark class constructor should have a -constructor with signature `__init__(self, output_dir, data_dir, **kwargs)`. -Below is the usage for each arguments: - -- Benchmark method should put all generated files (e.g. logs) in `output_dir` so that PerfZero can -upload these files to Google Cloud Storage when `--output_gcs_url` is specified. - -- Benchmark method should read data from `root_data_dir`. For example, the benchmark method can read data from e.g. `${root_data_dir}/cifar-10-binary` - -- `**kwargs` is useful to make the benchmark constructor forward compatible when PerfZero provides more named arguments to the benchmark constructor before - updating the benchmark class. - - -2) At the end of the benchmark method execution, the method should call [report_benchmark()](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/platform/benchmark.py) -with the following parameters: - -``` -tf.test.Benchmark.report_benchmark( - iters=num_iteration, # The number of iterations of the benchmark. - - wall_time=wall_time_sec, # Total wall time in sec for all iterations. - metrics = [ # List of metric entries - { - 'name': 'accuracy_top_5', # Metric name - 'value': 80, # Metric value - 'min_value': 90, # Optional. Minimum acceptable metric value for the benchmark to succeed. - 'max_value': 99 # Optional. Maximum acceptable metric value for the benchmark to succeed. - }, - { - 'name': 'accuracy_top_1', - 'value': 99.5 - } - ] -) -``` - -This format allows PerfZero to specify whether the benchmark has succeeded -(e.g. for convergence test) in its summary based on the logic determined by -the benchmark developer. - - -3) Include dependent libraries in `--git_repos` and `--python_path`. These -libraries will be checked-out in the directory -`path_to_perfzero/workspace/site-packages` by default. Developer can edit these -libraries directly and execute benchmark with the local change. - - -# Deep dive into individual tools - -The sections below go into details about the individual components of PerfZero. - -## Build docker image - -The command below builds the docker image named `perfzero/tensorflow` which contains the -libraries (e.g. TensorFlow) needed for benchmark. - -``` -python3 benchmarks/perfzero/lib/setup.py -``` - -Here are a few selected optional flags. Run `python3 setup.py -h` to see -detailed documentation for all supported flags. - -1) Use `--dockerfile_path=docker/Dockerfile_ubuntu_1804_tf_v2` to build docker image for TensorFlow v2 -2) Use `--tensorflow_pip_spec` to specify the tensorflow pip package name (and optionally version) to be -installed in the docker image, e.g. `--tensorflow_pip_spec=tensorflow==1.12.0`. - - -## Run benchmark - -The command below executes the benchmark method specified by `--benchmark_methods`. - -``` -export ROOT_DATA_DIR=/data - -nvidia-docker run -it --rm -v $(pwd):/workspace -v $ROOT_DATA_DIR:$ROOT_DATA_DIR perfzero/tensorflow \ -python3 /workspace/benchmarks/perfzero/lib/benchmark.py \ ---gcloud_key_file_url="" \ ---git_repos="https://github.com/tensorflow/models.git;benchmark" \ ---python_path=models \ ---benchmark_methods=official.r1.resnet.estimator_benchmark.Resnet50EstimatorBenchmarkSynth.benchmark_graph_1_gpu \ ---root_data_dir=$ROOT_DATA_DIR -``` - -`${ROOT_DATA_DIR}` should be the directory which contains the dataset files -required by the benchmark method. If the flag `--data_downloads` is specified, -PerfZero will download files from the specified url to the directory specified -by the flag `--root_data_dir`. Otherwise, user needs to manually download and -move the dataset files into the directory specified by `--root_data_dirs`. The -default `root_data_dir` is `/data`. Some benchmark methods, like the one run in -the sample command above, do not require any dataset files. - -Here are a few selected optional flags. Run `python3 benchmark.py -h` to see -detailed documentation for all supported flags. - -1) Use `--workspace=unique_workspace_name` if you need to run multiple benchmark -using different workspace setup. One example usecase is that you may want to -test a branch from a pull request without changing your existing workspace. - -2) Use `--debug` if you need to see the debug level logging - -3) Use `--git_repos="git_url;git_branch;git_hash"` to checkout a git repo with -the specified git_branch at the specified git_hash to the local folder with the -specified folder name. **Note that** the value of the flag `--git_repos` is -wrapped by the quotation mark `"` so that `;` will not be interpreted by the -bash as the end of the command. Specify the flag once for each repository you -want to checkout. - -5) Use `--profiler_enabled_time=start_time:end_time` to collect profiler data -during period `[start_time, end_time)` after the benchmark method execution -starts. Skip `end_time` in the flag value to collect data until the end of -benchmark method execution. See [here](#visualize-tensorflow-graph-etc-using-tensorboard) -for instructions on how to use the generated profiler data. - -## Instructions for managing Google Cloud Platform computing instance - -PerfZero aims to make it easy to run and debug TensorFlow which is usually run -with GPU. However, most users do not have dedicated machine with expensive -hardware. One cost-effective solution is for users to create machine with the -desired hardware on demand in a public cloud when they need to debug TensorFlow. - -We provide a script in PerfZero to make it easy to manage computing instance in -Google Cloud Platform. This assumes that you have access to an existing project -in GCP. - -Run `python perfzero/lib/cloud_manager.py -h` for list of commands supported -by the script. Run `cloud_manager.py -h` to see detailed documentation -for all supported flags for the specified `command`. - -In most cases, user only needs to run the following commands: - -``` -# Create a new instance that is unique to your username -python perfzero/lib/cloud_manager.py create --project=project_name - -# Query the status of the existing instanced created by your and its IP address -python perfzero/lib/cloud_manager.py status --project=project_name - -# Stop the instance -python perfzero/lib/cloud_manager.py stop --project=project_name - -# Start the instance -python perfzero/lib/cloud_manager.py start --project=project_name - -# Delete the instance -python perfzero/lib/cloud_manager.py delete --project=project_name -``` - -## Understand the benchmark execution output - -### Json formatted benchmark summary - -PerfZero outputs a json-formatted summary that provides the information needed -to understand the benchmark result. The summary is printed in the stdout and -in the file `path_to_perfzero/${workspace}/output/${execution_id}/perfzero.log`. - -Additionally, Perfzero outputs a pure json file containing the summary at -`path_to_perfzero/${workspace}/output/${execution_id}/perfzero_summary.json` - -Here is an example output from PerfZero. Explanation is provided inline for each -key when the name of the key is not sufficiently self-explanary. - -``` - { - "ml_framework_info": { # Summary of the machine learning framework - "version": "1.13.0-dev20190206", # Short version. It is tf.__version__ for TensorFlow - "name": "tensorflow", # Machine learning framework name such as PyTorch - "build_label": "ml_framework_build_label", # Specified by the flag --ml_framework_build_label - "build_version": "v1.12.0-7504-g9b32b5742b" # Long version. It is tf.__git_version__ for TensorFlow - }, - "execution_timestamp": 1550040322.8991697, # Timestamp when the benchmark is executed - "execution_id": "2019-02-13-06-45-22-899128", # A string that uniquely identify this benchmark execution - - "benchmark_info": { # Summary of the benchmark framework setup - "output_url": "gs://tf-performance/test-results/2019-02-13-06-45-22-899128/", # Google storage url that contains the log file from this benchmark execution - "has_exception": false, - "site_package_info": { - "models": { - "branch": "benchmark", - "url": "https://github.com/tensorflow/models.git", - "hash": "f788046ca876a8820e05b0b48c1fc2e16b0955bc" - }, - "benchmarks": { - "branch": "master", - "url": "https://github.com/tensorflow/benchmarks.git", - "hash": "af9e0ef36fc6867d9b63ebccc11f229375cd6a31" - } - }, - "harness_name": "perfzero", - "harness_info": { - "url": "https://github.com/tensorflow/benchmarks.git", - "branch": "master", - "hash": "75d2991b88630dde10ef65aad8082a6d5cd8b5fc" - }, - "execution_label": "execution_label" # Specified by the flag --execution_label - }, - - "system_info": { # Summary of the resources in the system that is used to execute the benchmark - "system_name": "system_name", # Specified by the flag --system_name - "accelerator_count": 2, # Number of GPUs in the system - "physical_cpu_count": 8, # Number of physical cpu cores in the system. Hyper thread CPUs are excluded. - "logical_cpu_count": 16, # Number of logical cpu cores in the system. Hyper thread CPUs are included. - "cpu_socket_count": 1, # Number of cpu socket in the system. - "platform_name": "platform_name", # Specified by the flag --platform_name - "accelerator_model": "Tesla V100-SXM2-16GB", - "accelerator_driver_version": "410.48", - "cpu_model": "Intel(R) Xeon(R) CPU @ 2.20GHz" - }, - - "process_info": { # Summary of the resources used by the process to execute the benchmark - "max_rss": 4269047808, # maximum physical memory in bytes used by the process - "max_vms": 39894450176, # maximum virtual memory in bytes used by the process - "max_cpu_percent": 771.1 # CPU utilization as a percentage. See psutil.Process.cpu_percent() for more information - }, - - "benchmark_result": { # Summary of the benchmark execution results. This is pretty much the same data structure defined in test_log.proto. - # Most values are read from test_log.proto which is written by tf.test.Benchmark.report_benchmark() defined in TensorFlow library. - - "metrics": [ # This is derived from `extras` [test_log.proto](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/util/test_log.proto) - # which is written by report_benchmark(). - # If the EntryValue is double, then name is the extra's key and value is extra's double value. - # If the EntryValue is string, then name is the extra's key. The string value will be a json formated string whose keys - # include `value`, `succeeded` and `description`. Benchmark method can provide arbitrary metric key/value pairs here. - { - "name": "accuracy_top_5", - "value": 0.7558000087738037 - }, - { - "name": "accuracy_top_1", - "value": 0.2639999985694885 - } - ], - "name": "official.resnet.estimator_cifar_benchmark.EstimatorCifar10BenchmarkTests.unit_test", # Full path to the benchmark method, i.e. module_path.class_name.method_name - "succeeded": true, # True iff benchmark method execution finishes without exception and no metric in metrics show succeeded = false - "wall_time": 14.552583694458008 # The value is determined by tf.test.Benchmark.report_benchmark() called by the benchmark method. It is -1 if report_benchmark() is not called. - } -} -``` - -### Profiling - -When the flag `--profiler_enabled_time=start_time:end_time` is specified, the -profiler data will be collected and stored in -`path_to_perfzero/${workspace}/output/${execution_id}/profiler_data`. - -#### Visualize in TensorBoard - -Firstly, install the profile plugin for TensorBoard. -``` -pip install -U tensorboard-plugin-profile -``` - -Run `tensorboard --logdir=path_to_perfzero/workspace/output/${execution_id}/profiler_data` or -`python3 -m tensorboard.main --logdir=path_to_perfzero/workspace/output/${execution_id}/profiler_data` to open -TensorBoard server. - -If PerfZero is executed on a remote machine, run `ssh -L -6006:127.0.0.1:6006 remote_ip` before opening `http://localhost:6006` in your -browser to access the TensorBoard UI. - -You can also run TensorBoard inside the docker, e.g. -`tensorboard --logdir=/workspace/perfzero/workspace/output/${execution_id}/profiler_data --bind_all` - -In this case, you have to start docker with port mapping, i.e. with "-p 6006:6006" flag, .e.g -``` -nvidia-docker run -it --rm -v $(pwd):/workspace -p 6006:6006 perfzero/tensorflow -``` - -Normally, the pages you see will look like: - -![Screenshot](screenshots/profiling_overview.png "Profiling Overview") -![Screenshot](screenshots/profiling_trace_view.png "Profiling Trace View") - -### Visualize system metric values over time - -PerfZero also records a few useful system metrics (e.g. rss, vms) over time in -the file `path_to_perfzero/${workspace}/output/${execution_id}/process_info.log`. -Run `python perfzero/scripts/plot_process_info.py process_info.log` to generate a -pdf showing the value of these metrics over time. - - -# PerfZero development - - -Avoid importing `tensorflow` package in any place that requires the `logging` -package because tensorflow package appears to prevent logging from working -properly. Importing `tensorflow` package only in the method that requires it. - -Here are the commands to run unit tests and check code style. - -``` -# Run all unit tests -# This must be executed in directory perfzero/lib -python3 -B -m unittest discover -p "*_test.py" - -# Format python code in place -find perfzero/lib -name *.py -exec pyformat --in_place {} \; - -# Check python code format and report warning and errors -find perfzero/lib -name *.py -exec gpylint3 {} \; -``` - -Here is the command to generate table-of-contents for this README. Run this -command and copy/paste it to the README.md. - -``` -./perfzero/scripts/generate-readme-header.sh perfzero/README.md -``` diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_build b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_build deleted file mode 100644 index 2a84858a49f3d3967823f030aac3041c9877b188..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_build +++ /dev/null @@ -1,97 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 10 and the following: -# - Installs tf-nightly-gpu-2.0-preview -# - Installs requirements.txt for tensorflow/models -# - Install bazel for building TF from source - -FROM nvidia/cuda:10.0-base-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu-2.0-preview" -ARG extra_pip_specs="" -ARG local_tensorflow_pip_spec="" - -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cuda-command-line-tools-10-0 \ - cuda-cublas-dev-10-0 \ - cuda-cufft-dev-10-0 \ - cuda-curand-dev-10-0 \ - cuda-cusolver-dev-10-0 \ - cuda-cusparse-dev-10-0 \ - libcudnn7=7.6.2.24-1+cuda10.0 \ - libcudnn7-dev=7.6.2.24-1+cuda10.0 \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl \ - && \ - find /usr/local/cuda-10.0/lib64/ -type f -name 'lib*_static.a' -not -name 'libcudart_static.a' -delete && \ - rm /usr/lib/x86_64-linux-gnu/libcudnn_static_v7.a - -RUN apt-get update && \ - apt-get install -y --no-install-recommends libnvinfer5=5.1.5-1+cuda10.0 \ - libnvinfer-dev=5.1.5-1+cuda10.0 \ - && apt-get clean - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - build-essential \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -# Install / update Python -# (bulding TF needs py2 even if building for Python3 as of 06-AUG-2019) -RUN apt-get install -y --no-install-recommends \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-venv \ - python - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN pip3 install --upgrade pip -# setuptools upgraded to fix install requirements from model garden. -RUN pip install wheel -RUN pip install --upgrade setuptools google-api-python-client pyyaml google-cloud google-cloud-bigquery google-cloud-datastore mock -RUN pip install absl-py -RUN pip install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} -RUN pip install tfds-nightly -RUN pip install -U scikit-learn - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN pip3 install -r /tmp/requirements.txt - -RUN pip3 freeze - -# Install bazel -ARG BAZEL_VERSION=0.24.1 -RUN mkdir /bazel && \ - wget -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \ - wget -O /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \ - chmod +x /bazel/installer.sh && \ - /bazel/installer.sh && \ - rm -f /bazel/installer.sh - -RUN git clone https://github.com/tensorflow/tensorflow.git /tensorflow_src diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_s4tf_cuda10.0 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_s4tf_cuda10.0 deleted file mode 100644 index e454f97607330af178194789172b3c8303223e3a..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_s4tf_cuda10.0 +++ /dev/null @@ -1,124 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 10 and the following: -# - Installs tf-nightly-gpu (this is TF 2.0) -# - Installs requirements.txt for tensorflow/models -# Additionally also installs: -# - Latest S4TF development snapshot for cuda 10.0 - -FROM nvidia/cuda:10.0-base-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG local_tensorflow_pip_spec="" -ARG extra_pip_specs="" -ARG swift_tf_url=https://storage.googleapis.com/swift-tensorflow-artifacts/nightlies/latest/swift-tensorflow-DEVELOPMENT-cuda10.0-cudnn7-ubuntu18.04.tar.gz - -# setup.py passes the base path of local .whl file is chosen for the docker image. -# Otherwise passes an empty existing file from the context. -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn7-dev only needed because of libnvinfer-dev which may not -# really be needed. -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cuda-command-line-tools-10-0 \ - cuda-cublas-10-0 \ - cuda-cublas-dev-10-0 \ - cuda-cufft-10-0 \ - cuda-curand-10-0 \ - cuda-cusolver-10-0 \ - cuda-cusparse-10-0 \ - libcudnn7=7.6.2.24-1+cuda10.0 \ - libcudnn7-dev=7.6.2.24-1+cuda10.0 \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -RUN apt-get update && \ - apt-get install -y --no-install-recommends libnvinfer5=5.1.5-1+cuda10.0 \ - libnvinfer-dev=5.1.5-1+cuda10.0 \ - && apt-get clean - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-venv - - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN pip3 install --upgrade pip -# setuptools upgraded to fix install requirements from model garden. -RUN pip install wheel -RUN pip install --upgrade setuptools google-api-python-client pyyaml google-cloud google-cloud-bigquery mock -RUN pip install absl-py -RUN pip install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} -RUN pip install tfds-nightly -RUN pip install -U scikit-learn - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt - -RUN pip freeze - -### Install Swift deps. -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - curl \ - git \ - python \ - python-dev \ - python-pip \ - python-setuptools \ - python-tk \ - python3 \ - python3-pip \ - python3-setuptools \ - clang \ - libcurl4-openssl-dev \ - libicu-dev \ - libpython-dev \ - libpython3-dev \ - libncurses5-dev \ - libxml2 \ - libblocksruntime-dev - -# Download and extract S4TF -WORKDIR /swift-tensorflow-toolchain -RUN if ! curl -fSsL --retry 5 $swift_tf_url -o swift.tar.gz; \ - then sleep 30 && curl -fSsL --retry 5 $swift_tf_url -o swift.tar.gz; \ - fi; - -RUN mkdir usr \ - && tar -xzf swift.tar.gz --directory=usr --strip-components=1 \ - && rm swift.tar.gz -ENV PATH="/swift-tensorflow-toolchain/usr/bin:${PATH}" -ENV LD_LIBRARY_PATH="/swift-tensorflow-toolchain/usr/lib/swift/linux/:${LD_LIBRARY_PATH}" diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_s4tf_cuda10.1 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_s4tf_cuda10.1 deleted file mode 100644 index bb7df8eddbfa7bb90635eafd74b8c2bec1368358..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_s4tf_cuda10.1 +++ /dev/null @@ -1,133 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 10.1 and the following: -# - Installs tf-nightly-gpu (this is TF 2.1) -# - Installs requirements.txt for tensorflow/models -# - TF 2.0 tested with cuda 10.0, but we need to test tf 2.1 with cuda 10.1. -# Additionally also installs -# - Latest S4TF development snapshot for cuda 10.1 - -FROM nvidia/cuda:10.1-base-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG local_tensorflow_pip_spec="" -ARG extra_pip_specs="" -ARG swift_tf_url=https://storage.googleapis.com/swift-tensorflow-artifacts/nightlies/latest/swift-tensorflow-DEVELOPMENT-cuda10.1-cudnn7-stock-ubuntu18.04.tar.gz - -# setup.py passes the base path of local .whl file is chosen for the docker image. -# Otherwise passes an empty existing file from the context. -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn7-dev only needed because of libnvinfer-dev which may not -# really be needed. -# In the future, add the following lines in a shell script running on the -# benchmark vm to get the available dependent versions when updating cuda -# version (e.g to 10.2 or something later): -# sudo apt-cache search cuda-command-line-tool -# sudo apt-cache search cuda-cublas -# sudo apt-cache search cuda-cufft -# sudo apt-cache search cuda-curand -# sudo apt-cache search cuda-cusolver -# sudo apt-cache search cuda-cusparse - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cuda-command-line-tools-10-1 \ - cuda-cufft-10-1 \ - cuda-curand-10-1 \ - cuda-cusolver-10-1 \ - cuda-cusparse-10-1 \ - libcudnn7=7.6.4.38-1+cuda10.1 \ - libcudnn7-dev=7.6.4.38-1+cuda10.1 \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -RUN apt-get update && \ - apt-get install -y --no-install-recommends libnvinfer5=5.1.5-1+cuda10.1 \ - libnvinfer-dev=5.1.5-1+cuda10.1 \ - libnvinfer6=6.0.1-1+cuda10.1 \ - && apt-get clean - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-venv - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN pip3 install --upgrade pip -# setuptools upgraded to fix install requirements from model garden. -RUN pip install wheel -RUN pip install --upgrade setuptools google-api-python-client pyyaml google-cloud google-cloud-bigquery google-cloud-datastore mock -RUN pip install absl-py -RUN pip install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} -RUN pip install tfds-nightly -RUN pip install -U scikit-learn - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt - -RUN pip freeze - -### Install Swift deps. -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - curl \ - git \ - python \ - python-dev \ - python-pip \ - python-setuptools \ - python-tk \ - python3 \ - python3-pip \ - python3-setuptools \ - clang \ - libcurl4-openssl-dev \ - libicu-dev \ - libpython-dev \ - libpython3-dev \ - libncurses5-dev \ - libxml2 \ - libblocksruntime-dev - -# Download and extract S4TF -WORKDIR /swift-tensorflow-toolchain -RUN if ! curl -fSsL --retry 5 $swift_tf_url -o swift.tar.gz; \ - then sleep 30 && curl -fSsL --retry 5 $swift_tf_url -o swift.tar.gz; \ - fi; - -RUN mkdir usr \ - && tar -xzf swift.tar.gz --directory=usr --strip-components=1 \ - && rm swift.tar.gz -ENV PATH="/swift-tensorflow-toolchain/usr/bin:${PATH}" -ENV LD_LIBRARY_PATH="/swift-tensorflow-toolchain/usr/lib/swift/linux/:${LD_LIBRARY_PATH}" diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_s4tf_cuda11.0 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_s4tf_cuda11.0 deleted file mode 100644 index ee328fd64f9d24cbf44462137e4909a4e5efa8e2..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_s4tf_cuda11.0 +++ /dev/null @@ -1,128 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 11.0 and the following: -# - Installs tf-nightly-gpu (this is TF 2.4) -# - Installs requirements.txt for tensorflow/models -# Additionally also installs -# - Latest S4TF development snapshot for cuda 11.0 - -FROM nvidia/cuda:11.0-base-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG local_tensorflow_pip_spec="" -ARG extra_pip_specs="" -ARG swift_tf_url=https://storage.googleapis.com/swift-tensorflow-artifacts/nightlies/latest/swift-tensorflow-DEVELOPMENT-cuda11.0-cudnn8-stock-ubuntu18.04.tar.gz - -# setup.py passes the base path of local .whl file is chosen for the docker image. -# Otherwise passes an empty existing file from the context. -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn8-dev only needed because of libnvinfer-dev which may not -# really be needed. -# In the future, add the following lines in a shell script running on the -# benchmark vm to get the available dependent versions when updating cuda -# version (e.g to 10.2 or something later): -# sudo apt-cache search cuda-command-line-tool -# sudo apt-cache search cuda-cublas -# sudo apt-cache search cuda-cufft -# sudo apt-cache search cuda-curand -# sudo apt-cache search cuda-cusolver -# sudo apt-cache search cuda-cusparse - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cuda-tools-11-0 \ - cuda-toolkit-11-0 \ - libcudnn8=8.0.4.30-1+cuda11.0 \ - libcudnn8-dev=8.0.4.30-1+cuda11.0 \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -RUN apt-get update && \ - apt-get install -y --no-install-recommends libnvinfer7=7.2.0-1+cuda11.0 \ - libnvinfer-dev=7.2.0-1+cuda11.0 \ - && apt-get clean - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-venv - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN pip3 install --upgrade pip -# setuptools upgraded to fix install requirements from model garden. -RUN pip install wheel -RUN pip install --upgrade setuptools google-api-python-client pyyaml google-cloud google-cloud-bigquery google-cloud-datastore mock -RUN pip install absl-py -RUN pip install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} -RUN pip install tfds-nightly -RUN pip install -U scikit-learn - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt - -RUN pip freeze - -### Install Swift deps. -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - curl \ - git \ - python \ - python-dev \ - python-pip \ - python-setuptools \ - python-tk \ - python3 \ - python3-pip \ - python3-setuptools \ - clang \ - libcurl4-openssl-dev \ - libicu-dev \ - libpython-dev \ - libpython3-dev \ - libncurses5-dev \ - libxml2 \ - libblocksruntime-dev - -# Download and extract S4TF -WORKDIR /swift-tensorflow-toolchain -RUN if ! curl -fSsL --retry 5 $swift_tf_url -o swift.tar.gz; \ - then sleep 30 && curl -fSsL --retry 5 $swift_tf_url -o swift.tar.gz; \ - fi; - -RUN mkdir usr \ - && tar -xzf swift.tar.gz --directory=usr --strip-components=1 \ - && rm swift.tar.gz -ENV PATH="/swift-tensorflow-toolchain/usr/bin:${PATH}" -ENV LD_LIBRARY_PATH="/swift-tensorflow-toolchain/usr/lib/swift/linux/:${LD_LIBRARY_PATH}" diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_testing_apiclient b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_testing_apiclient deleted file mode 100644 index 0b8eadf400880dfbe729ddcdaaed6c326fe1c385..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_testing_apiclient +++ /dev/null @@ -1,94 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 10.1 and the following: -# - Installs tf-nightly-gpu (this is TF 2.1) -# - Installs requirements.txt for tensorflow/models -# - TF 2.0 tested with cuda 10.0, but we need to test tf 2.1 with cuda 10.1. - -FROM nvidia/cuda:10.1-base-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG local_tensorflow_pip_spec="" -ARG extra_pip_specs="" - -# setup.py passes the base path of local .whl file is chosen for the docker image. -# Otherwise passes an empty existing file from the context. -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn7-dev only needed because of libnvinfer-dev which may not -# really be needed. -# In the future, add the following lines in a shell script running on the -# benchmark vm to get the available dependent versions when updating cuda -# version (e.g to 10.2 or something later): -# sudo apt-cache search cuda-command-line-tool -# sudo apt-cache search cuda-cublas -# sudo apt-cache search cuda-cufft -# sudo apt-cache search cuda-curand -# sudo apt-cache search cuda-cusolver -# sudo apt-cache search cuda-cusparse - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cuda-command-line-tools-10-1 \ - cuda-cufft-10-1 \ - cuda-curand-10-1 \ - cuda-cusolver-10-1 \ - cuda-cusparse-10-1 \ - libcudnn7=7.6.4.38-1+cuda10.1 \ - libcudnn7-dev=7.6.4.38-1+cuda10.1 \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -RUN apt-get update && \ - apt-get install -y --no-install-recommends libnvinfer5=5.1.5-1+cuda10.1 \ - libnvinfer-dev=5.1.5-1+cuda10.1 \ - libnvinfer6=6.0.1-1+cuda10.1 \ - && apt-get clean - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-venv - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN pip3 install --upgrade pip -# setuptools upgraded to fix install requirements from model garden. -RUN pip install wheel -RUN pip install --upgrade setuptools google-api-python-client==1.8.0 pyyaml google-cloud google-cloud-bigquery mock -RUN pip install absl-py -RUN pip install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} -RUN pip install tfds-nightly -RUN pip install -U scikit-learn - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt - -RUN pip freeze diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_11 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_11 deleted file mode 100644 index 4385d95a5d222502dd680b922d1b2b4af5645a50..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_11 +++ /dev/null @@ -1,109 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 11 and the following: -# - Installs tf-nightly-gpu (this is TF 2.3) -# - Installs requirements.txt for tensorflow/models - -FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG local_tensorflow_pip_spec="" -ARG extra_pip_specs="" -ENV PIP_CMD="python3.9 -m pip" - -# setup.py passes the base path of local .whl file is chosen for the docker image. -# Otherwise passes an empty existing file from the context. -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn7-dev only needed because of libnvinfer-dev which may not -# really be needed. -# In the future, add the following lines in a shell script running on the -# benchmark vm to get the available dependent versions when updating cuda -# version (e.g to 10.2 or something later): -# sudo apt-cache search cuda-command-line-tool -# sudo apt-cache search cuda-cublas -# sudo apt-cache search cuda-cufft -# sudo apt-cache search cuda-curand -# sudo apt-cache search cuda-cusolver -# sudo apt-cache search cuda-cusparse - -# Needed to disable prompts during installation. -ENV DEBIAN_FRONTEND noninteractive - - -RUN apt-get update && apt-get install -y --no-install-recommends \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -# Python 3.9 related deps in this ppa. -RUN add-apt-repository ppa:deadsnakes/ppa - - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3.9 \ - python3-pip \ - python3.9-dev \ - python3-setuptools \ - python3.9-venv \ - python3.9-distutils \ - python3.9-lib2to3 - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN ${PIP_CMD} install --upgrade pip -RUN ${PIP_CMD} install --upgrade distlib -# setuptools upgraded to fix install requirements from model garden. -RUN ${PIP_CMD} install --upgrade setuptools - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda-11.2/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -RUN ${PIP_CMD} install --upgrade pyyaml -RUN ${PIP_CMD} install --upgrade google-api-python-client==1.8.0 -RUN ${PIP_CMD} install --upgrade google-cloud google-cloud-bigquery google-cloud-datastore mock - - -RUN ${PIP_CMD} install wheel -RUN ${PIP_CMD} install absl-py -RUN ${PIP_CMD} install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} - -RUN ${PIP_CMD} install tfds-nightly -RUN ${PIP_CMD} install -U scikit-learn - -# Install dependnecies needed for tf.distribute test utils -RUN ${PIP_CMD} install dill tblib portpicker - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN ${PIP_CMD} install -r /tmp/requirements.txt - -RUN ${PIP_CMD} install tf-estimator-nightly -RUN ${PIP_CMD} install tensorflow-text-nightly - -# RUN nvidia-smi - -RUN nvcc --version - - -RUN pip freeze diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_11_0 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_11_0 deleted file mode 100644 index 2ad70524f58eb9ad11ee29ffd97c34d679da84de..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_11_0 +++ /dev/null @@ -1,94 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 11 and the following: -# - Installs tf-nightly-gpu (this is TF 2.3) -# - Installs requirements.txt for tensorflow/models - -FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG local_tensorflow_pip_spec="" -ARG extra_pip_specs="" - -# setup.py passes the base path of local .whl file is chosen for the docker image. -# Otherwise passes an empty existing file from the context. -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn7-dev only needed because of libnvinfer-dev which may not -# really be needed. -# In the future, add the following lines in a shell script running on the -# benchmark vm to get the available dependent versions when updating cuda -# version (e.g to 10.2 or something later): -# sudo apt-cache search cuda-command-line-tool -# sudo apt-cache search cuda-cublas -# sudo apt-cache search cuda-cufft -# sudo apt-cache search cuda-curand -# sudo apt-cache search cuda-cusolver -# sudo apt-cache search cuda-cusparse - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cuda-tools-11-0 \ - cuda-toolkit-11-0 \ - libcudnn8=8.0.4.30-1+cuda11.0 \ - libcudnn8-dev=8.0.4.30-1+cuda11.0 \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda-11.0/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-venv - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN pip3 install --upgrade pip -# setuptools upgraded to fix install requirements from model garden. -RUN pip install wheel -RUN pip install --upgrade setuptools google-api-python-client==1.8.0 pyyaml google-cloud google-cloud-bigquery google-cloud-datastore mock -RUN pip install absl-py -RUN pip install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} -RUN pip install tfds-nightly -RUN pip install -U scikit-learn -# Install dependnecies needed for tf.distribute test utils -RUN pip install dill tblib portpicker - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt - -RUN pip install tf-estimator-nightly -RUN pip install tensorflow-text-nightly - -# RUN nvidia-smi - -RUN nvcc --version - - -RUN pip freeze diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_11_py36 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_11_py36 deleted file mode 100644 index 959809412501766533ebc1e2079d1dcf09ba2195..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_11_py36 +++ /dev/null @@ -1,90 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 11 and the following: -# - Installs tf-nightly-gpu (this is TF 2.3) -# - Installs requirements.txt for tensorflow/models - -FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG local_tensorflow_pip_spec="" -ARG extra_pip_specs="" - -# setup.py passes the base path of local .whl file is chosen for the docker image. -# Otherwise passes an empty existing file from the context. -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn7-dev only needed because of libnvinfer-dev which may not -# really be needed. -# In the future, add the following lines in a shell script running on the -# benchmark vm to get the available dependent versions when updating cuda -# version (e.g to 10.2 or something later): -# sudo apt-cache search cuda-command-line-tool -# sudo apt-cache search cuda-cublas -# sudo apt-cache search cuda-cufft -# sudo apt-cache search cuda-curand -# sudo apt-cache search cuda-cusolver -# sudo apt-cache search cuda-cusparse - -RUN apt-get update && apt-get install -y --no-install-recommends \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda-11.2/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-venv - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN pip3 install --upgrade pip -# setuptools upgraded to fix install requirements from model garden. -RUN pip install wheel -RUN pip install --upgrade setuptools google-api-python-client==1.8.0 pyyaml google-cloud google-cloud-bigquery google-cloud-datastore mock -RUN pip install absl-py -RUN pip install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} -RUN pip install tfds-nightly -RUN pip install -U scikit-learn -# Install dependnecies needed for tf.distribute test utils -RUN pip install dill tblib portpicker - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt - -RUN pip install tf-estimator-nightly -RUN pip install tensorflow-text-nightly -RUN pip install psutil - -# RUN nvidia-smi - -RUN nvcc --version - - -RUN pip freeze diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_11_rollback b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_11_rollback deleted file mode 100644 index 0bc1769e17fedc65fd85fdad8b314a57bffdb856..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_11_rollback +++ /dev/null @@ -1,110 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 11 and the following: -# - Installs tf-nightly-gpu (this is TF 2.3) -# - Installs requirements.txt for tensorflow/models - -FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG local_tensorflow_pip_spec="" -ARG extra_pip_specs="" -ENV PIP_CMD="python3.9 -m pip" - -# setup.py passes the base path of local .whl file is chosen for the docker image. -# Otherwise passes an empty existing file from the context. -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn7-dev only needed because of libnvinfer-dev which may not -# really be needed. -# In the future, add the following lines in a shell script running on the -# benchmark vm to get the available dependent versions when updating cuda -# version (e.g to 10.2 or something later): -# sudo apt-cache search cuda-command-line-tool -# sudo apt-cache search cuda-cublas -# sudo apt-cache search cuda-cufft -# sudo apt-cache search cuda-curand -# sudo apt-cache search cuda-cusolver -# sudo apt-cache search cuda-cusparse - -# Needed to disable prompts during installation. -ENV DEBIAN_FRONTEND noninteractive - - -RUN apt-get update && apt-get install -y --no-install-recommends \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -# Python 3.9 related deps in this ppa. -RUN add-apt-repository ppa:deadsnakes/ppa - - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3.9 \ - python3-pip \ - python3.9-dev \ - python3-setuptools \ - python3.9-venv \ - python3.9-distutils \ - python3.9-lib2to3 - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN ${PIP_CMD} install --upgrade pip -RUN ${PIP_CMD} install --upgrade distlib -# setuptools upgraded to fix install requirements from model garden. -RUN ${PIP_CMD} install --upgrade setuptools - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda-11.2/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -RUN ${PIP_CMD} install --upgrade pyyaml -RUN ${PIP_CMD} install --upgrade google-api-python-client==1.8.0 -RUN ${PIP_CMD} install --upgrade google-cloud google-cloud-bigquery google-cloud-datastore mock - - -RUN ${PIP_CMD} install wheel -RUN ${PIP_CMD} install absl-py -RUN ${PIP_CMD} install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} - -RUN ${PIP_CMD} install tfds-nightly -RUN ${PIP_CMD} install -U scikit-learn - -# Install dependnecies needed for tf.distribute test utils -RUN ${PIP_CMD} install dill tblib portpicker - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN ${PIP_CMD} install -r /tmp/requirements.txt - -RUN ${PIP_CMD} install tf-estimator-nightly -RUN ${PIP_CMD} install tensorflow-text-nightly -RUN ${PIP_CMD} install keras-nightly==2.7.0.dev2021082607 - -# RUN nvidia-smi - -RUN nvcc --version - - -RUN pip freeze diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_11_rollback_2 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_11_rollback_2 deleted file mode 100644 index a9ba568d6ff99adabc9ca7e4bc7eaebf27e695be..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_11_rollback_2 +++ /dev/null @@ -1,110 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 11 and the following: -# - Installs tf-nightly-gpu (this is TF 2.3) -# - Installs requirements.txt for tensorflow/models - -FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG local_tensorflow_pip_spec="" -ARG extra_pip_specs="" -ENV PIP_CMD="python3.9 -m pip" - -# setup.py passes the base path of local .whl file is chosen for the docker image. -# Otherwise passes an empty existing file from the context. -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn7-dev only needed because of libnvinfer-dev which may not -# really be needed. -# In the future, add the following lines in a shell script running on the -# benchmark vm to get the available dependent versions when updating cuda -# version (e.g to 10.2 or something later): -# sudo apt-cache search cuda-command-line-tool -# sudo apt-cache search cuda-cublas -# sudo apt-cache search cuda-cufft -# sudo apt-cache search cuda-curand -# sudo apt-cache search cuda-cusolver -# sudo apt-cache search cuda-cusparse - -# Needed to disable prompts during installation. -ENV DEBIAN_FRONTEND noninteractive - - -RUN apt-get update && apt-get install -y --no-install-recommends \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -# Python 3.9 related deps in this ppa. -RUN add-apt-repository ppa:deadsnakes/ppa - - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3.9 \ - python3-pip \ - python3.9-dev \ - python3-setuptools \ - python3.9-venv \ - python3.9-distutils \ - python3.9-lib2to3 - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN ${PIP_CMD} install --upgrade pip -RUN ${PIP_CMD} install --upgrade distlib -# setuptools upgraded to fix install requirements from model garden. -RUN ${PIP_CMD} install --upgrade setuptools - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda-11.2/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -RUN ${PIP_CMD} install --upgrade pyyaml -RUN ${PIP_CMD} install --upgrade google-api-python-client==1.8.0 -RUN ${PIP_CMD} install --upgrade google-cloud google-cloud-bigquery google-cloud-datastore mock - - -RUN ${PIP_CMD} install wheel -RUN ${PIP_CMD} install absl-py -RUN ${PIP_CMD} install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} - -RUN ${PIP_CMD} install tfds-nightly -RUN ${PIP_CMD} install -U scikit-learn - -# Install dependnecies needed for tf.distribute test utils -RUN ${PIP_CMD} install dill tblib portpicker - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN ${PIP_CMD} install -r /tmp/requirements.txt - -RUN ${PIP_CMD} install tf-estimator-nightly -RUN ${PIP_CMD} install tensorflow-text-nightly -RUN ${PIP_CMD} install keras-nightly==2.7.0.dev2021070900 - -# RUN nvidia-smi - -RUN nvcc --version - - -RUN pip freeze diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_testing b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_testing deleted file mode 100644 index 2ad70524f58eb9ad11ee29ffd97c34d679da84de..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_cuda_testing +++ /dev/null @@ -1,94 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 11 and the following: -# - Installs tf-nightly-gpu (this is TF 2.3) -# - Installs requirements.txt for tensorflow/models - -FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG local_tensorflow_pip_spec="" -ARG extra_pip_specs="" - -# setup.py passes the base path of local .whl file is chosen for the docker image. -# Otherwise passes an empty existing file from the context. -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn7-dev only needed because of libnvinfer-dev which may not -# really be needed. -# In the future, add the following lines in a shell script running on the -# benchmark vm to get the available dependent versions when updating cuda -# version (e.g to 10.2 or something later): -# sudo apt-cache search cuda-command-line-tool -# sudo apt-cache search cuda-cublas -# sudo apt-cache search cuda-cufft -# sudo apt-cache search cuda-curand -# sudo apt-cache search cuda-cusolver -# sudo apt-cache search cuda-cusparse - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cuda-tools-11-0 \ - cuda-toolkit-11-0 \ - libcudnn8=8.0.4.30-1+cuda11.0 \ - libcudnn8-dev=8.0.4.30-1+cuda11.0 \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda-11.0/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-venv - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN pip3 install --upgrade pip -# setuptools upgraded to fix install requirements from model garden. -RUN pip install wheel -RUN pip install --upgrade setuptools google-api-python-client==1.8.0 pyyaml google-cloud google-cloud-bigquery google-cloud-datastore mock -RUN pip install absl-py -RUN pip install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} -RUN pip install tfds-nightly -RUN pip install -U scikit-learn -# Install dependnecies needed for tf.distribute test utils -RUN pip install dill tblib portpicker - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt - -RUN pip install tf-estimator-nightly -RUN pip install tensorflow-text-nightly - -# RUN nvidia-smi - -RUN nvcc --version - - -RUN pip freeze diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_custom_pip b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_custom_pip deleted file mode 100644 index 69abee6e4d6edbd44e2008fd070036786620c1dd..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_custom_pip +++ /dev/null @@ -1,88 +0,0 @@ -# Ubuntu 18.04 Python3.6 with CUDA 10 and the following: -# - Installs custom TensorFlow pip package -# - Installs requirements.txt for tensorflow/models - -# NOTE: Branched from Dockerfile_ubuntu_1804_tf_v1 with changes relevant to -# tensorflow_pip_spec. When updating please keep the difference minimal. - -FROM nvidia/cuda:10.0-base-ubuntu18.04 as base - -# Location of custom TF pip package, must be relative to docker context. -# Note that the version tag in the name of wheel file is meaningless. -ARG tensorflow_pip_spec="resources/tensorflow-0.0.1-cp36-cp36m-linux_x86_64.whl" -ARG extra_pip_specs="" -ARG local_tensorflow_pip_spec="" - -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -COPY ${tensorflow_pip_spec} /tensorflow-0.0.1-cp36-cp36m-linux_x86_64.whl - -# Pick up some TF dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cuda-command-line-tools-10-0 \ - cuda-cublas-10-0 \ - cuda-cufft-10-0 \ - cuda-curand-10-0 \ - cuda-cusolver-10-0 \ - cuda-cusparse-10-0 \ - libcudnn7=7.4.1.5-1+cuda10.0 \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -RUN apt-get update && \ - apt-get install nvinfer-runtime-trt-repo-ubuntu1804-5.0.2-ga-cuda10.0 \ - && apt-get update \ - && apt-get install -y --no-install-recommends libnvinfer5=5.0.2-1+cuda10.0 \ - && apt-get clean - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - build-essential \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-venv - - -# Setup Python3 environment -RUN pip3 install --upgrade pip==9.0.1 -# setuptools upgraded to fix install requirements from model garden. -RUN pip3 install wheel -RUN pip3 install --upgrade setuptools google-api-python-client pyyaml google-cloud google-cloud-bigquery -RUN pip3 install absl-py -RUN pip3 install --upgrade --force-reinstall /tensorflow-0.0.1-cp36-cp36m-linux_x86_64.whl ${extra_pip_specs} -RUN pip3 install tfds-nightly -RUN pip3 install -U scikit-learn - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN pip3 install -r /tmp/requirements.txt - -RUN pip3 freeze diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_v1 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_v1 deleted file mode 100644 index 9e89c1bb2a0c941c81712d41c0ea315b7e378dbf..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_v1 +++ /dev/null @@ -1,88 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 10 and the following: -# - Installs tf-nightly-gpu -# - Installs requirements.txt for tensorflow/models -# -# This docker is not needed and is the same as the tf_v2 docker. The -# User can pass in the desired `ARG tensorflow_pip_spec` Remove -# one TF 1.0 testing is done or KOKORO jobs are updated to use the -# tensorfow_pip_spec rather than docker path to control TF version. - -FROM nvidia/cuda:10.0-base-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG extra_pip_specs="" -ARG local_tensorflow_pip_spec="" - -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn7-dev only needed because of libnvinfer-dev which may not -# really be needed. -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cuda-command-line-tools-10-0 \ - cuda-cublas-10-0 \ - cuda-cublas-dev-10-0 \ - cuda-cufft-10-0 \ - cuda-curand-10-0 \ - cuda-cusolver-10-0 \ - cuda-cusparse-10-0 \ - libcudnn7=7.6.0.64-1+cuda10.0 \ - libcudnn7-dev=7.6.0.64-1+cuda10.0 \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -RUN apt-get update && \ - apt-get install -y --no-install-recommends libnvinfer5=5.1.5-1+cuda10.0 \ - libnvinfer-dev=5.1.5-1+cuda10.0 \ - && apt-get clean - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-venv - - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN pip3 install --upgrade pip -# setuptools upgraded to fix install requirements from model garden. -RUN pip install wheel -RUN pip install --upgrade setuptools google-api-python-client pyyaml google-cloud google-cloud-bigquery mock -RUN pip install absl-py -RUN pip install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} -RUN pip install tfds-nightly -RUN pip install -U scikit-learn - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt - -RUN pip freeze diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_v2 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_v2 deleted file mode 100644 index 7bd56a37d4333a2f337270d4b45c0a7b990bd7de..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_v2 +++ /dev/null @@ -1,85 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 10 and the following: -# - Installs tf-nightly-gpu (this is TF 2.0) -# - Installs requirements.txt for tensorflow/models - -FROM nvidia/cuda:10.0-base-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG local_tensorflow_pip_spec="" -ARG extra_pip_specs="" - -# setup.py passes the base path of local .whl file is chosen for the docker image. -# Otherwise passes an empty existing file from the context. -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn7-dev only needed because of libnvinfer-dev which may not -# really be needed. -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cuda-command-line-tools-10-0 \ - cuda-cublas-10-0 \ - cuda-cublas-dev-10-0 \ - cuda-cufft-10-0 \ - cuda-curand-10-0 \ - cuda-cusolver-10-0 \ - cuda-cusparse-10-0 \ - libcudnn7=7.6.2.24-1+cuda10.0 \ - libcudnn7-dev=7.6.2.24-1+cuda10.0 \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -RUN apt-get update && \ - apt-get install -y --no-install-recommends libnvinfer5=5.1.5-1+cuda10.0 \ - libnvinfer-dev=5.1.5-1+cuda10.0 \ - && apt-get clean - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-venv - - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN pip3 install --upgrade pip -# setuptools upgraded to fix install requirements from model garden. -RUN pip install wheel -RUN pip install --upgrade setuptools google-api-python-client pyyaml google-cloud google-cloud-bigquery mock -RUN pip install absl-py -RUN pip install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} -RUN pip install tfds-nightly -RUN pip install -U scikit-learn - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt - -RUN pip freeze diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_v2_1 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_v2_1 deleted file mode 100644 index 82016b03d7fcf1748d00cd289210260cfe2af5de..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tf_v2_1 +++ /dev/null @@ -1,96 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 10.1 and the following: -# - Installs tf-nightly-gpu (this is TF 2.1) -# - Installs requirements.txt for tensorflow/models -# - TF 2.0 tested with cuda 10.0, but we need to test tf 2.1 with cuda 10.1. - -FROM nvidia/cuda:10.1-base-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG local_tensorflow_pip_spec="" -ARG extra_pip_specs="" - -# setup.py passes the base path of local .whl file is chosen for the docker image. -# Otherwise passes an empty existing file from the context. -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn7-dev only needed because of libnvinfer-dev which may not -# really be needed. -# In the future, add the following lines in a shell script running on the -# benchmark vm to get the available dependent versions when updating cuda -# version (e.g to 10.2 or something later): -# sudo apt-cache search cuda-command-line-tool -# sudo apt-cache search cuda-cublas -# sudo apt-cache search cuda-cufft -# sudo apt-cache search cuda-curand -# sudo apt-cache search cuda-cusolver -# sudo apt-cache search cuda-cusparse - -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cuda-command-line-tools-10-1 \ - cuda-cufft-10-1 \ - cuda-curand-10-1 \ - cuda-cusolver-10-1 \ - cuda-cusparse-10-1 \ - libcudnn7=7.6.4.38-1+cuda10.1 \ - libcudnn7-dev=7.6.4.38-1+cuda10.1 \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -RUN apt-get update && \ - apt-get install -y --no-install-recommends libnvinfer5=5.1.5-1+cuda10.1 \ - libnvinfer-dev=5.1.5-1+cuda10.1 \ - libnvinfer6=6.0.1-1+cuda10.1 \ - && apt-get clean - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-venv - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN pip3 install --upgrade pip -# setuptools upgraded to fix install requirements from model garden. -RUN pip install wheel -RUN pip install --upgrade setuptools google-api-python-client==1.8.0 pyyaml google-cloud google-cloud-bigquery google-cloud-datastore mock -RUN pip install absl-py -RUN pip install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} -RUN pip install tfds-nightly -RUN pip install -U scikit-learn -# Install dependnecies needed for tf.distribute test utils -RUN pip install dill tblib portpicker - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt - -RUN pip freeze diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tfx b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tfx deleted file mode 100644 index 6f6aac3ad1537e6a01aa8547153e9b636780b3f3..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_1804_tfx +++ /dev/null @@ -1,262 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 10 and the following: -# - Installs tf-nightly-gpu (this is TF 2.0) -# - Installs requirements.txt for tensorflow/models -# -# NOTE: Branched from Dockerfile_ubuntu_1804_tf_v2 with changes for -# TFX benchmarks. - -FROM nvidia/cuda:10.0-base-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG local_tensorflow_pip_spec="" -ARG extra_pip_specs="" - -# Specifies the default package version to use if no corresponding commit_id -# override is specified. -# If "head", uses the GitHub HEAD version. -# If "release", uses the latest released version from PyPI, REGARDLESS of -# package-compatibility requirements (e.g. even if tfx requires -# tensorflow-model-analysis<0.22, if tensorflow-model-analysis==0.22.0 is -# the latest released version on PyPI, we will install that). -# Packages include: tfx, tensorflow-transform, tensorflow-model-analysis, -# tensorflow-data-validation, tensorflow-metadata, tfx-bsl -ARG default_package_version="head" - -# Specifies the package version to use for the corresponding packages. -# If empty, uses the default specified by default_package_version. -# If "head", uses the GitHub HEAD version. -# If "release", uses the latest released version from PyPI, REGARDLESS of -# package-compatibility requirements. -# If "github_commit:", uses the given commit ID from GitHub. -# If "github_tag:" uses the given tag from GitHub. -# If "pypi:", uses the given package version from PyPI. -ARG tfx_package_version="" -ARG tensorflow_transform_package_version="" -ARG tensorflow_model_analysis_package_version="" -ARG tensorflow_data_validation_package_version="" -ARG tensorflow_metadata_package_version="" -ARG tfx_bsl_version="" - -# setup.py passes the base path of local .whl file is chosen for the docker image. -# Otherwise passes an empty existing file from the context. -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn7-dev only needed because of libnvinfer-dev which may not -# really be needed. -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cuda-command-line-tools-10-0 \ - cuda-cublas-10-0 \ - cuda-cublas-dev-10-0 \ - cuda-cufft-10-0 \ - cuda-curand-10-0 \ - cuda-cusolver-10-0 \ - cuda-cusparse-10-0 \ - libcudnn7=7.6.2.24-1+cuda10.0 \ - libcudnn7-dev=7.6.2.24-1+cuda10.0 \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -RUN apt-get update && \ - apt-get install -y --no-install-recommends libnvinfer5=5.1.5-1+cuda10.0 \ - libnvinfer-dev=5.1.5-1+cuda10.0 \ - && apt-get clean - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-venv - - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN pip3 install --upgrade pip -# setuptools upgraded to fix install requirements from model garden. -RUN pip install wheel -RUN pip install --upgrade setuptools google-api-python-client pyyaml google-cloud google-cloud-bigquery google-cloud-datastore mock -RUN pip install absl-py -RUN if [ ! -z "${extra_pip_specs}" ]; then pip install --upgrade --force-reinstall ${extra_pip_specs}; fi -RUN pip install tfds-nightly -RUN pip install -U scikit-learn - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt - -# Install yolk3k, for getting package versions from PyPI (so we can pull -# TFX from GitHub even when we need to install from the released version) -RUN pip install yolk3k - -# Install protoc -RUN PROTOC_ZIP=protoc-3.7.1-linux-x86_64.zip; \ - curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/$PROTOC_ZIP; \ - unzip -o $PROTOC_ZIP -d /usr/local bin/protoc; \ - unzip -o $PROTOC_ZIP -d /usr/local 'include/*'; \ - rm -f $PROTOC_ZIP; - -# Install Bazel -RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - -RUN echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list -RUN apt update -RUN apt install bazel -# Create symlink to "python3" binary under the name "python" so Bazel doesn't complain about "python" not being found -RUN ln -s $(which python3) /usr/bin/python - -SHELL ["/bin/bash", "-c"] -RUN \ - function install_package { \ - # e.g. "head" or "release" \ - default_version="$1"; \ - # e.g "tensorflow-model-analysis" \ - package_name="$2"; \ - # e.g "model-analysis" \ - package_repo_name="$3"; \ - # How this package should be installed if pulled from GitHub. \ - # "none" for no extra installation steps required \ - # "bdist_wheel" for python setup.py bdist_wheel \ - package_install_type=$4; \ - # e.g. "head" or "release" or "pypi:0.21.4" or "github:[commit hash]" \ - package_version="$5"; \ - \ - # e.g. "tensorflow_model_analysis" \ - package_name_underscores=${package_name//-/_}; \ - if [ "$package_version" == "" ]; then \ - package_version="$default_version"; \ - fi; \ - \ - commit_id=""; \ - pypi_version=""; \ - if [ "$package_version" == "head" ]; then \ - commit_id=$(git ls-remote https://github.com/tensorflow/${package_repo_name} refs/heads/master | cut -f1); \ - echo ${package_name}: latest commit from GitHub: ${commit_id}; \ - elif [ "$package_version" == "release" ]; then \ - pypi_version=$(yolk -V $package_name | head -n 1 | cut -d' ' -f 2-); \ - echo ${package_name}: latest version from PyPI: ${pypi_version}; \ - elif [ "${package_version:0:5}" == "pypi:" ]; then \ - pypi_version="${package_version:5}"; \ - echo ${package_name}: using specified PyPI version: ${pypi_version}; \ - elif [ "${package_version:0:7}" == "github:" ]; then \ - commit_id="${package_version:7}"; \ - echo ${package_name}: using specified GitHub commit: ${commit_id}; \ - else \ - echo Unknown package version for ${package_name}: ${package_version}; \ - exit 1; \ - fi; \ - \ - if [ "$commit_id" != "" ]; then \ - if [ "$package_install_type" == "none" ]; then \ - # Package doesn't need extra installation steps - install directly from GitHub. \ - pip install -e git+https://github.com/tensorflow/${package_repo_name}.git@${commit_id}#egg=${package_name_underscores}; \ - install_commands+=("pip install --force --no-deps -e git+https://github.com/tensorflow/${package_repo_name}.git@${commit_id}#egg=${package_name_underscores}"); \ - echo Installed ${package_name} from GitHub commit ${commit_id}; \ - elif [ "$package_install_type" == "bdist_wheel" ]; then \ - # Package needs extra installation steps. Clone from GitHub, then build and install. \ - git clone https://github.com/tensorflow/${package_repo_name}.git; \ - pushd ${package_repo_name}; \ - git checkout ${commit_id}; \ - if [ "$package_name" == "tfx" ]; then \ - echo Building TFX pip package from source; \ - sed -i 's@packages=packages,@packages=packages, package_data={package_name: ["benchmarks/datasets/chicago_taxi/data/taxi_1M.tfrecords.gz"]},@' setup.py; \ - package_build/initialize.sh; \ - python package_build/ml-pipelines-sdk/setup.py bdist_wheel; \ - python package_build/tfx/setup.py bdist_wheel; \ - else \ - echo Using python setup.py bdist_wheel to build package; \ - python setup.py bdist_wheel; \ - fi; \ - pip install dist/*.whl; \ - install_commands+=("pip install --force --no-deps ${PWD}/dist/*.whl"); \ - popd; \ - echo Built and installed ${package_name} from GitHub commit ${commit_id}; \ - fi; \ - # Write GIT_COMMIT_ID attribute to the installed package. \ - package_path=$(python3 -c "import ${package_name_underscores}; print(list(${package_name_underscores}.__path__)[0])"); \ - echo "GIT_COMMIT_ID='${commit_id}'" >> ${package_path}/__init__.py; \ - install_commands+=("echo \"GIT_COMMIT_ID='${commit_id}'\" >> ${package_path}/__init__.py;"); \ - elif [ "$pypi_version" != "" ]; then \ - if [ "$package_name" == "tfx" ]; then \ - # Special handling for TFX - we want to install from GitHub, and get \ - # the data files as well (they are not included in the pip package). \ - # Install from the corresponding tag in GitHub. \ - echo Special handling for tfx: will install tfx from GitHub tag for version ${pypi_version}; \ - git clone --depth 1 --branch v${pypi_version} https://github.com/tensorflow/tfx.git; \ - pushd tfx; \ - echo Building TFX pip package from source; \ - sed -i 's@packages=packages,@packages=packages, package_data={package_name: ["benchmarks/datasets/chicago_taxi/data/taxi_1M.tfrecords.gz"]},@' setup.py; \ - package_build/initialize.sh; \ - python package_build/ml-pipelines-sdk/setup.py bdist_wheel; \ - python package_build/tfx/setup.py bdist_wheel; \ - pip install dist/*.whl; \ - install_commands+=("pip install --force --no-deps ${PWD}/dist/*.whl"); \ - popd; \ - echo Installed tfx from GitHub tag for version ${pypi_version}; \ - else \ - pip install ${package_name}==${pypi_version}; \ - install_commands+=("pip install --force --no-deps ${package_name}==${pypi_version}"); \ - echo Installed ${package_name} from PyPI version ${pypi_version}; \ - fi; \ - else \ - echo Neither commit_id nor pypi_version was set for ${package_name}; \ - exit 1; \ - fi; \ - }; \ - \ - # Array of commands to run post-installation. This is for forcing \ - # installation of packages without regard to the requirements of other \ - # packages. \ - # The first round of installations installs the packages and their \ - # requirements. This may result in some packages being re-installed at \ - # versions other than the requested versions due to requirements from \ - # other packages. \ - # The second round of installations via install_commands \ - # forces installations of the packages at the desired versions, ignoring \ - # any dependencies of these packages or other packages. Note that if there \ - # are incompatible package dependencies (e.g. tfx depends on \ - # apache-beam==2.21 and tensorflow-transform depends on apache-beam==2.22 \ - # then either could be installed depending on the installation order). \ - install_commands=(); \ - install_package "${default_package_version}" "tfx" "tfx" "bdist_wheel" "${tfx_package_version}"; \ - install_package "${default_package_version}" "tensorflow-transform" "transform" "none" "${tensorflow_transform_package_version}"; \ - install_package "${default_package_version}" "tensorflow-model-analysis" "model-analysis" "none" "${tensorflow_model_analysis_package_version}"; \ - install_package "${default_package_version}" "tensorflow-data-validation" "data-validation" "bdist_wheel" "${tensorflow_data_validation_package_version}"; \ - install_package "${default_package_version}" "tensorflow-metadata" "metadata" "bdist_wheel" "${tensorflow_metadata_package_version}"; \ - install_package "${default_package_version}" "tfx-bsl" "tfx-bsl" "bdist_wheel" "${tfx_bsl_package_version}"; \ - for cmd in "${install_commands[@]}"; do \ - echo Running "${cmd}"; \ - eval $cmd; \ - done; - -# Uninstall the TensorFlow version that TFX / the TFX components installed, and -# force install the version requested. -RUN pip uninstall -y tensorflow -RUN pip install --upgrade --force-reinstall ${tensorflow_pip_spec} - -RUN pip freeze diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_cuda11_8_0_0_180 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_cuda11_8_0_0_180 deleted file mode 100644 index 9fe2c684187f157c45381370d4a1d0509c12485c..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_cuda11_8_0_0_180 +++ /dev/null @@ -1,95 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 11 and the following: -# - Installs tf-nightly-gpu (this is TF 2.3) -# - Installs requirements.txt for tensorflow/models - -FROM nvidia/cuda:11.0-cudnn8-devel-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG local_tensorflow_pip_spec="" -ARG extra_pip_specs="" - -# setup.py passes the base path of local .whl file is chosen for the docker image. -# Otherwise passes an empty existing file from the context. -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn7-dev only needed because of libnvinfer-dev which may not -# really be needed. -# In the future, add the following lines in a shell script running on the -# benchmark vm to get the available dependent versions when updating cuda -# version (e.g to 10.2 or something later): -# sudo apt-cache search cuda-command-line-tool -# sudo apt-cache search cuda-cublas -# sudo apt-cache search cuda-cufft -# sudo apt-cache search cuda-curand -# sudo apt-cache search cuda-cusolver -# sudo apt-cache search cuda-cusparse - -RUN apt-get update && apt-get install -y --no-install-recommends \ - --allow-downgrades --allow-change-held-packages \ - build-essential \ - cuda-tools-11-0 \ - cuda-toolkit-11-0 \ - libcudnn8=8.0.0.180-1+cuda11.0 \ - libcudnn8-dev=8.0.0.180-1+cuda11.0 \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda-11.0/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends --allow-downgrades --allow-change-held-packages \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends --allow-downgrades --allow-change-held-packages \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-venv - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN pip3 install --upgrade pip -# setuptools upgraded to fix install requirements from model garden. -RUN pip install wheel -RUN pip install --upgrade setuptools google-api-python-client==1.8.0 pyyaml google-cloud google-cloud-bigquery google-cloud-datastore mock -RUN pip install absl-py -RUN pip install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} -RUN pip install tfds-nightly -RUN pip install -U scikit-learn -# Install dependnecies needed for tf.distribute test utils -RUN pip install dill tblib portpicker - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt - -RUN pip install tf-estimator-nightly -RUN pip install tensorflow-text-nightly - -# RUN nvidia-smi - -RUN nvcc --version - - -RUN pip freeze diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_experimental_cuda11 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_experimental_cuda11 deleted file mode 100644 index 4385d95a5d222502dd680b922d1b2b4af5645a50..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/docker/Dockerfile_ubuntu_experimental_cuda11 +++ /dev/null @@ -1,109 +0,0 @@ -# Ubuntu 18.04 Python3 with CUDA 11 and the following: -# - Installs tf-nightly-gpu (this is TF 2.3) -# - Installs requirements.txt for tensorflow/models - -FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu18.04 as base -ARG tensorflow_pip_spec="tf-nightly-gpu" -ARG local_tensorflow_pip_spec="" -ARG extra_pip_specs="" -ENV PIP_CMD="python3.9 -m pip" - -# setup.py passes the base path of local .whl file is chosen for the docker image. -# Otherwise passes an empty existing file from the context. -COPY ${local_tensorflow_pip_spec} /${local_tensorflow_pip_spec} - -# Pick up some TF dependencies -# cublas-dev and libcudnn7-dev only needed because of libnvinfer-dev which may not -# really be needed. -# In the future, add the following lines in a shell script running on the -# benchmark vm to get the available dependent versions when updating cuda -# version (e.g to 10.2 or something later): -# sudo apt-cache search cuda-command-line-tool -# sudo apt-cache search cuda-cublas -# sudo apt-cache search cuda-cufft -# sudo apt-cache search cuda-curand -# sudo apt-cache search cuda-cusolver -# sudo apt-cache search cuda-cusparse - -# Needed to disable prompts during installation. -ENV DEBIAN_FRONTEND noninteractive - - -RUN apt-get update && apt-get install -y --no-install-recommends \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libzmq3-dev \ - libpng-dev \ - pkg-config \ - software-properties-common \ - unzip \ - lsb-core \ - curl - -# Python 3.9 related deps in this ppa. -RUN add-apt-repository ppa:deadsnakes/ppa - - -# Install / update Python and Python3 -RUN apt-get install -y --no-install-recommends \ - python3.9 \ - python3-pip \ - python3.9-dev \ - python3-setuptools \ - python3.9-venv \ - python3.9-distutils \ - python3.9-lib2to3 - -# Upgrade pip, need to use pip3 and then pip after this or an error -# is thrown for no main found. -RUN ${PIP_CMD} install --upgrade pip -RUN ${PIP_CMD} install --upgrade distlib -# setuptools upgraded to fix install requirements from model garden. -RUN ${PIP_CMD} install --upgrade setuptools - -# For CUDA profiling, TensorFlow requires CUPTI. -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda-11.2/lib64:$LD_LIBRARY_PATH - -# See http://bugs.python.org/issue19846 -ENV LANG C.UTF-8 - -# Add google-cloud-sdk to the source list -RUN echo "deb http://packages.cloud.google.com/apt cloud-sdk-$(lsb_release -c -s) main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - - -# Install extras needed by most models -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - ca-certificates \ - wget \ - htop \ - zip \ - google-cloud-sdk - -RUN ${PIP_CMD} install --upgrade pyyaml -RUN ${PIP_CMD} install --upgrade google-api-python-client==1.8.0 -RUN ${PIP_CMD} install --upgrade google-cloud google-cloud-bigquery google-cloud-datastore mock - - -RUN ${PIP_CMD} install wheel -RUN ${PIP_CMD} install absl-py -RUN ${PIP_CMD} install --upgrade --force-reinstall ${tensorflow_pip_spec} ${extra_pip_specs} - -RUN ${PIP_CMD} install tfds-nightly -RUN ${PIP_CMD} install -U scikit-learn - -# Install dependnecies needed for tf.distribute test utils -RUN ${PIP_CMD} install dill tblib portpicker - -RUN curl https://raw.githubusercontent.com/tensorflow/models/master/official/requirements.txt > /tmp/requirements.txt -RUN ${PIP_CMD} install -r /tmp/requirements.txt - -RUN ${PIP_CMD} install tf-estimator-nightly -RUN ${PIP_CMD} install tensorflow-text-nightly - -# RUN nvidia-smi - -RUN nvcc --version - - -RUN pip freeze diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/dockertest/cuda_diff.sh b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/dockertest/cuda_diff.sh deleted file mode 100644 index 59a20515fc613ce89e0b64431c6f08c8b05847c8..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/dockertest/cuda_diff.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -set -e -set -x - -git clone https://github.com/tensorflow/benchmarks.git -cd benchmarks - -tf_spec="tf-nightly-gpu==2.5.0.dev20210212" - -CIFAR10_DATA="gs://tf-perf-imagenet-uswest1/tensorflow/cifar10_data/cifar-10-batches-bin" -CIFAR10_BENCHMARKS="official.benchmark.keras_cifar_benchmark.Resnet56KerasBenchmarkReal.benchmark_1_gpu_no_dist_strat" - -RESNET50_DATA="gs://tf-perf-imagenet-uswest1/tensorflow/imagenet" -RESNET50_BENCHMARKS="official.benchmark.resnet_ctl_imagenet_benchmark.Resnet50CtlBenchmarkReal.benchmark_1_gpu_fp16" - -function run_single_benchmark() { - docker_name=$1 - label=$2 - data_downloads=$3 - benchmark_methods=$4 - - perfzero_pwd=`pwd` - - nvidia-docker run \ - -v ${perfzero_pwd}:/workspace \ - -v /data:/data \ - -e PERFZERO_RUN_TAGS= \ - -e PERFZERO_TRACKING_ID= \ - -e PERFZERO_COMMIT_LABEL= \ - -e PERFZERO_EXECUTION_BRANCH=master \ - -e CUDNN_LOGINFO_DBG=1 \ - -e CUDNN_LOGDEST_DBG=stderr \ - ${docker_name} \ - python3 /workspace/perfzero/lib/benchmark.py \ - --bigquery_dataset_table_name="" \ - --data_downloads="${data_downloads}" \ - --ml_framework_build_label=v2-nightly-gpu \ - --execution_label="${label}" \ - --platform_name=kokoro-gcp \ - --system_name=n1-standard-8-1xA100 \ - --output_gcs_url="" \ - --benchmark_class_type= \ - --scratch_gcs_url= \ - --root_data_dir=/data \ - --benchmark_num_trials=2 \ - --bigquery_project_name="" \ - --git_repos="https://github.com/tensorflow/models.git;benchmark" \ - --python_path=models \ - --benchmark_methods=${benchmark_methods} \ - --result_upload_methods="" \ - --gcloud_key_file_url="${PERFZERO_GCLOUD_KEY_FILE_URL}" \ - --tpu_parameters= -} - -function run_benchmarks() { - docker_name=$1 - label=$2 - - run_single_benchmark ${docker_name} ${label} "${CIFAR10_DATA}" "${CIFAR10_BENCHMARKS}" - run_single_benchmark ${docker_name} ${label} "${RESNET50_DATA}" "${RESNET50_BENCHMARKS}" -} - -function setup_docker() { - label=$1 - dockerfile=$2 - - echo "`date` Setting up ${label} docker..." - sudo python3 perfzero/lib/setup.py \ - --gce_nvme_raid= \ - --docker_tag="${label}" \ - --gcloud_key_file_url= \ - --tensorflow_pip_spec=${tf_spec} \ - --dockerfile_path=${dockerfile} - echo "`date` Finished setting up ${label} docker." -} - -function diff_benchmarks() { - python3 perfzero/dockertest/diff_benchmarks.py `pwd` -} - -baseline_docker="docker/Dockerfile_ubuntu_cuda11_8_0_0_180" -experiment_docker="docker/Dockerfile_ubuntu_1804_tf_cuda_11" - -setup_docker "control/tensorflow" ${baseline_docker} -run_benchmarks "control/tensorflow" "control-8-0-0-180" - -setup_docker "experiment/tensorflow" ${experiment_docker} -run_benchmarks "experiment/tensorflow" "experiment-8-0-4-30" - -diff_benchmarks diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/dockertest/diff_benchmarks.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/dockertest/diff_benchmarks.py deleted file mode 100644 index e5c0c75fdddb8cd2ed88ccb0cfa7936c54712500..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/dockertest/diff_benchmarks.py +++ /dev/null @@ -1,117 +0,0 @@ -# Lint as: python3 -# Copyright 2021 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ============================================================================== -"""Simple script to diff benchmark results. - -This script will read all the summary files from a base output directory and -print a human readable diff report. -""" - -import json -import os -import sys - - -def _find_perfzero_logs(docker_output_dir): - """Finds pairs of json_file, output_log file from all methods.""" - summary_files = [] - for root, _, files in os.walk(docker_output_dir): - for summary_file in files: - if summary_file.endswith('perfzero_summary.json'): - full_summary_file = os.path.join(root, summary_file) - summary_files.append(full_summary_file) - sys.stdout.write('Found json {}\n'.format(full_summary_file)) - return summary_files - - -def _load_summaries(summary_files): - """Loads input json file paths and returns json objects.""" - summary_jsons = [] - for summary_file in summary_files: - with open(summary_file, 'r') as f: - summary_json = json.load(f) - summary_jsons.append(summary_json) - return summary_jsons - - -def _summarize_benchmarks(summary_files): - """Remaps list of json files -> summaries by benchmark method.""" - summary_jsons = _load_summaries(summary_files) - performance_by_method = {} - - for summary_json in summary_jsons: - method = summary_json['benchmark_result']['name'] - trial = summary_json['benchmark_result']['trial_id'] - metrics_list = summary_json['benchmark_result']['metrics'] - metrics = {} - for metric_info in metrics_list: - metrics[metric_info['name']] = metric_info['value'] - metrics['wall_time'] = summary_json['benchmark_result']['wall_time'] - label = summary_json['benchmark_info']['execution_label'] - - performance_by_method.setdefault(method, {}).setdefault(label, []) - performance_by_method[method][label].append((trial, metrics)) - - return performance_by_method - - -def _print_diff_report(performance_by_method): - """Prints a diff report of benchmark performance.""" - print('Performance report:') - print(json.dumps(performance_by_method, indent=2)) - - method_to_metric_to_perf = {} - for method in performance_by_method: - for label, label_data in performance_by_method[method].items(): - latest_trial_data = max(label_data, key=lambda x: x[0]) - latest_metrics = latest_trial_data[1] - for metric, value in latest_metrics.items(): - method_to_metric_to_perf.setdefault(method, {}).setdefault(metric, []) - method_to_metric_to_perf[method][metric].append((label, value)) - - print('Diff report:') - for method in sorted(method_to_metric_to_perf): - print('-- benchmark: {}'.format(method)) - for metric in sorted(method_to_metric_to_perf[method].keys()): - value_list = [] - for label, value in sorted( - method_to_metric_to_perf[method][metric], key=lambda x: x[0]): - print(' {}: {}: {}'.format(metric, label, value)) - value_list.append(value) - - if len(value_list) == 2: - control_val = float(value_list[0]) - expt_val = float(value_list[1]) - if abs(control_val) > 1e-5: - diff_pct = (expt_val / control_val - 1.0) * 100.0 - else: - diff_pct = -1.0 - print(' diff: {:2.2f}%'.format(diff_pct)) - - -def main(): - if len(sys.argv) != 2: - raise RuntimeError('Usage: {} '.format( - sys.argv[0])) - - perfzero_output_dir = sys.argv[1] - summary_files = _find_perfzero_logs(perfzero_output_dir) - performance_by_method = _summarize_benchmarks(summary_files) - _print_diff_report(performance_by_method) - - -if __name__ == '__main__': - main() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/dockertest/requirements_temp.txt b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/dockertest/requirements_temp.txt deleted file mode 100644 index a4249c32ea1ecc6bf5d9289218c96859764953c0..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/dockertest/requirements_temp.txt +++ /dev/null @@ -1,27 +0,0 @@ -six -google-api-python-client>=1.6.7 -kaggle>=1.3.9 -numpy>=1.15.4 -oauth2client -pandas>=0.22.0 -psutil>=5.4.3 -py-cpuinfo>=3.3.0 -scipy>=0.19.1 -tensorflow-hub>=0.6.0 -tensorflow-model-optimization>=0.4.1 -tensorflow-datasets -tensorflow-addons -dataclasses;python_version<"3.7" -gin-config -tf_slim>=1.1.0 -Cython -matplotlib -pyyaml>=5.1 -# CV related dependencies -opencv-python-headless -Pillow -pycocotools -# NLP related dependencies -seqeval -sentencepiece -sacrebleu diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/dockertest/resnet50_synth.sh b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/dockertest/resnet50_synth.sh deleted file mode 100644 index 29497089c80d4157cf5fb433ba66f7d7a7d6ec54..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/dockertest/resnet50_synth.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -set -e -set -x - -# To run this script from a GCP VM / host connected to GPUs: -# git clone https://github.com/tensorflow/benchmarks.git -# cd benchmarks -# bash perfzero/dockertest/resnet50_synth.sh -# Output log files/results will be stored at perfzero/workspace/output/ - -# Modify INPUT_PARAMS variables below to tweak the tf whl under test / benchmark methods / dataset paths. -# You can comment out "build_docker" call at the end, if the docker's already built. - -## INPUT PARAMS: start - -# Acceptable formats for TF_PIP_SPEC -# pypi nightlies: tf-nightly-gpu==2.6.0.dev20210521 -# gcs path to whls: gs://some-path-to-tf.whl -# Local path to whl: file://some-local-path-to-tf.whl -TF_PIP_SPEC="tf-nightly-gpu==2.6.0.dev20210624" - -# Path to GCS or local files containing the input datasets (if they need to be fetched into the docker). -DATA_DOWNLOADS="" - -# Comma separated list of strings. -BENCHMARK_METHODS="official.benchmark.keras_imagenet_benchmark.Resnet50KerasBenchmarkSynth.benchmark_1_gpu_fp16" - -# If either the tf_pip_spec or data downloads reference private GCP, then we -# need to set GCLOUD_KEY_FILE_URL to a credentials file. -GCLOUD_KEY_FILE_URL="" - -# Commit id under repository tensorflow/models, branch='benchmark' which has the benchmarks. -MODELS_GIT_HASH="169e4051aef247c27a95748a8015b2f35f509e1a" -## INPUT PARAMS: end - - - -build_docker() { - echo "building docker" -sudo python3 perfzero/lib/setup.py \ - --dockerfile_path=docker/Dockerfile_ubuntu_1804_tf_cuda_11 \ - --tensorflow_pip_spec="${TF_PIP_SPEC}" \ - --gcloud_key_file_url="${GCLOUD_KEY_FILE_URL}" \ - --extra_docker_build_args= -sudo docker images -} - - -run_benchmark() { - echo "running benchmark" - benchmark_tag=$1 - env_var=$2 - -sudo nvidia-docker run \ - -v ${PWD}:/workspace \ - -v /data:/data \ - -e PERFZERO_EXECUTION_MODE=test \ - -e TF_ENABLE_LEGACY_FILESYSTEM=1 \ - -e ${env_var} \ - perfzero/tensorflow python3 \ - /workspace/perfzero/lib/benchmark.py \ - --root_data_dir=/data \ - --bigquery_dataset_table_name="" \ - --benchmark_class_type= \ - --ml_framework_build_label=v2-nightly-gpu-${benchmark_tag} \ - --execution_label=test-benchmark \ - --platform_name=kokoro-gcp \ - --system_name=n1-standard-8-1xV100 \ - --output_gcs_url="" \ - --benchmark_num_trials=1 \ - --scratch_gcs_url= \ - --bigquery_project_name="" \ - --git_repos="https://github.com/tensorflow/models.git;benchmark;${MODELS_GIT_HASH}" \ - --data_downloads="${DATA_DOWNLOADS}"\ - --python_path=models \ - --benchmark_methods="${BENCHMARK_METHODS}" \ - --gcloud_key_file_url="${GCLOUD_KEY_FILE_URL}" -} - -build_docker -run_benchmark "control" "TF_CUDNN_USE_FRONTEND=false" -run_benchmark "experiment" "TF_CUDNN_USE_FRONTEND=true" diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/dockertest/run_single_benchmark.sh b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/dockertest/run_single_benchmark.sh deleted file mode 100644 index f1466fe77608d95dceac6fa3bb889c1f94088889..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/dockertest/run_single_benchmark.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -set -e -set -x - -# To run this script from a GCP VM / host connected to GPUs: -# git clone https://github.com/tensorflow/benchmarks.git -# cd benchmarks -# bash perfzero/dockertest/run_single_benchmark.sh -# Output log files/results will be stored at perfzero/workspace/output/ - -# Modify INPUT_PARAMS variables below to tweak the tf whl under test / benchmark methods / dataset paths. -# You can comment out "build_docker" call at the end, if the docker's already built. - -## INPUT PARAMS: start - -# Acceptable formats for TF_PIP_SPEC -# pypi nightlies: tf-nightly-gpu==2.6.0.dev20210521 -# gcs path to whls: gs://some-path-to-tf.whl -# Local path to whl: file://some-local-path-to-tf.whl -TF_PIP_SPEC="tf-nightly-gpu==2.6.0.dev20210521" - -# Path to GCS or local files containing the input datasets (if they need to be fetched into the docker). -DATA_DOWNLOADS="" - -# Comma separated list of strings. -BENCHMARK_METHODS="official.benchmark.keras_imagenet_benchmark.Resnet50KerasBenchmarkSynth.benchmark_xla_1_gpu_fp16" - -# If either the tf_pip_spec or data downloads reference private GCP, then we -# need to set GCLOUD_KEY_FILE_URL to a credentials file. -GCLOUD_KEY_FILE_URL="" -## INPUT PARAMS: end - - - -build_docker() { - echo "building docker" -sudo python3 perfzero/lib/setup.py \ - --dockerfile_path=docker/Dockerfile_ubuntu_1804_tf_cuda_11 \ - --tensorflow_pip_spec="${TF_PIP_SPEC}" \ - --gcloud_key_file_url="${GCLOUD_KEY_FILE_URL}" \ - --extra_docker_build_args= -sudo docker images -} - - -run_benchmark() { - echo "running benchmark" -sudo nvidia-docker run \ - -v ${PWD}:/workspace \ - -v /data:/data \ - -e PERFZERO_EXECUTION_MODE=test \ - -e TF_ENABLE_LEGACY_FILESYSTEM=1 \ - perfzero/tensorflow python3 \ - /workspace/perfzero/lib/benchmark.py \ - --root_data_dir=/data \ - --bigquery_dataset_table_name="" \ - --benchmark_class_type= \ - --ml_framework_build_label=v2-nightly-gpu \ - --execution_label=test-benchmark \ - --platform_name=kokoro-gcp \ - --system_name=n1-standard-8-1xV100 \ - --output_gcs_url="" \ - --benchmark_num_trials=1 \ - --scratch_gcs_url= \ - --bigquery_project_name="" \ - --git_repos='https://github.com/tensorflow/models.git;benchmark;f7938e6ad46fecfa1112eda579eb046eb3f3bf96' \ - --data_downloads="${DATA_DOWNLOADS}"\ - --python_path=models \ - --benchmark_methods="${BENCHMARK_METHODS}" \ - --gcloud_key_file_url="${GCLOUD_KEY_FILE_URL}" -} - -build_docker -run_benchmark diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/benchmark.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/benchmark.py deleted file mode 100644 index 37291a4b96f393506a96df8f091702e7c941cd8f..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/benchmark.py +++ /dev/null @@ -1,193 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Execute benchmark.""" -from __future__ import print_function - -import argparse -import json -import logging -import multiprocessing -import os -import re -import sys -import time - -import perfzero.benchmark_method_runner as benchmark_method_runner -import perfzero.perfzero_config as perfzero_config -import perfzero.tpu_runtime_utils as tpu_runtime_utils -import perfzero.utils as utils - - -class BenchmarkRunner(object): - """Execute benchmark and report results.""" - - def __init__(self, config): - self.config = config - self.project_dir = os.path.abspath( - os.path.dirname(os.path.dirname(__file__))) - self.workspace_dir = os.path.join(self.project_dir, config.workspace) - self.site_packages_dir = os.path.join(self.workspace_dir, 'site-packages') - self.root_output_dir = os.path.join(self.workspace_dir, 'output') - self.benchmark_execution_time = {} - - def _setup(self): - """Download data and checkout git repository.""" - - # Acticate gcloud service - start_time = time.time() - utils.setup_python_path(self.site_packages_dir, self.config.python_path_str) - utils.active_gcloud_service(self.config.gcloud_key_file_url, - self.workspace_dir) - utils.make_dir_if_not_exist(self.root_output_dir) - self.benchmark_execution_time['activate_gcloud_service'] = ( - time.time() - start_time) - - # Download data - start_time = time.time() - utils.download_data(utils.parse_data_downloads_str( - self.config.root_data_dir, self.config.gcs_downloads_str)) - utils.download_data(utils.parse_data_downloads_str( - self.config.root_data_dir, self.config.data_downloads_str)) - self.benchmark_execution_time['download_data'] = time.time() - start_time - - # Checkout git repositories - start_time = time.time() - site_package_info = utils.checkout_git_repos( - self.config.get_git_repos(self.site_packages_dir), - self.config.use_cached_site_packages) - self.benchmark_execution_time['checkout_repository'] = ( - time.time() - start_time) - - # Start cloud TPU. - if self.config.tpu_parameters is not None: - start_time = time.time() - utils.setup_tpu(self.config.tpu_parameters) - tpu_info = tpu_runtime_utils.configure_tpu(self.config.tpu_parameters) - site_package_info['tpu_version'] = tpu_info - self.benchmark_execution_time['start_tpu'] = time.time() - start_time - - self.stream_handler = logging.StreamHandler(sys.stdout) - self.stream_handler.setFormatter( - logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) - logging.getLogger().addHandler(self.stream_handler) - return site_package_info - - def _get_benchmark_methods(self): - """Returns list of benchmark methods to execute.""" - filter_prefix = 'filter:' - benchmark_methods = [] - for benchmark_method_pattern in self.config.benchmark_method_patterns: - if filter_prefix not in benchmark_method_pattern: - benchmark_methods.append(benchmark_method_pattern) - else: - index = benchmark_method_pattern.find(filter_prefix) - benchmark_class = benchmark_method_pattern[:index - 1] - pattern = benchmark_method_pattern[index + len(filter_prefix):] - class_instance = utils.instantiate_benchmark_class( - benchmark_class, '/dev/null', '', None, {}, - benchmark_class_type=self.config.benchmark_class_type) - - for benchmark_method_name in dir(class_instance): - if re.match(pattern, benchmark_method_name): - benchmark_methods.append(benchmark_class + '.' + - benchmark_method_name) - - logging.info('The following benchmark methods will be executed: %s', - benchmark_methods) - return benchmark_methods - - def _run_benchmarks_trial(self, harness_info, site_package_info, - benchmark_methods, trial_id): - """Runs a single trial of all benchmark methods.""" - # Run the benchmark method in a separate process so that its memory usage - # will not affect the execution of other benchmark method - # This is a walkaround before we fix all memory leak issues in TensorFlow - has_exception = False - benchmark_success_results = {} - benchmark_output_dirs = {} - benchmark_execution_time = {} - for benchmark_method in benchmark_methods: - queue = multiprocessing.Queue() - process = multiprocessing.Process(target=benchmark_method_runner.run, - args=(benchmark_method, - harness_info, - site_package_info, - self.root_output_dir, - self.config, queue, trial_id)) - process.start() - process.join() - method_has_exception, method_execution_time, succeeded, output_dir = queue.get() # pylint: disable=line-too-long - has_exception |= method_has_exception - benchmark_execution_time[benchmark_method] = method_execution_time - benchmark_success_results[benchmark_method] = succeeded - benchmark_output_dirs[benchmark_method] = output_dir - return (has_exception, benchmark_success_results, - benchmark_output_dirs, benchmark_execution_time) - - def run_benchmark(self): - """Run benchmark.""" - harness_info = utils.get_git_repo_info(self.project_dir) - has_exception = False - benchmark_success_results = {} - benchmark_output_dirs = {} - num_trials = self.config.benchmark_num_trials - - try: - site_package_info = self._setup() - benchmark_methods = self._get_benchmark_methods() - - print('Setup complete. Running {} trials'.format(num_trials)) - for trial_id in range(1, num_trials + 1): - print('Running trial {} / {}'.format(trial_id, num_trials)) - (trial_has_exception, trial_success_results, - trial_output_dirs, trial_execution_time) = self._run_benchmarks_trial( - harness_info, site_package_info, benchmark_methods, trial_id) - - trial_key = 'trial_{}'.format(trial_id) - has_exception |= trial_has_exception - self.benchmark_execution_time[trial_key] = trial_execution_time - benchmark_success_results[trial_key] = trial_success_results - benchmark_output_dirs[trial_key] = trial_output_dirs - finally: - if self.config.tpu_parameters is not None: - has_exception |= utils.cleanup_tpu(self.config.tpu_parameters) - - print('Benchmark execution time in seconds by operation:\n {}'.format( - json.dumps(self.benchmark_execution_time, indent=2))) - print('Benchmark success results:\n{}'.format( - json.dumps(benchmark_success_results, indent=2))) - print('Benchmark local output directories:\n{}'.format( - json.dumps(benchmark_output_dirs, indent=2))) - if has_exception: - sys.exit(1) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - perfzero_config.add_benchmark_parser_arguments(parser) - FLAGS, unparsed = parser.parse_known_args() - - level = logging.DEBUG if FLAGS.debug else logging.INFO - logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', - level=level) - - if unparsed: - logging.error('Arguments %s are not recognized', unparsed) - sys.exit(1) - - config_ = perfzero_config.PerfZeroConfig(mode='flags', flags=FLAGS) - benchmark_runner = BenchmarkRunner(config_) - benchmark_runner.run_benchmark() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/benchmark_test.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/benchmark_test.py deleted file mode 100644 index 926537e571178058764c6ea1c059a43e22d60f4d..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/benchmark_test.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for benchmark.py.""" -from __future__ import print_function - -import sys -import unittest - -import benchmark -import mock - - -class TestBenchmarkRunner(unittest.TestCase): - - def test_get_benchmark_methods_filter(self): - """Tests returning methods on a class based on a filter.""" - config = mock.Mock() - config.workspace = 'workspace' - config.benchmark_method_patterns = ['new_foo.BenchmarkClass.filter:bench.*'] - benchmark_runner = benchmark.BenchmarkRunner(config) - - mock_benchmark_class = mock.Mock() - mock_benchmark_class.benchmark_method_1 = 'foo' - - mock_module = mock.Mock() - sys.modules['new_foo'] = mock_module - mock_module.BenchmarkClass.return_value = mock_benchmark_class - - methods = benchmark_runner._get_benchmark_methods() - - self.assertEqual(1, len(methods)) - self.assertEqual('new_foo.BenchmarkClass.benchmark_method_1', methods[0]) - - def test_get_benchmark_methods_exact_match(self): - """Tests returning methods on a class based on a filter.""" - config = mock.Mock() - config.workspace = 'workspace' - config.benchmark_method_patterns = [ - 'new_foo.BenchmarkClass.benchmark_method_1', - 'new_foo.BenchmarkClass.benchmark_method_2'] - benchmark_runner = benchmark.BenchmarkRunner(config) - - methods = benchmark_runner._get_benchmark_methods() - self.assertEqual(['new_foo.BenchmarkClass.benchmark_method_1', - 'new_foo.BenchmarkClass.benchmark_method_2'], methods) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/cloud_manager.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/cloud_manager.py deleted file mode 100644 index e97d4316edaea4d95b79e49e465d1ded6bb0f619..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/cloud_manager.py +++ /dev/null @@ -1,431 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Helper script to create, query and stop machine in GCP.""" - -from __future__ import print_function - -import argparse -import getpass -import logging -import subprocess -import sys -import time - - -INSTANCE_NAME_PREFIX = 'perfzero-dev-' - - -def run_command(cmd, is_from_user=False): - """Runs list of command and throw error if return code is non-zero. - - Args: - cmd: Command to execute - is_from_user: If true, log the command and the command output in INFO level. - Otherwise, log these in the DEBUG level. - - Returns: - a string representing the command output - - Raises: - Exception: raised when the command execution has non-zero exit code - """ - _log = logging.info if is_from_user else logging.debug - _log('Executing command: {}'.format(cmd)) - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, shell=True) - - exit_code = None - line = '' - stdout = '' - while exit_code is None or line: - exit_code = p.poll() - line = p.stdout.readline().decode('utf-8') - stdout += line - _log(line) - if exit_code and is_from_user: - sys.exit(exit_code) - elif exit_code: - raise Exception('Command:\n{}\nfailed with output:\n{}'.format(cmd, stdout)) - - return stdout - - -def get_instance_name(username): - return INSTANCE_NAME_PREFIX + username - - -def get_machine_type(machine_type, accelerator_count): - """Get machine type for the instance. - - - Use the user-specified machine_type if it is not None - - Otherwise, use the standard type with cpu_count = 8 x accelerator_count - if user-specified accelerator_count > 0 - - Otherwise, use the standard type with 8 cpu - - Args: - machine_type: machine_type specified by the user - accelerator_count: accelerator count - - Returns: - the machine type used for the instance - """ - if machine_type: - return machine_type - cpu_count = max(accelerator_count, 1) * 8 - return 'n1-standard-{}'.format(cpu_count) - - -def _ssh_prefix(project, zone, internal_ip, key_file): - if internal_ip: - ssh_prefix = 'gcloud beta compute ssh --internal-ip' - else: - ssh_prefix = 'gcloud compute ssh' - if key_file: - ssh_prefix = '{} --ssh-key-file={}'.format(ssh_prefix, key_file) - return '{} --project={} --zone={}'.format(ssh_prefix, project, zone) - - -def create(username, project, zone, machine_type, accelerator_count, - accelerator_type, image, nvme_count, ssh_internal_ip, ssh_key_file, - cpu_min_platform=None, boot_ssd_size=None): - """Create gcloud computing instance. - - Args: - username: the username of the current user - project: project name - zone: zone of the GCP computing instance - machine_type: the machine type used for the instance - accelerator_count: the number of pieces of the accelerator to attach to - the instance - accelerator_type: the specific type of accelerator to attach to the instance - image: the name of the image that the disk will be initialized with - nvme_count: the number of NVME local SSD devices to attach to the instance - ssh_internal_ip: internal ip to use for ssh. - ssh_key_file: ssh key file to use to connect to instance. - cpu_min_platform: minimum CPU platform to use, if None use default. - boot_ssd_size: If set boot disk is changed to SSD and this size(GB) is used. - """ - instance_name = get_instance_name(username) - machine_type = get_machine_type(machine_type, accelerator_count) - logging.debug('Creating gcloud computing instance %s', instance_name) - - cmd = '''gcloud compute instances create {} \ ---image={} \ ---project={} \ ---zone={} \ ---machine-type={} \ ---maintenance-policy=TERMINATE \ -'''.format(instance_name, image, project, zone, machine_type) - - if boot_ssd_size: - cmd += '--boot-disk-size={}GB --boot-disk-type=pd-ssd '.format( - boot_ssd_size) - - if accelerator_count > 0: - cmd += '--accelerator=count={},type={} '.format( - accelerator_count, accelerator_type) - - if cpu_min_platform: - cmd += '--min-cpu-platform="{}" '.format(cpu_min_platform) - - for _ in range(nvme_count): - cmd += '--local-ssd=interface=NVME ' - - run_command(cmd, is_from_user=True) - logging.info('Successfully created gcloud computing instance %s ' - 'with %s accelerator.\n', instance_name, accelerator_count) - - ssh_prefix = _ssh_prefix(project, zone, ssh_internal_ip, ssh_key_file) - # Wait until we can ssh to the newly created computing instance - cmd = '{} --strict-host-key-checking=no --command="exit" {}'.format( - ssh_prefix, instance_name) - ssh_remaining_retries = 12 - ssh_error = None - while ssh_remaining_retries > 0: - ssh_remaining_retries -= 1 - try: - run_command(cmd, is_from_user=False) - ssh_error = None - except Exception as error: # pylint: disable=broad-except - ssh_error = error - if ssh_remaining_retries: - logging.info('Cannot ssh to the computing instance. ' - 'Try again after 5 seconds') - time.sleep(5) - else: - logging.error('Cannot ssh to the computing instance after ' - '60 seconds due to error:\n%s', str(ssh_error)) - - if ssh_error: - logging.info('Run the commands below manually after ssh into the computing ' - 'instance:\n' - 'git clone https://github.com/tensorflow/benchmarks.git\n' - 'sudo usermod -a -G docker $USER\n') - else: - cmd = '{} --command="git clone {}" {}'.format( - ssh_prefix, 'https://github.com/tensorflow/benchmarks.git', - instance_name) - run_command(cmd, is_from_user=True) - logging.info('Successfully checked-out PerfZero code on the ' - 'computing instance\n') - - cmd = '{} --command="sudo usermod -a -G docker $USER" {}'.format( - ssh_prefix, instance_name) - run_command(cmd, is_from_user=True) - logging.info('Successfully added user to the docker group\n') - - cmd = '{} {} -- -L 6006:127.0.0.1:6006'.format(ssh_prefix, instance_name) - logging.info('Run the command below to ssh to the instance together with ' - 'port forwarding for tensorboard:\n%s\n', cmd) - - -def status(username, project, zone, ssh_internal_ip, ssh_key_file): - """Query the status of the computing instance. - - Args: - username: the username of the current user. - project: project name. - zone: zone of the GCP computing instance. - ssh_internal_ip: internal ip of the instance. - ssh_key_file: SSH key file to use to connect to the instance. - """ - instance_name = get_instance_name(username) - logging.debug('Querying status of gcloud computing instance %s of ' - 'project %s in zone %s', instance_name, project, zone) - - cmd = 'gcloud compute instances list --filter="name={} AND zone:{}" --project {}'.format( # pylint: disable=line-too-long - instance_name, zone, project) - stdout = run_command(cmd, is_from_user=True) - - num_instances = len(stdout.splitlines()) - 1 - logging.info('\nFound %s gcloud computing instance with name %s.\n', - num_instances, instance_name) - - if num_instances == 1: - cmd = '{} {} -- -L 6006:127.0.0.1:6006'.format( - _ssh_prefix(project, zone, ssh_internal_ip, ssh_key_file), - instance_name) - logging.info('Run the command below to ssh to the instance together with ' - 'port forwarding for tensorboard:\n%s\n', cmd) - - -def list_all(project): - logging.debug('Finding all gcloud computing instance of project %s created ' - 'for PerfZero test', project) - cmd = 'gcloud compute instances list --filter="name ~ {}" --project={}'.format( # pylint: disable=line-too-long - INSTANCE_NAME_PREFIX, project) - stdout = run_command(cmd, is_from_user=True) - num_instances = len(stdout.splitlines()) - 1 - logging.info('\nFound %s gcloud computing instance of project %s created ' - 'for PerfZero test', num_instances, project) - - -def start(username, project, zone): - instance_name = get_instance_name(username) - logging.debug('Starting gcloud computing instance %s of project %s ' - 'in zone %s', instance_name, project, zone) - - cmd = 'gcloud compute instances start {} --project={} --zone={}'.format( - instance_name, project, zone) - run_command(cmd, is_from_user=True) - logging.debug('\nSuccessfully started gcloud computing instance %s of ' - 'project %s in zone %s', instance_name, project, zone) - - -def stop(username, project, zone): - instance_name = get_instance_name(username) - logging.debug('Stopping gcloud computing instance %s of project %s in ' - 'zone %s', instance_name, project, zone) - - cmd = 'gcloud compute instances stop {} --project={} --zone={}'.format( - instance_name, project, zone) - run_command(cmd, is_from_user=True) - logging.debug('\nSuccessfully stopped gcloud computing instance %s of ' - 'project %s in zone %s', instance_name, project, zone) - - -def delete(username, project, zone): - instance_name = get_instance_name(username) - logging.debug('Deleting gcloud computing instance %s of project %s in ' - 'zone %s', instance_name, project, zone) - - cmd = 'echo Y | gcloud compute instances delete {} --project={} --zone={}'.format( # pylint: disable=line-too-long - instance_name, project, zone) - run_command(cmd, is_from_user=True) - logging.debug('\nSuccessfully deleted gcloud computing instance %s of ' - 'project %s in zone %s', instance_name, project, zone) - - -def parse_arguments(argv, command): # pylint: disable=redefined-outer-name - """Parse command line arguments and return parsed flags. - - Args: - argv: command line arguments - command: the subcommand requested by the user - - Returns: - parsed flags - """ - - # pylint: disable=redefined-outer-name - parser = argparse.ArgumentParser( - usage='cloud_manager.py {} []'.format(command), - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument( - '--debug', - action='store_true', - help='If set, use debug level logging. Otherwise, use info level logging' - ) - parser.add_argument( - '--project', - default='google.com:tensorflow-performance', - type=str, - help='Google Cloud Platform project name to use for this invocation' - ) - - if command in ['create', 'start', 'stop', 'delete', 'status']: - parser.add_argument( - '--username', - default=getpass.getuser(), - type=str, - help='''Username that uniquely identifies the name of computing instance created for PerfZero. - The default value is your ldap username. - ''') - parser.add_argument( - '--zone', - default='us-west1-b', - type=str, - help='Zone of the instance to create.' - ) - parser.add_argument( - '--ssh-internal-ip', - action='store_true', - help='If set, use internal IP for ssh with `gcloud beta compute ssh`.' - ) - parser.add_argument( - '--ssh-key-file', - default=None, - type=str, - help='The ssh key to use with with `gcloud (beta) compute ssh`.' - ) - - if command == 'create': - parser.add_argument( - '--accelerator_count', - default=1, - type=int, - help='The number of pieces of the accelerator to attach to the instance' - ) - parser.add_argument( - '--accelerator_type', - default='nvidia-tesla-v100', - type=str, - help='''The specific type (e.g. nvidia-tesla-v100 for nVidia Tesla V100) of - accelerator to attach to the instance. Use 'gcloud compute accelerator-types list --project=${project_name}' to - learn about all available accelerator types. - ''') - parser.add_argument( - '--cpu_min_platform', - default=None, - type=str, - help='''Minimum cpu platform, only needed for CPU only instances.''') - parser.add_argument( - '--machine_type', - default=None, - type=str, - help='''The machine type used for the instance. To get a list of available machine - types, run 'gcloud compute machine-types list --project=${project_name}' - ''') - parser.add_argument( - '--image', - default='tf-ubuntu-1604-20180927-410', - type=str, - help='''Specifies the name of the image that the disk will be initialized with. - A new disk will be created based on the given image. To view a list of - public images and projects, run 'gcloud compute images list --project=${project_name}'. It is best - practice to use image when a specific version of an image is needed. - ''') - parser.add_argument( - '--nvme_count', - default=0, - type=int, - help='''Specifies the number of NVME local SSD devices to attach to the instance. - ''' - ) - parser.add_argument( - '--boot_ssd_size', - default=None, - type=int, - help='''Specifies the size (GB) of the boot disk or size is the image - size. Setting this also changes boot disk to Persistent SSD. - ''' - ) - - flags, unparsed = parser.parse_known_args(argv) # pylint: disable=redefined-outer-name - if unparsed: - logging.error('Arguments %s are not recognized', unparsed) - sys.exit(1) - - level = logging.DEBUG if flags.debug else logging.INFO - logging.basicConfig(format='%(message)s', level=level) - - return flags -if __name__ == '__main__': - parser = argparse.ArgumentParser( - usage='''cloud_manager.py [] - -The supported commands are: - create: Create a computing instance in gcloud that is unique to the specified username, which is your ldap by default - start: Start the computing instance in gcloud that is unique to the specified username, which is your ldap by default - stop: Stop the computing instance in gcloud that is unique to the specified username, which is your ldap by default - delete: Delete the computing instance in gcloud that is unique to the specified username, which is your ldap by default - status: Query the status and information of the computing instance in gcloud that is unique to the specified username, which is your ldap by default - list_all: Query the status of all computing instances that are created by this script.''' - ) - parser.add_argument( - 'command', - type=str - ) - - flags = parser.parse_args(sys.argv[1:2]) - command = flags.command - if not hasattr(sys.modules[__name__], command): - print('Error: The command <{}> is not recognized\n'.format(command)) - parser.print_help() - sys.exit(1) - - flags = parse_arguments(sys.argv[2:], command) - - if command == 'create': - create(flags.username, flags.project, flags.zone, flags.machine_type, - flags.accelerator_count, flags.accelerator_type, flags.image, - flags.nvme_count, flags.ssh_internal_ip, flags.ssh_key_file, - cpu_min_platform=flags.cpu_min_platform, - boot_ssd_size=flags.boot_ssd_size) - elif command == 'start': - start(flags.username, flags.project, flags.zone) - elif command == 'stop': - stop(flags.username, flags.project, flags.zone) - elif command == 'delete': - delete(flags.username, flags.project, flags.zone) - elif command == 'status': - status(flags.username, flags.project, flags.zone, flags.ssh_internal_ip, - flags.ssh_key_file) - elif command == 'list_all': - list_all(flags.project) - - diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/benchmark_method_runner.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/benchmark_method_runner.py deleted file mode 100644 index f5b877714b125bba0e5ddb4a4386de12085c6a7c..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/benchmark_method_runner.py +++ /dev/null @@ -1,187 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Execute a single benchmark method.""" -from __future__ import print_function - -import datetime -import json -import logging -import os -import time -import traceback - -from perfzero.process_info_tracker import ProcessInfoTracker -import perfzero.report_utils as report_utils -from perfzero.tensorflow_profiler import TensorFlowProfiler -import perfzero.utils as utils - - -def run(benchmark_method, harness_info, site_package_info, - root_output_dir, config, queue, trial_id): - try: - _run_internal(benchmark_method, harness_info, site_package_info, - root_output_dir, config, queue, trial_id) - except Exception: # pylint: disable=broad-except - logging.error('Benchmark execution for %s failed due to error:\n %s', - benchmark_method, traceback.format_exc()) - queue.put((True, None, False, None)) - - -def _set_file_contents(content_str, output_filename): - with open(output_filename, 'w') as f: - f.write(content_str) - logging.info('Wrote summary to file %s', output_filename) - - -def _run_internal(benchmark_method, harness_info, site_package_info, - root_output_dir, config, queue, trial_id): - """Run benchmark method and put result to the queue. - - Args: - benchmark_method: Canonical path to the benchmark method - harness_info: Description of the benchmark harness used in the benchmark - site_package_info: Description of the site-package used in the benchmark - root_output_dir: Directory under which to put the benchmark output - config: An instance of perfzero_config - queue: An interprocess queue to transfer benchmark result to the caller. - trial_id: An integer trial id to annotate in the benchmark result. - """ - start_timestamp = time.time() - execution_timestamp = start_timestamp - method_has_exception = False - execution_id = (config.execution_id if config.execution_id else - datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f')) - output_dir = os.path.join(root_output_dir, execution_id) - if config.scratch_gcs_url: - model_output_dir = os.path.join(config.scratch_gcs_url, execution_id) - else: - model_output_dir = output_dir - utils.make_dir_if_not_exist(output_dir) - benchmark_class, benchmark_method_name = benchmark_method.rsplit('.', 1) - benchmark_class_name = benchmark_class.rsplit('.', 1)[1] - - tensorflow_profiler = TensorFlowProfiler( - config.profiler_enabled_time_str, output_dir) - process_info_tracker = ProcessInfoTracker(output_dir) - process_info = None - - # Setup per-method file logger - filehandler = logging.FileHandler( - filename=os.path.join(output_dir, 'perfzero.log'), mode='w') - filehandler.setFormatter( - logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) - logging.getLogger().addHandler(filehandler) - - try: - if config.tpu_parameters: - tpu = config.tpu_parameters.get('name') - else: - tpu = None - if config.perfzero_constructor_args: - constructor_args = json.loads(config.perfzero_constructor_args) - else: - constructor_args = {} - class_instance = utils.instantiate_benchmark_class( - benchmark_class=benchmark_class, - output_dir=model_output_dir, - root_data_dir=config.root_data_dir, - tpu=tpu, - constructor_args=constructor_args, - benchmark_class_type=config.benchmark_class_type) - # tf.test.Benchmark.report_benchmark() writes results to a file with - # path benchmark_result_file_path_prefix + benchmark_method - benchmark_result_file_path_prefix = os.path.join(output_dir, 'proto_') - os.environ['TEST_REPORT_FILE_PREFIX'] = benchmark_result_file_path_prefix - benchmark_result_file_path = '{}{}.{}'.format( - benchmark_result_file_path_prefix, - benchmark_class_name, - benchmark_method_name) - - # Start background threads for profiler and system info tracker - tensorflow_profiler.start() - process_info_tracker.start() - - # Run benchmark method - execution_timestamp = time.time() - logging.info('Starting benchmark execution: %s', benchmark_method) - getattr(class_instance, benchmark_method_name)() - logging.info('Stopped benchmark: %s', benchmark_method) - - # Read and build benchmark results - raw_benchmark_result = utils.read_benchmark_result( - benchmark_result_file_path) - # Explicitly overwrite the name to be the full path to benchmark method - raw_benchmark_result['name'] = benchmark_method - except Exception: # pylint: disable=broad-except - logging.error('Benchmark execution for %s failed due to error:\n %s', - benchmark_method, traceback.format_exc()) - method_has_exception = True - raw_benchmark_result = {} - raw_benchmark_result['name'] = benchmark_method - raw_benchmark_result['wall_time'] = -1 - raw_benchmark_result['extras'] = {} - finally: - # Stop background threads for profiler and system info tracker - process_info = process_info_tracker.stop() - tensorflow_profiler.stop() - - upload_timestamp = time.time() - benchmark_result = report_utils.build_benchmark_result( - raw_benchmark_result, method_has_exception, trial_id) - execution_summary = report_utils.build_execution_summary( - execution_timestamp, - execution_id, - config.ml_framework_build_label, - config.execution_label, - config.platform_name, - config.system_name, - config.output_gcs_url, - benchmark_result, - config.get_env_vars(), - config.get_flags(), - harness_info, - site_package_info, - process_info, - method_has_exception, - is_tpu_benchmark = (config.tpu_parameters != None)) - report_utils.upload_execution_summary( - config.bigquery_project_name, - config.bigquery_dataset_table_name, - execution_summary) - report_utils.execute_methods( - config.result_upload_methods, - execution_summary) - logging.info('Benchmark execution for %s completed with summary:\n %s', - benchmark_method, json.dumps(execution_summary, indent=2)) - _set_file_contents(json.dumps(execution_summary, indent=2), - os.path.join(output_dir, 'perfzero_summary.json')) - utils.maybe_upload_to_gcs(output_dir, config.output_gcs_url) - logging.getLogger().removeHandler(filehandler) - method_execution_time = { - 'class_initialization': execution_timestamp - start_timestamp, - 'method_execution': upload_timestamp - execution_timestamp, - 'log_upload': time.time() - upload_timestamp - } - - if config.profiler_enabled_time_str: - relative_output_dir = output_dir[output_dir.find('benchmark'):] - print('\nExecute the command below to start tensorboard server using ' - 'the collected profiler data:\ntensorboard --logdir={}\n\n' - 'Open localhost:6006 in your browser to access the Tensorbord ' - 'GUI. Use ssh with port forwarding if tensorboard is running on ' - 'a remote machine.\n'.format(relative_output_dir)) - - queue.put((method_has_exception, method_execution_time, - benchmark_result['succeeded'], output_dir)) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/device_utils.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/device_utils.py deleted file mode 100644 index dd3e14be1bf36dc08be1cfae1282be490e603fc9..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/device_utils.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Setup the data drive with raid, RAM, or mount network drives.""" -from __future__ import print_function - -import logging - -import perfzero.utils as utils - - -def create_drive_from_devices(data_dir, gce_nvme_raid): - """Creates a drive at data directory.""" - if not gce_nvme_raid: - return - - devices = _get_nvme_devices() - cmd = 'mountpoint -q {}'.format(data_dir) - retcode, _ = utils.run_command(cmd) - if retcode: - if len(devices) > 1: - _create_drive_raid(data_dir, devices) - else: - _create_single_drive(data_dir, devices[0]) - - -def _get_nvme_devices(): - """Returns list paths to nvme devices.""" - devices = [] - cmd = 'lsblk' - retcode, log = utils.run_command(cmd) - if retcode: - raise Exception('"{}" failed with code:{} and log:\n{}'.format( - cmd, retcode, log)) - - lines = log.splitlines() - if lines: - for line in lines: - if line.startswith('nvme'): - parts = line.split() - devices.append('/dev/' + parts[0].strip()) - return devices - - -def _create_single_drive(data_dir, device): - """Creates a data drive out of a single device.""" - cmds = [] - cmds.append('mkfs.ext4 -F {}'.format(device)) - cmds.append('mkdir -p {}'.format(data_dir)) - cmds.append('mount {} {}'.format(device, data_dir)) - cmds.append('chmod a+w {}'.format(data_dir)) - - utils.run_commands(cmds) - logging.info('Created and mounted device %s at %s', device, data_dir) - - -def _create_drive_raid(data_dir, devices): - """Creates a raid zero array of nvme drives.""" - cmds = [] - # Passing 'yes' because GCE nvme drive are sometimes in an odd state and - # think they are in another raid. mdadm does not have -y option. - # Or the kokoro images were left dirty? and that is where the info - # comes from. - cmds.append('yes | mdadm --create /dev/md0 --level=0 ' - '--raid-devices={} {}'.format( - len(devices), ' '.join(devices))) - cmds.append('mkfs.ext4 -F /dev/md0') - cmds.append('mkdir -p {}'.format(data_dir)) - cmds.append('mount /dev/md0 {}'.format(data_dir)) - cmds.append('chmod a+w {}'.format(data_dir)) - - utils.run_commands(cmds) - logging.info('Created and mounted RAID array at %s', data_dir) - - diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/perfzero_config.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/perfzero_config.py deleted file mode 100644 index 2fa217c92b23278e9e6cd360325e242b3b116e5e..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/perfzero_config.py +++ /dev/null @@ -1,367 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""PerfZero configs provided by user.""" -from __future__ import print_function - -import json -import logging -import os - - -def add_setup_parser_arguments(parser): - """Add arguments to the parser used by the setup.py.""" - parser.add_argument( - '--dockerfile_path', - default='docker/Dockerfile_ubuntu_1804_tf_v1', - type=str, - help='''Build the docker image using docker file located at the ${pwd}/${dockerfile_path} if - it exists, where ${pwd} is user's current work directory. Otherwise, build - the docker image using the docker file located at path_to_perfzero/${dockerfile_path}. - ''') - parser.add_argument( - '--workspace', - default='workspace', - type=str, - help='''The gcloud key file will be downloaded under directory path_to_perfzero/${workspace} - ''') - parser.add_argument( - '--gcloud_key_file_url', - default='', - type=str, - help='''DEPRECATED: Use --gcloud_key_file_url of setup.py instead. - The gcloud key file url. When specified, it will be downloaded to the - directory specified by the flag --workspace. Each url can start with file://, gcs://, http:// or https://. - ''') - parser.add_argument( - '--root_data_dir', - default='/data', - type=str, - help='The directory which should contain the dataset required by the becnhmark method.' - ) - parser.add_argument( - '--gce_nvme_raid', - default=None, - type=str, - help='If set to non-empty string, create raid 0 array with devices at the directory specified by the flag --root_data_dir' - ) - parser.add_argument( - '--tensorflow_pip_spec', - default=None, - type=str, - help='''The tensorflow pip package specfication. The format can be either ${package_name}, or ${package_name}==${package_version}. - Example values include tf-nightly-gpu, and tensorflow==1.12.0. If it is specified, the corresponding tensorflow pip package/version - will be installed. Otherwise, the default tensorflow pip package specified in the docker file will be installed. - ''') - parser.add_argument( - '--extra_pip_specs', - default='', - type=str, - help='''Additional specifications to pass to `pip install`. (e.g. pinning certain dependencies) - Specifications should be semicolon separated: e.g. `numpy==1.16.4;scipy==1.3.1` - ''') - parser.add_argument( - '--docker_tag', - default='perfzero/tensorflow', - type=str, - help='The docker tag to use if building a docker image.' - ) - parser.add_argument( - '--site_package_downloads', - default='', - type=str, - help='''Comma separated list of dirs in the external vm to copy to the docker\'s site package dir. - Format: /src/dir:new_base_dir_name,/src/dir2>:new_name,.... - This will copy /src/dir to /new_base_dir_name. - ''' - ) - parser.add_argument( - '--extra_docker_build_args', - nargs='*', - default='', - type=str, - help='''Additional build-args to pass to `docker build`. - Example: --extra_docker_build_args arg0 arg1=value1 "arg2=value with space" arg3=300. - Each string will be passed directly as a build-arg to docker, so the above example will be passed as follows: - --build-arg arg0 --build-arg arg1=value1 --build-arg "arg2=value with space" --build-arg arg3=300 - ''' - ) - - -def add_benchmark_parser_arguments(parser): - """Add arguments to the parser used by the benchmark.py.""" - parser.add_argument( - '--use_cached_site_packages', - action='store_true', - help='If set, skip git pull for dependent git repositories if it already exists in path_to_perfzero/${workspace}/site-packages' - ) - parser.add_argument( - '--gcs_downloads', - default=None, - type=str, - help='This flag is deprecated. Use the flag --data_downloads instead') - parser.add_argument( - '--git_repos', - default=None, - type=str, - help='''A string representing git repositories to checkout. The format is url_1;branch_1;hash_1,url_2;branch_2;hash_2,... - Git repositories will be checked-out under directory path_to_perfzero/${workspace}/site-packages, - where ${workspace} either defaults to 'workspace', or takes the value of the flag --workspace. - branch and hash can be skipped if user wants to use the head of the master branch, - which shortens the format to url_1,url_2,... - ''') - parser.add_argument( - '--benchmark_num_trials', - default=1, - type=int, - help='''Configures number of times to run each benchmark method - after setup completion.''') - parser.add_argument( - '--benchmark_methods', - action='append', - default=[], - type=str, - help='''This string specifies the benchmark_method to be executed. The flag can be specified multiple times in which case - the union of methods matched by these flags will be executed. The format can be module_path.class_name.method_name in which - case the corresponding method is executed. The format can also be module_path.class_name.filter:regex_pattern, in which case all methods - of the given class whose method name matches the given regular expression are executed. - ''') - parser.add_argument( - '--ml_framework_build_label', - default=None, - type=str, - help='A string that identified the machine learning framework build, e.g. nightly-gpu-build' - ) - parser.add_argument( - '--execution_label', - default=None, - type=str, - help='A string that identified the benchmark execution type, e.g. test, prod' - ) - parser.add_argument( - '--platform_name', - default=None, - type=str, - help='A string that identified the computing platform, e.g. gcp, aws' - ) - parser.add_argument( - '--system_name', - default=None, - type=str, - help='A string that identified the hardware system, e.g. n1-standard-64-8xV100' - ) - parser.add_argument( - '--output_gcs_url', - default=None, - type=str, - help='''If specified, log files generated by the benchmark execution will be uploaded to output_gcs_url/${execution_id}, - where ${execution_id} is a string that generated by PerfZero which uniquely identifies the execution of one benchmark method - ''') - parser.add_argument( - '--scratch_gcs_url', - default=None, - type=str, - help='''If specified, intermediate files like model outputs will be stored in scratch_gcs_url/${execution_id}, where - ${execution_id} is a string that is generated by PerfZero which uniquely identifies the execution of one benchmark method. - If not specified, intermediate files will be stored in a local folder on the host. - ''') - parser.add_argument( - '--bigquery_project_name', - default=None, - type=str, - help='''If both --bigquery_project_name and --bigquery_dataset_table_name are specified, for each benchmark method, the benchmark - summary will be uploaded to the specified bigquery table whose schema is defined in perfzero/scripts/create_big_table.txt. - The value of each field can in turn be a json-formatted string. See README.md for example output. - ''') - parser.add_argument( - '--bigquery_dataset_table_name', - default=None, - type=str, - help='''If both --bigquery_project_name and --bigquery_dataset_table_name are specified, for each benchmark method, the benchmark - summary will be uploaded to the specified bigquery table whose schema is defined in perfzero/scripts/create_big_table.txt. - The value of each field can in turn be a json-formatted string. See README.md for example output. - ''') - parser.add_argument( - '--python_path', - default=None, - type=str, - help='''A string of format path_1,path_2,... For each ${path} specified in the string, - path_to_perfzero/${workspace}/site-packages/${path} will be added to python path so that libraies downloaded by --git_repos can - be loaded and executed. - ''') - parser.add_argument( - '--workspace', - default='workspace', - type=str, - help='''The log files, gcloud key file and git repositories will be generated and downloaded under the - directory path_to_perfzero/${workspace} - ''') - parser.add_argument( - '--root_data_dir', - default='/data', - type=str, - help='The directory which should contain the dataset required by the becnhmark method.' - ) - parser.add_argument( - '--data_downloads', - default=None, - type=str, - help='''A string of format url_1;relative_path_1,url_2;relative_path_2,... - Data will be copied from ${url} to ${root_data_dir}/${relative_path}. ${relative_path} can be skipped if it is the same as the - base name of the url, which shortens the format to url_1,url_2,... ${root_data_dir} is specified by the flag --root_data_dir. - File will be de-compressed in ${root_data_dir} if its name ends with 'gz'. Only url prefixed with gcs, http or https are supported. - Each url can start with file://, gcs://, http:// or https://. - ''') - parser.add_argument( - '--gcloud_key_file_url', - default='gs://tf-performance/auth_tokens/benchmark_upload_gce.json', - type=str, - help='''The gcloud key file url. When specified, it will be downloaded to the - directory specified by the flag --workspace. Each url can start with file://, gcs://, http:// or https://. - The key file will then be activated and used as gcloud authentication credential. - ''') - parser.add_argument( - '--debug', - action='store_true', - help='If set, use debug level logging. Otherwise, use info level logging' - ) - parser.add_argument( - '--profiler_enabled_time', - default=None, - type=str, - help='''A string of format begin_time_1:end_time_1,begin_time_2:end_time_2,.... PerfZero will start to collect profiler - data ${begin_time} sec after benchmark method execution starts. The data collection continues for ${end_time - begin_time} - sec or until the benchmark method execution finishes, whichever occurs first. If ${end_time} is not explicitly - specified, it is assumed to be MAX_LONG. - ''') - parser.add_argument( - '--execution_id', - default=None, - type=str, - help='A string that uniquely identifies the benchmark execution.') - parser.add_argument( - '--result_upload_methods', - default=None, - type=str, - help='A comma separated list of class.method values to upload results.') - parser.add_argument( - '--tpu_parameters', - default=None, - type=str, - help='''A json dictionary of cloud tpu parameters. The format must look like the following: - {"name": "my-tpu-name", project": "my-gcp-project-id", "zone": "europe-west4-a", "size": "v3-8", "version": "nightly-2.x"} - It can have an optional key value pair "version_id" -> "nightly version" to change the tpu version id. - Example "version_id": "2.4.0-dev20200728". - ''') - parser.add_argument( - '--perfzero_constructor_args', - nargs='*', - default='', - type=str, - help='''A json dictionary of additional args to pass to the perfzero - constructor.''' - ) - parser.add_argument( - '--benchmark_class_type', - default=None, - type=str, - help='''The benchmark class type. If none, assumed perfzero_benchmark. Set to "tf_benchmark" - for tf.test.Benchmark benchmarks.''') - - -class PerfZeroConfig(object): - """Creates and contains config for PerfZero.""" - - def __init__(self, mode, flags=None): - self.mode = mode - self.flags = flags - if mode == 'flags': - self.gcs_downloads_str = flags.gcs_downloads - self.data_downloads_str = flags.data_downloads - self.git_repos_str = flags.git_repos - self.benchmark_method_patterns = [] - for value in flags.benchmark_methods: - self.benchmark_method_patterns.extend(value.split(',')) - self.ml_framework_build_label = flags.ml_framework_build_label - self.execution_label = flags.execution_label - self.platform_name = flags.platform_name - self.system_name = flags.system_name - self.output_gcs_url = flags.output_gcs_url - self.scratch_gcs_url = flags.scratch_gcs_url - self.bigquery_project_name = flags.bigquery_project_name - self.bigquery_dataset_table_name = flags.bigquery_dataset_table_name - self.python_path_str = flags.python_path - self.workspace = flags.workspace - self.benchmark_class_type = flags.benchmark_class_type - self.use_cached_site_packages = flags.use_cached_site_packages - self.root_data_dir = flags.root_data_dir - self.gcloud_key_file_url = flags.gcloud_key_file_url - self.profiler_enabled_time_str = flags.profiler_enabled_time - self.execution_id = flags.execution_id - self.result_upload_methods = flags.result_upload_methods - self.perfzero_constructor_args = flags.perfzero_constructor_args - self.benchmark_num_trials = flags.benchmark_num_trials - if flags.tpu_parameters: - self.tpu_parameters = json.loads(flags.tpu_parameters) - else: - self.tpu_parameters = None - - if not flags.benchmark_methods: - logging.warning('No benchmark method is specified by ' - '--benchmark_methods') - - if flags.bigquery_project_name and not flags.bigquery_dataset_table_name: - raise ValueError('--bigquery_project_name is specified but ' - '--bigquery_dataset_table_name is not') - - if not flags.bigquery_project_name and flags.bigquery_dataset_table_name: - raise ValueError('--bigquery_dataset_table_name is specified but ' - '--bigquery_project_name is not') - - def get_env_vars(self): - env_vars = {} - for key in os.environ.keys(): - if key.startswith('PERFZERO_'): - env_vars[key] = os.environ[key] - return env_vars - - def get_flags(self): - not_none_flags = {} - for key in vars(self.flags): - value = getattr(self.flags, key) - if value is not None: - not_none_flags[key] = value - return not_none_flags - - def get_git_repos(self, site_packages_dir): - """Parse git repository string.""" - git_repos = [] - if not self.git_repos_str: - return git_repos - - for repo_entry in self.git_repos_str.split(','): - parts = repo_entry.split(';') - git_repo = {} - git_repo['url'] = parts[0] - # Assume the git url has format */{dir_name}.git - git_repo['dir_name'] = parts[0].rsplit('/', 1)[-1].rsplit('.', 1)[0] - git_repo['local_path'] = os.path.join(site_packages_dir, - git_repo['dir_name']) - if len(parts) >= 2: - git_repo['branch'] = parts[1] - if len(parts) >= 3: - git_repo['git_hash'] = parts[2] - git_repos.append(git_repo) - - return git_repos diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/perfzero_config_test.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/perfzero_config_test.py deleted file mode 100644 index e02cd9f4645454df56522559a8f663acc07f3daf..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/perfzero_config_test.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for perfzero_config.py.""" -from __future__ import print_function - -import os -import unittest -import perfzero.perfzero_config as perfzero_config - - -class TestPerfZeroConfig(unittest.TestCase): - - def test_get_git_repos(self): - config = perfzero_config.PerfZeroConfig(mode='mock') - config.git_repos_str = 'https://github.com/tensorflow/benchmarks.git;branch_1;hash_1,https://github.com/tensorflow/models.git;branch_2' - git_repos = config.get_git_repos('/site_package_dir') - - git_repo_1 = {} - git_repo_1['url'] = 'https://github.com/tensorflow/benchmarks.git' - git_repo_1['dir_name'] = 'benchmarks' - git_repo_1['local_path'] = '/site_package_dir/benchmarks' - git_repo_1['branch'] = 'branch_1' - git_repo_1['git_hash'] = 'hash_1' - - git_repo_2 = {} - git_repo_2['url'] = 'https://github.com/tensorflow/models.git' - git_repo_2['dir_name'] = 'models' - git_repo_2['local_path'] = '/site_package_dir/models' - git_repo_2['branch'] = 'branch_2' - - self.assertEqual(2, len(git_repos)) - self.assertEqual(git_repo_1, git_repos[0]) - self.assertEqual(git_repo_2, git_repos[1]) - - def test_get_env_vars(self): - config = perfzero_config.PerfZeroConfig(mode='mock') - self.assertEqual({}, config.get_env_vars()) - - os.environ['PERFZERO_VAR1'] = 'value1' - self.assertEqual({'PERFZERO_VAR1': 'value1'}, config.get_env_vars()) - - diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/process_info_tracker.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/process_info_tracker.py deleted file mode 100644 index 4d8c27379d22a1240f6bd1ee403e3ce8b809b08c..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/process_info_tracker.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Keep track of process information such as maximum memory usage with a separate thread.""" - -from __future__ import absolute_import - -import json -import logging -import os -import sched -import threading -import time -import traceback -import psutil - - -class ProcessInfoTracker(object): - """Keep track of process information such as maximum memory usage with separate thread.""" - - def __init__(self, output_dir): - self.process_info_log = open(os.path.join(output_dir, 'process_info.log'), - 'w') - self.scheduler = sched.scheduler(time.time, time.sleep) - self.process_info = {} - self.process_info['max_rss'] = 0 - self.process_info['max_vms'] = 0 - self.process_info['max_cpu_percent'] = 0 - self.exit_event = threading.Event() - self.last_exception = None - self.start_time = None - - def start(self): - self.start_time = time.time() - # 4th positional arg added to support Python2 for the short-term. - self.scheduler.enter(1, 1, self._update_process_info, ()) # pylint: disable=no-value-for-parameter - threading.Thread(target=self.scheduler.run).start() - logging.info('Started process information tracker.') - - def stop(self): - self.exit_event.set() - self.process_info_log.flush() - logging.info('Stopped process information tracker.') - - if self.last_exception is not None: - raise self.last_exception # pylint: disable=raising-bad-type - - return dict(self.process_info) - - def _update_process_info(self): - """Read and update process info using background thread every 1 second.""" - try: - p = psutil.Process(os.getpid()) - memory_info = p.memory_info() - # This is a blocking call which takes 0.1 second. - # This affects the interval # at which the metrics are reported - cpu_percent = p.cpu_percent(interval=0.1) - - self.process_info['max_rss'] = max(self.process_info['max_rss'], - memory_info.rss) - self.process_info['max_vms'] = max(self.process_info['max_vms'], - memory_info.vms) - self.process_info['max_cpu_percent'] = max( - self.process_info['max_cpu_percent'], cpu_percent) - - entry = {} - entry['time'] = time.time() - self.start_time - entry['rss'] = memory_info.rss - entry['vms'] = memory_info.vms - entry['cpu_percent'] = cpu_percent - self.process_info_log.write(json.dumps(entry) + '\n') - if not self.exit_event.is_set(): - # Schedule the next event to be run after 1 second - # 4th positional arg added to support Python2 for the short-term. - self.scheduler.enter(1, 1, self._update_process_info, ()) # pylint: disable=no-value-for-parameter - except Exception as e: # pylint: disable=broad-except - logging.error('Process info tracker failed due to error:\n %s', - traceback.format_exc()) - self.last_exception = e - - diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/report_utils.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/report_utils.py deleted file mode 100644 index 5a7441f0e5c821d562e2381db21257aefbdd1484..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/report_utils.py +++ /dev/null @@ -1,237 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Upload test results.""" -from __future__ import print_function - -import importlib -import json -import logging -import perfzero.utils as utils -import psutil -import socket - -from six import u as unicode # pylint: disable=W0622 - - -def execute_methods(method_names_str, *args, **kwargs): - """Calls a list of method names on given function params. - - Args: - method_names_str: String - Comma-separated module.foo.bar.method strings. - This function imports module.foo.bar for each such method and calls it - with *args and **kwargs. - *args: Function params common to each method. - **kwargs: Function params common to each method. - - Raises: - RuntimeError: If any of the invoked methods raised an exception. - """ - if not method_names_str: - return - errors = [] - module_methods_list = method_names_str.split(',') - for module_method in module_methods_list: - try: - logging.info('Trying to call %s', module_method) - module_path, method_path = module_method.rsplit('.', 1) - this_module = importlib.import_module(module_path) - logging.info('Found module %s, looking for %s', module_path, method_path) - this_method = getattr(this_module, method_path) - logging.info('Found method %s', method_path) - this_method(*args, **kwargs) - except Exception as e: # pylint: disable=broad-except - errors.append(str(e)) - if errors: - raise RuntimeError('\n' + '\n'.join(errors)) - - -def upload_execution_summary(bigquery_project_name, bigquery_dataset_table_name, - execution_summary): - """Upload benchmark summary. - - Note: Using stream=False has a 1000 per day insert limit per table. Using - stream=True, the documented limit is 50K+. With streaming there can be - a small and possibly not noticeable delay to seeing the results the BigQuery - UI, but there can be a 90 minute more or less delay in the results being part - of exports. - - Note: BigQuery maps unicode() to STRING for python2. If str is used that is - mapped to BYTE. - - Args: - bigquery_project_name: Name of the gcp project. - bigquery_dataset_table_name: data_set and table name. - execution_summary: benchmark summary dictionary of results. - """ - - # pylint: disable=C6204 - import google.auth - from google.cloud import bigquery - - if not bigquery_project_name: - logging.info( - 'Skipped uploading benchmark result to bigquery because bigquery table name is not set.' - ) - return - - if not bigquery_dataset_table_name: - logging.info( - 'Skipped uploading benchmark result to bigquery because bigquery project name is not set.' - ) - return - - credentials = google.auth.default()[0] - dataset_name = bigquery_dataset_table_name.split('.')[0] - table_name = bigquery_dataset_table_name.split('.')[1] - client = bigquery.Client( - project=bigquery_project_name, credentials=credentials) - - benchmark_summary_input = {} - for key, value in execution_summary.items(): - if isinstance(value, dict): - benchmark_summary_input[key] = unicode(json.dumps(value)) - else: - benchmark_summary_input[key] = unicode(value) - logging.debug('Bigquery input for benchmark_summary table is %s', - json.dumps(benchmark_summary_input, indent=2)) - - errors = [] - # TODO(tobyboyd): Shim to direct results to new table until all jobs - # are updated. - if 'benchmark_results' in dataset_name: - if dataset_name == 'benchmark_results_dev': - table_ref = client.dataset('perfzero_dev').table('benchmark_summary') - table_obj = client.get_table(table_ref) - elif dataset_name == 'benchmark_results': - table_ref = client.dataset('perfzero').table('benchmark_summary') - table_obj = client.get_table(table_ref) - else: - table_ref = client.dataset(dataset_name).table(table_name) - table_obj = client.get_table(table_ref) - - errors.extend(client.insert_rows(table_obj, [benchmark_summary_input])) - - if errors: - logging.error( - 'Failed to upload benchmark result to bigquery due to errors %s', - errors) - else: - logging.info( - 'Uploaded benchmark result to the table %s of the bigquery project %s.', - bigquery_dataset_table_name, - bigquery_project_name) - - -def build_benchmark_result(raw_benchmark_result, has_exception, trial_id): - """Converts test_log.proto format to PerfZero format.""" - benchmark_result = {} - benchmark_result['name'] = raw_benchmark_result['name'] - benchmark_result['wall_time'] = raw_benchmark_result['wall_time'] - - succeeded = not has_exception - extras = [] - for name in raw_benchmark_result.get('extras', {}): - entry = {} - entry['name'] = name - - if 'double_value' in raw_benchmark_result['extras'][name]: - entry['value'] = raw_benchmark_result['extras'][name]['double_value'] - else: - entry['value'] = raw_benchmark_result['extras'][name]['string_value'] - extras.append(entry) - - metrics = [] - for metric in raw_benchmark_result.get('metrics', []): - value = metric['value'] - if 'min_value' in metric and metric['min_value'] > value: - succeeded = False - if 'max_value' in metric and metric['max_value'] < value: - succeeded = False - metrics.append(metric) - - benchmark_result['succeeded'] = succeeded - benchmark_result['extras'] = extras - benchmark_result['metrics'] = metrics - benchmark_result['trial_id'] = trial_id - - return benchmark_result - - -def build_execution_summary(execution_timestamp, execution_id, - ml_framework_build_label, execution_label, - platform_name, system_name, output_gcs_url, - benchmark_result, env_vars, flags, harness_info, - site_package_info, process_info, has_exception, - is_tpu_benchmark): - """Builds summary of the execution.""" - # Avoids module not found during setup phase when tf is not installed yet. - # pylint: disable=C6204 - import tensorflow as tf - - benchmark_info = {} - benchmark_info['harness_name'] = 'perfzero' - benchmark_info['harness_info'] = harness_info - benchmark_info['has_exception'] = has_exception - if execution_label: - benchmark_info['execution_label'] = execution_label - if output_gcs_url: - benchmark_info['output_url'] = '{}/{}/'.format(output_gcs_url, execution_id) - if env_vars: - benchmark_info['env_vars'] = env_vars - if flags: - benchmark_info['flags'] = flags - benchmark_info['site_package_info'] = site_package_info - - ml_framework_info = {} - ml_framework_info['name'] = 'tensorflow' - ml_framework_info['version'] = tf.__version__ - # tf.__git_version__ in Python3 has format b'version_string' - if tf.__git_version__[0] == 'b': - ml_framework_info['build_version'] = tf.__git_version__[2:-1] - else: - ml_framework_info['build_version'] = tf.__git_version__ - - if ml_framework_build_label: - ml_framework_info['build_label'] = ml_framework_build_label - - system_info = {} - if platform_name: - system_info['platform_name'] = platform_name - if system_name: - system_info['system_name'] = system_name - if not is_tpu_benchmark: - gpu_info = utils.get_gpu_info() - if gpu_info: - system_info['accelerator_driver_version'] = gpu_info['gpu_driver_version'] - system_info['accelerator_model'] = gpu_info['gpu_model'] - system_info['accelerator_count'] = gpu_info['gpu_count'] - system_info['cpu_model'] = utils.get_cpu_name() - system_info['physical_cpu_count'] = psutil.cpu_count(logical=False) - system_info['logical_cpu_count'] = psutil.cpu_count(logical=True) - system_info['cpu_socket_count'] = utils.get_cpu_socket_count() - system_info['hostname'] = socket.gethostname() - - execution_summary = {} - execution_summary['execution_id'] = execution_id - execution_summary['execution_timestamp'] = execution_timestamp - execution_summary['benchmark_result'] = benchmark_result - execution_summary['benchmark_info'] = benchmark_info - execution_summary['setup_info'] = {} - execution_summary['ml_framework_info'] = ml_framework_info - execution_summary['system_info'] = system_info - if process_info: - execution_summary['process_info'] = process_info - - return execution_summary diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/tensorflow_profiler.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/tensorflow_profiler.py deleted file mode 100644 index bb1839c4d2bec091a852d1ca0da1b116ccc8b31d..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/tensorflow_profiler.py +++ /dev/null @@ -1,128 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Collect profiler data for Tensorboard with a separate thread.""" - -from __future__ import print_function - -import logging -import os -import sched -import threading -import time -import traceback - -import perfzero.utils as utils - - -def _start_profiler(output_dir): - """Start profiler. - - Args: - output_dir: log directory to place the profiler data - """ - import tensorflow as tf # pylint: disable=g-import-not-at-top - - profiler_data_dir = os.path.join(output_dir, 'profiler_data') - utils.make_dir_if_not_exist(profiler_data_dir) - logging.info('Starting TensorFlow profiler and saving data to dir %s', - profiler_data_dir) - try: - tf.profiler.experimental.start(profiler_data_dir) - logging.info('Started TensorFlow profiler') - except Exception: # pylint: disable=broad-except - logging.error('TensorFlow profiler failed to start due to error:\n %s', - traceback.format_exc()) - - -def _stop_profiler(): - """Stop profiler.""" - - import tensorflow as tf # pylint: disable=g-import-not-at-top - - try: - tf.profiler.experimental.stop() - logging.info('Stopped TensorFlow profiler.') - except Exception: # pylint: disable=broad-except - logging.error('TensorFlow profiler failed to stop due to error:\n %s', - traceback.format_exc()) - - -class TensorFlowProfiler(object): - """Collect profiler data for Tensorboard with a separate thread.""" - - def __init__(self, profiler_enabled_time_str, output_dir): - """Constructor. - - Args: - profiler_enabled_time_str: the value of the config --profiler_enabled_time - output_dir: log directory to place the profiler data - """ - - self.profiler_enabled_time_str = profiler_enabled_time_str - self.output_dir = output_dir - self.exit_event = threading.Event() - self.scheduler = sched.scheduler(time.time, self._sleep_until_exit) - - def _sleep_until_exit(self, timeout): - start_time = time.time() - cur_time = time.time() - while cur_time - start_time < timeout and not self.exit_event.is_set(): - time.sleep(min(1, timeout + start_time - cur_time)) - cur_time = time.time() - - def start(self): - """Schedule start/stop profiler event specified in profiler_enabled_time_str.""" - - if not self.profiler_enabled_time_str: - return - - last_end_time = -1 - for time_str in self.profiler_enabled_time_str.split(','): - begin_time = int(time_str.split(':')[0].strip()) - end_time_str = time_str.split(':')[1].strip() if ':' in time_str else None - end_time = int(end_time_str) if end_time_str else 365 * 24 * 60 * 60 - if begin_time <= last_end_time: - raise ValueError('begin_time {} is no larger than the last ' - 'end_time {}'.format(begin_time, last_end_time)) - if end_time <= begin_time: - raise ValueError('end_time {} is no larger than begin_time {}'.format( - end_time, begin_time)) - # 4th positional arg added to support Python2 for the short-term. - self.scheduler.enter(begin_time, 1, _start_profiler, - argument=(self.output_dir,)) - self.scheduler.enter(end_time, 1, _stop_profiler, ()) # pylint: disable=no-value-for-parameter - last_end_time = end_time - - threading.Thread(target=self.scheduler.run).start() - - def stop(self): - """Stop scheduler and save profiler data if any event is cancelled.""" - - event_canceled = False - for event in self.scheduler.queue: - try: - self.scheduler.cancel(event) - event_canceled = True - except ValueError: - # This is OK because the event may have been just canceled - pass - - # Signal the scheduler thread to stop sleeping - self.exit_event.set() - - # Save the profiler data if any event is canceled - if event_canceled: - _stop_profiler() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/test_files/example_nvidia-smi_no_processes.txt b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/test_files/example_nvidia-smi_no_processes.txt deleted file mode 100644 index a467ea6436e5e9eea1a25ee1733d07d0f1e615eb..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/test_files/example_nvidia-smi_no_processes.txt +++ /dev/null @@ -1,40 +0,0 @@ -Tue Jan 9 09:34:25 2018 -+-----------------------------------------------------------------------------+ -| NVIDIA-SMI 384.81 Driver Version: 384.81 | -|-------------------------------+----------------------+----------------------+ -| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | -| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | -|===============================+======================+======================| -| 0 Tesla P100-SXM2... On | 00000000:06:00.0 Off | 0 | -| N/A 50C P0 196W / 300W | 15643MiB / 16276MiB | 97% Default | -+-------------------------------+----------------------+----------------------+ -| 1 Tesla P100-SXM2... On | 00000000:07:00.0 Off | 0 | -| N/A 41C P0 50W / 300W | 15483MiB / 16276MiB | 0% Default | -+-------------------------------+----------------------+----------------------+ -| 2 Tesla P100-SXM2... On | 00000000:0A:00.0 Off | 0 | -| N/A 33C P0 48W / 300W | 15483MiB / 16276MiB | 0% Default | -+-------------------------------+----------------------+----------------------+ -| 3 Tesla P100-SXM2... On | 00000000:0B:00.0 Off | 0 | -| N/A 34C P0 49W / 300W | 15483MiB / 16276MiB | 0% Default | -+-------------------------------+----------------------+----------------------+ -| 4 Tesla P100-SXM2... On | 00000000:85:00.0 Off | 0 | -| N/A 36C P0 50W / 300W | 15483MiB / 16276MiB | 0% Default | -+-------------------------------+----------------------+----------------------+ -| 5 Tesla P100-SXM2... On | 00000000:86:00.0 Off | 0 | -| N/A 33C P0 48W / 300W | 15483MiB / 16276MiB | 0% Default | -+-------------------------------+----------------------+----------------------+ -| 6 Tesla P100-SXM2... On | 00000000:89:00.0 Off | 0 | -| N/A 38C P0 48W / 300W | 15483MiB / 16276MiB | 0% Default | -+-------------------------------+----------------------+----------------------+ -| 7 Tesla P100-SXM2... On | 00000000:8A:00.0 Off | 0 | -| N/A 34C P0 49W / 300W | 15483MiB / 16276MiB | 0% Default | -+-------------------------------+----------------------+----------------------+ - -+-----------------------------------------------------------------------------+ -| Processes: GPU Memory | -| GPU PID Type Process name Usage | -|=============================================================================| -| No running processes found | -+-----------------------------------------------------------------------------+ - - diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/test_files/example_nvidia-smi_processes.txt b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/test_files/example_nvidia-smi_processes.txt deleted file mode 100644 index 2eedb2f25d7dca69f89fbdf8dc267b248c6d2480..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/test_files/example_nvidia-smi_processes.txt +++ /dev/null @@ -1,43 +0,0 @@ -Tue Jan 9 09:34:25 2018 -+-----------------------------------------------------------------------------+ -| NVIDIA-SMI 384.81 Driver Version: 384.81 | -|-------------------------------+----------------------+----------------------+ -| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | -| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | -|===============================+======================+======================| -| 0 Tesla P100-SXM2... On | 00000000:06:00.0 Off | 0 | -| N/A 50C P0 196W / 300W | 15643MiB / 16276MiB | 97% Default | -+-------------------------------+----------------------+----------------------+ -| 1 Tesla P100-SXM2... On | 00000000:07:00.0 Off | 0 | -| N/A 41C P0 50W / 300W | 15483MiB / 16276MiB | 0% Default | -+-------------------------------+----------------------+----------------------+ -| 2 Tesla P100-SXM2... On | 00000000:0A:00.0 Off | 0 | -| N/A 33C P0 48W / 300W | 15483MiB / 16276MiB | 0% Default | -+-------------------------------+----------------------+----------------------+ -| 3 Tesla P100-SXM2... On | 00000000:0B:00.0 Off | 0 | -| N/A 34C P0 49W / 300W | 15483MiB / 16276MiB | 0% Default | -+-------------------------------+----------------------+----------------------+ -| 4 Tesla P100-SXM2... On | 00000000:85:00.0 Off | 0 | -| N/A 36C P0 50W / 300W | 15483MiB / 16276MiB | 0% Default | -+-------------------------------+----------------------+----------------------+ -| 5 Tesla P100-SXM2... On | 00000000:86:00.0 Off | 0 | -| N/A 33C P0 48W / 300W | 15483MiB / 16276MiB | 0% Default | -+-------------------------------+----------------------+----------------------+ -| 6 Tesla P100-SXM2... On | 00000000:89:00.0 Off | 0 | -| N/A 38C P0 48W / 300W | 15483MiB / 16276MiB | 0% Default | -+-------------------------------+----------------------+----------------------+ -| 7 Tesla P100-SXM2... On | 00000000:8A:00.0 Off | 0 | -| N/A 34C P0 49W / 300W | 15483MiB / 16276MiB | 0% Default | -+-------------------------------+----------------------+----------------------+ - -+-----------------------------------------------------------------------------+ -| Processes: GPU Memory | -| GPU PID Type Process name Usage | -|=============================================================================| -| 0 44454 C /usr/bin/python 15631MiB | -| 1 44454 C /usr/bin/python 15471MiB | -| 2 44454 C /usr/bin/python 15471MiB | -| 3 44454 C /usr/bin/python 15471MiB | -+-----------------------------------------------------------------------------+ - - diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/test_files/nvme_device_log.txt b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/test_files/nvme_device_log.txt deleted file mode 100644 index 96965acf22ae921fd2202646892b57748bc896d3..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/test_files/nvme_device_log.txt +++ /dev/null @@ -1,15 +0,0 @@ -NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT -nvme0n8 259:7 0 375G 0 disk -nvme0n6 259:5 0 375G 0 disk -sdb 8:16 0 50G 0 disk - -└─sdb1 8:17 0 50G 0 part /tmpfs -nvme0n4 259:3 0 375G 0 disk -nvme0n2 259:1 0 375G 0 disk -nvme0n7 259:6 0 375G 0 disk -nvme0n5 259:4 0 375G 0 disk -sda 8:0 0 100G 0 disk - -└─sda1 8:1 0 100G 0 part / -nvme0n3 259:2 0 375G 0 disk -nvme0n1 259:0 0 375G 0 disk diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/tpu_runtime_utils.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/tpu_runtime_utils.py deleted file mode 100644 index 2cbcaa92afea839c92eac039570ced74c36d0d2b..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/tpu_runtime_utils.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Utility to manage the tpu version before starting the benchmark.""" - -import json -from absl import logging - -from six.moves.urllib import request - -try: - from cloud_tpu_client import client # pylint: disable=g-import-not-at-top -except ImportError: - print( - 'Falling back to TensorFlow client; we recommended you install the Cloud ' - 'TPU client directly with pip install cloud-tpu-client.') - from tensorflow.python.tpu.client import client # pylint: disable=g-import-not-at-top - - -def _as_text(s): - """Converts a byte/string into string.""" - if isinstance(s, bytes): - return s.decode('utf-8') - return s - - -def _get_content(url): - """Opens the url and loads the response into json.""" - logging.info('opening url %s', url) - req = request.Request(url) - resp = request.urlopen(req) - resp_text = _as_text(resp.read()) - logging.info('response text = %s', resp_text) - return json.loads(resp_text) - - -def _get_version_info(url, version_label): - """Constructs a version info from the response.""" - json_data = _get_content(url) - logging.info('json_data = %s', json_data) - if 'currentVersion' in json_data: - commit_id = json_data['currentVersion'] - elif 'buildLabel' in json_data: - commit_id = json_data['buildLabel'] - else: - commit_id = '' - - info = { - 'url': '', - 'hash': commit_id, - 'branch': version_label, - 'piper_id': json_data.get('piperOriginRevId', '') - } - return info - - - -def _configure_tpu_version(tpu_name, version_label, new_version_id): - """Returns the current tpu version after resetting to an optional version.""" - # The tpu_name is arbitrary / user chosen unique string for this tpu. - logging.info('Trying to connect to tpu %s', tpu_name) - tpu_client = client.Client(tpu=tpu_name) - tpu_client.wait_for_healthy() - - if new_version_id: - logging.info('Trying to reset tpu version to %s', new_version_id) - tpu_client.configure_tpu_version(version=new_version_id) - tpu_client.wait_for_healthy() - logging.info('TPU healthy after version reset.') - else: - logging.info('Using the default tpu version id.') - - workers = tpu_client.network_endpoints() - if workers: - ip_addr = workers[0]['ipAddress'] - url = 'http://{}:8475/requestversion'.format(ip_addr) - return _get_version_info(url, version_label) - else: - logging.error('No tpu endpoint info') - return { - 'url': '', - 'hash': '', - 'branch': version_label, - 'piper_id': '', - } - - -def configure_tpu(tpu_params): - return _configure_tpu_version( - tpu_params.get('name'), - version_label=tpu_params.get('version'), - new_version_id=tpu_params.get('version_id')) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/utils.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/utils.py deleted file mode 100644 index 3a4f28ed695e85c63da5a5bf0911e3a0bec94d2a..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/utils.py +++ /dev/null @@ -1,546 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""PerfZero utility methods.""" -from __future__ import print_function - -import importlib -import logging -import os -import shutil -import subprocess -import sys -import threading -import traceback -import requests -import json -import re - -def create_empty_file(parent_directory, file_basename): - """Creates an empty file with a given basename in a parent directory. - - Creates parent_directory and intermediate directories if it doesn't exist. - This is mostly used for creating no-op actions in the Dockerfile. - - Args: - parent_directory: The path to the parent directory. - file_basename: The basename for the empty file. - """ - if not os.path.isdir(parent_directory): - os.makedirs(parent_directory) - full_file_name = os.path.join(parent_directory, file_basename) - with open(full_file_name, 'w'): - print('Creating empty file: {}'.format(full_file_name)) - - -def checkout_git_repos(git_repos, use_cached_site_packages): - """Clone, update, or sync a repo. - - Args: - git_repos: array of dict containing attributes of the git repo to checkout. - use_cached_site_packages: If true, skip git pull if git_repo already exists. - - Returns: - A dict containing attributes of the git repositories - """ - site_package_info = {} - for repo in git_repos: - logging.info('Checking out repository from %s to %s', - repo['url'], repo['local_path']) - if not os.path.isdir(repo['local_path']): - run_commands(['git clone {} {}'.format(repo['url'], repo['local_path'])]) - if 'branch' in repo: - run_commands(['git -C {} checkout {}'.format( - repo['local_path'], repo['branch'])]) - if not use_cached_site_packages or 'git_hash' in repo: - run_commands(['git -C {} pull --rebase'.format(repo['local_path'])]) - if 'git_hash' in repo: - run_commands(['git -C {} reset --hard {}'.format( - repo['local_path'], repo['git_hash'])]) - logging.info('Checked-out repository from %s to %s', - repo['url'], repo['local_path']) - site_package_info[repo['dir_name']] = get_git_repo_info(repo['local_path']) - - return site_package_info - - -def get_git_repo_info(local_path): - """Get information of the git repository specified by the local_path.""" - git_repo_info = {} - - # Get git url - cmd = 'git -C {} config --get remote.origin.url'.format(local_path) - exit_code, result = run_command(cmd) - lines = result.splitlines() - if exit_code == 0 and lines: - git_repo_info['url'] = lines[0] - else: - logging.error('Error getting git url for repository %s due to %s', - local_path, result) - return {} - - # Get git branch - cmd = 'git -C {} rev-parse --abbrev-ref HEAD'.format(local_path) - exit_code, result = run_command(cmd) - lines = result.splitlines() - if exit_code == 0 and lines: - git_repo_info['branch'] = lines[0] - else: - logging.error('Error getting git branch for repository %s due to %s', - local_path, result) - return {} - - # Get git hash - cmd = 'git -C {} rev-parse HEAD'.format(local_path) - exit_code, result = run_command(cmd) - lines = result.splitlines() - if exit_code == 0 and lines: - git_repo_info['hash'] = lines[0] - else: - logging.error('Error getting git hash for repository %s due to %s', - local_path, result) - return {} - - return git_repo_info - - -def setup_python_path(site_packages_dir, python_path_str): - if python_path_str: - python_paths = python_path_str.split(',') - for python_path in python_paths: - logging.info('Adding path %s to sys.path', python_path) - sys.path.append(os.path.join(site_packages_dir, python_path)) - logging.debug('PYTHONPATH: %s', sys.path) - - -def active_gcloud_service(gcloud_key_file_url, workspace_dir, - download_only=False): - """Download key file and setup gcloud service credential using the key file. - - Args: - gcloud_key_file_url: gcloud key file url - workspace_dir: directory that the key file is downloaded to - download_only: skip setting up the gcloud service credential if this is true - """ - - if not gcloud_key_file_url: - return - - local_path = os.path.join(workspace_dir, - os.path.basename(gcloud_key_file_url)) - if not os.path.exists(local_path): - download_data([{'url': gcloud_key_file_url, 'local_path': local_path}]) - - if not download_only: - os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = local_path - run_commands(['gcloud auth activate-service-account --key-file {}'.format( - local_path)]) - logging.info('Activated gcloud service account credential') - - -def setup_gsutil_credential(): - run_commands(['gcloud config set pass_credentials_to_gsutil true']) - - -def download_data(download_infos): - """Download data from url to local_path for each (url, local_path) pair in the download_infos. - - Each url should start with either gs://, http:// or https:// - Downloaded file whose name ends with .gz will be decompressed in its - current directory - - Args: - download_infos: array of dict which specifies the url and local_path for - data download - """ - for info in download_infos: - if os.path.exists(info['local_path']): - continue - original_base_name = os.path.basename(info['url']) - expected_base_name = os.path.basename(info['local_path']) - local_path_parent = os.path.dirname(info['local_path']) - - logging.info('Downloading data from %s to %s', - info['url'], info['local_path']) - make_dir_if_not_exist(local_path_parent) - # Download data to the local path - if info['url'].startswith('http://') or info['url'].startswith('https://'): - request = requests.get(info['url'], allow_redirects=True) - f = open(info['local_path'], 'wb') - f.write(request.content) - f.close() - elif info['url'].startswith('gs://'): - cmd = ['gsutil', '-m', 'cp', '-r', '-n', info['url'], local_path_parent] - run_commands([cmd], shell=False) - elif info['url'].startswith('file://'): - cmd = ['cp', info['url'][7:], local_path_parent] - run_commands([cmd], shell=False) - else: - raise ValueError('Url {} with prefix {} is not supported.'.format( - info['url'], info['url'].split(':')[0])) - # Move data to the expected local path - if original_base_name != expected_base_name: - run_commands(['mv {} {}'.format( - os.path.join(local_path_parent, original_base_name), - os.path.join(local_path_parent, expected_base_name))]) - logging.info('Downloaded data from %s to %s', - info['url'], info['local_path']) - # Decompress file if file name ends with .gz unless caller sets 'decompress' - # to False in info. - if info['url'].endswith('.gz') and info.get('decompress', True): - run_commands(['tar xvf {} -C {}'.format( - info['local_path'], local_path_parent)]) - logging.info('Decompressed file %s', info['local_path']) - - -def parse_data_downloads_str(root_data_dir, data_downloads_str): - """Parse a comma separated string into array of dicts. - - Each dict specifies the url and local_path for a download. - - Args: - root_data_dir: the directory which should contain all the dataset files - data_downloads_str: a comma separated string specified by the - flag --data_downloads - - Returns: - An array of dict which specifies the url and local_path for data download - """ - - download_infos = [] - if not data_downloads_str: - return download_infos - - for entry in data_downloads_str.split(','): - info = {} - if ';' in entry: - info['url'] = entry.split(';')[0] - info['local_path'] = os.path.join(root_data_dir, entry.split(';')[1]) - else: - info['url'] = entry - info['local_path'] = os.path.join(root_data_dir, os.path.basename(entry)) - # Canonicalize url to remove trailing '/' and '*' - if info['url'].endswith('*'): - info['url'] = info['url'][:-1] - if info['url'].endswith('/'): - info['url'] = info['url'][:-1] - - download_infos.append(info) - - return download_infos - - -def maybe_upload_to_gcs(local_dir, output_gcs_url): - if not output_gcs_url: - return - run_commands(['gsutil -m cp -r {} {}'.format(local_dir, output_gcs_url)]) - logging.info('Uploaded data from local directory %s to gcs %s', - local_dir, output_gcs_url) - - -def make_dir_if_not_exist(local_path): - if not os.path.exists(local_path): - os.makedirs(local_path) - logging.info('Created directory %s', local_path) - - -def run_command(cmd, shell=True): - """Structures for a variety of different test results. - - Args: - cmd: Command to execute - shell: True to use shell, false otherwise. - - Returns: - Tuple of the command return value and the standard out in as a string. - """ - logging.debug('Executing command: %s', cmd) - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, shell=shell) - - exit_code = None - line = '' - stdout = '' - while exit_code is None or line: - exit_code = p.poll() - line = p.stdout.readline().decode('utf-8') - stdout += line - logging.debug(line) - - return exit_code, stdout - - -def run_commands(cmds, shell=True): - """Runs list of command and throw error if any fail.""" - for cmd in cmds: - exit_code, stdout = run_command(cmd, shell=shell) - if exit_code: - raise Exception('"{}" failed with code:{} and stdout:\n{}'.format( - cmd, exit_code, stdout)) - - -def get_cpu_name(): - cmd = "cat /proc/cpuinfo | grep 'model name' | sort --unique" - exit_code, result = run_command(cmd) - lines = result.splitlines() - if exit_code == 0 and lines: - model_name_parts = lines[0].split(':') - return model_name_parts[1].strip() - else: - logging.error('Error getting cpuinfo model name: %s', result) - return '' - - -def get_cpu_socket_count(): - cmd = 'grep -i "physical id" /proc/cpuinfo | sort -u | wc -l' - exit_code, result = run_command(cmd) - lines = result.splitlines() - if exit_code == 0 and lines: - return int(lines[0]) - else: - logging.error('Error getting cpuinfo scocket count: %s', result) - return -1 - - -def _get_amd_gpu_info(): - """Returns gpu information using rocm-smi. - - Note: Assumes if the system has multiple GPUs, that they are all the same - - Returns: - A dict containing gpu_driver_version, gpu_model and gpu_count or None if - `rocm-smi` is not found or fails. - """ - cmd = 'rocm-smi --json --showproductname --showdriverversion' - exit_code, result = run_command(cmd) - - if exit_code != 0: - logging.error('rocm-smi did not return as expected: %s', result) - return None - - def get_gpu_driver_version(rocm_smi_output): - return rocm_smi_output['system']['Driver version'] - - def get_gpu_model(rocm_smi_output): - gpu_model = "" - for key, value in rocm_smi_output.items(): - if re.match("card[0-9]+", key): - gpu_model = value['Card SKU'] - break - return gpu_model - - def get_gpu_count(rocm_smi_output): - gpu_count = 0 - for key, value in rocm_smi_output.items(): - if re.match("card[0-9]+", key): - gpu_count += 1 - return gpu_count - - rocm_smi_output= json.loads(result) - - gpu_info = {} - gpu_info['gpu_driver_version'] = get_gpu_driver_version(rocm_smi_output) - gpu_info['gpu_model'] = get_gpu_model(rocm_smi_output) - gpu_info['gpu_count'] = get_gpu_count(rocm_smi_output) - - return gpu_info - - -def _get_nvidia_gpu_info(): - """Returns gpu information using nvidia-smi. - - Note: Assumes if the system has multiple GPUs that they are all the same with - one exception. If the first result is a Quadro, the heuristic assumes - this may be a workstation and takes the second entry. - - Returns: - A dict containing gpu_driver_version, gpu_model and gpu_count or None if - `nvidia-smi` is not found or fails. - """ - cmd = 'nvidia-smi --query-gpu=driver_version,gpu_name --format=csv' - exit_code, result = run_command(cmd) - - if exit_code != 0: - logging.error('nvidia-smi did not return as expected: %s', result) - return None - - lines = result.splitlines() - gpu_info_line = lines[1] - if 'Quadro' in gpu_info_line and len(lines) >= 3: - gpu_info_line = lines[2] - - gpu_info = {} - gpu_info['gpu_driver_version'] = gpu_info_line.split(',')[0].strip() - gpu_info['gpu_model'] = gpu_info_line.split(',')[1].strip() - gpu_info['gpu_count'] = len(lines) - 1 - - return gpu_info - - -def get_gpu_info(): - """Returns gpu information using either nvidia-smi or rocm-smi. - - Returns: - A dict containing gpu_driver_version, gpu_model and gpu_count or None if - `nvidia-smi` is not found or fails. - """ - return _get_amd_gpu_info() if shutil.which("rocm-smi") \ - else _get_nvidia_gpu_info() - - -def _install_tpu_tool(): - """Installs the ctpu tool to managing cloud TPUs. - - Follows the instructions here: - https://github.com/tensorflow/tpu/tree/master/tools/ctpu - """ - if not os.path.exists('ctpu'): - logging.info('Installing TPU tool') - commands = [ - 'wget https://dl.google.com/cloud_tpu/ctpu/latest/linux/ctpu', - 'chmod a+x ctpu', - ] - run_commands(commands) - - -def setup_tpu(parameters): - """Sets up a TPU with a given set of parameters. - - Args: - parameters: dictionary of TPU parameters. - - Returns: - True if an error occurs during setup. - """ - try: - _install_tpu_tool() - - args = [ - '--name={}'.format(parameters.get('name')), - '--project={}'.format(parameters.get('project')), - '--zone={}'.format(parameters.get('zone')), - '--tpu-size={}'.format(parameters.get('size')), - '--tf-version={}'.format(parameters.get('version')), - '--tpu-only', - '-noconf', - ] - command = './ctpu up {}'.format(' '.join(args)) - logging.info('Setting up TPU: %s', command) - exit_code, output = run_command(command) - if exit_code != 0: - logging.error('Error in setup with output: %s', output) - return exit_code != 0 - except Exception: - logging.error('Unable to setup TPU') - run_command('rm -f ctpu') - sys.exit(1) - - -def cleanup_tpu(parameters): - """Cleans up an existing TPU. - - Args: - parameters: dictionary of TPU parameters. - - Returns: - True if an error occurs during cleanup. - """ - _install_tpu_tool() - - args = [ - '--name={}'.format(parameters.get('name')), - '--project={}'.format(parameters.get('project')), - '--zone={}'.format(parameters.get('zone')), - '--tpu-only', - '-noconf', - ] - command = './ctpu delete {}'.format(' '.join(args)) - logging.info('Cleaning up TPU: %s', command) - exit_code, output = run_command(command) - if exit_code != 0: - logging.error('Error in cleanup with output: %s', output) - return exit_code != 0 - - -def read_benchmark_result(benchmark_result_file_path): - """Read benchmark result from the protobuf file.""" - from google.protobuf import json_format # pylint: disable=g-import-not-at-top - from tensorflow.core.util import test_log_pb2 # pylint: disable=g-import-not-at-top - - if not os.path.isfile(benchmark_result_file_path): - logging.error('Failed to read benchmark result because ' - 'file %s does not exist', benchmark_result_file_path) - return {} - - with open(benchmark_result_file_path, 'rb') as f: - benchmark_entries = test_log_pb2.BenchmarkEntries() - benchmark_entries.ParseFromString(f.read()) - - return json_format.MessageToDict( - benchmark_entries, - preserving_proto_field_name=True, - including_default_value_fields=True)['entry'][0] - - -def print_thread_stacktrace(): - print('Here is the stacktrace for all threads:') - thread_names = {t.ident: t.name for t in threading.enumerate()} - for thread_id, frame in sys._current_frames().items(): # pylint: disable=protected-access - print('Thread {}'.format(thread_names.get(thread_id, thread_id))) - traceback.print_stack(frame) - - -def instantiate_benchmark_class( - benchmark_class, output_dir, root_data_dir, tpu, constructor_args, - benchmark_class_type=None): - """Return initialized benchmark class.""" - module_import_path, class_name = benchmark_class.rsplit('.', 1) - module = importlib.import_module(module_import_path) - class_ = getattr(module, class_name) - if benchmark_class_type == 'tf_benchmark': - # for benchmarks inheriting from tf.test.Benchmark, instantiate them directly. - instance = class_(**constructor_args) - else: - # Default instantiation for perfzero_benchmark classes. - instance = class_( - output_dir=output_dir, - root_data_dir=root_data_dir, - tpu=tpu, - **constructor_args) - - return instance - - -def copy_and_rename_dirs(dir_spec_string, dst_base_dir): - """Copies list of :new_name specs into a new dest dir. - - If a path /path1/path2/dir:new_dir is given, it copies /path1/path2/dir to - dst_base_dir/new_dir. - - Args: - dir_spec_string: Comma separated list of /path1/path2:new_name specs. - dst_base_dir: The base dir to contain the copies. - """ - if not dir_spec_string: - return - dir_specs = dir_spec_string.split(',') - for src_dir_with_name in dir_specs: - src_dir, final_basename = src_dir_with_name.split(':') - dst_dir = os.path.join(dst_base_dir, final_basename) - - if os.path.isdir(dst_dir): - logging.info('[DELETE] pre-existing %s', dst_dir) - shutil.rmtree(dst_dir) - logging.info('[COPY] %s -> %s', src_dir, dst_dir) - shutil.copytree(src_dir, dst_dir) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/utils_test.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/utils_test.py deleted file mode 100644 index 978a6aa0cf4237292ec34500c4a7d804e5e388e9..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/perfzero/utils_test.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests utils.py.""" - -import os -import unittest -from mock import call -from mock import MagicMock -from mock import patch -import perfzero.utils as utils -import tensorflow as tf # pylint: disable=g-bad-import-order - - -class TestUtils(unittest.TestCase, tf.test.Benchmark): - - def test_protobuf_read(self): - output_dir = '/tmp/' - os.environ['TEST_REPORT_FILE_PREFIX'] = output_dir - benchmark_result_file_path = os.path.join(output_dir, - 'TestUtils.testReportBenchmark') - if os.path.exists(benchmark_result_file_path): - os.remove(benchmark_result_file_path) - - self.report_benchmark( - iters=2000, - wall_time=1000, - name='testReportBenchmark', - metrics=[{'name': 'metric_name_1', 'value': 0, 'min_value': 1}, - {'name': 'metric_name_2', 'value': 90, 'min_value': 0, - 'max_value': 95}]) - - actual_result = utils.read_benchmark_result( - benchmark_result_file_path) - os.remove(benchmark_result_file_path) - - expected_result = { - 'name': 'TestUtils.testReportBenchmark', - # google.protobuf.json_format.MessageToDict() will convert - # int64 field to string. - 'iters': '2000', - 'wall_time': 1000, - 'cpu_time': 0, - 'throughput': 0, - 'extras': {}, - 'metrics': [ - { - 'name': 'metric_name_1', - 'value': 0, - 'min_value': 1 - }, - { - 'name': 'metric_name_2', - 'value': 90, - 'min_value': 0, - 'max_value': 95 - } - ] - } - - self.assertDictEqual(expected_result, actual_result) - - @patch('perfzero.utils.get_git_repo_info') - @patch('perfzero.utils.run_commands') - def test_checkout_git_repos(self, run_commands_mock, get_git_repo_info_mock): - git_repo_1 = {} - git_repo_1['url'] = 'url_1' - git_repo_1['local_path'] = 'local_path_1' - git_repo_1['dir_name'] = 'dir_name_1' - git_repo_1['branch'] = 'branch_1' - git_repo_1['git_hash'] = 'git_hash_1' - - git_repo_2 = {} - git_repo_2['url'] = 'url_2' - git_repo_2['local_path'] = 'local_path_2' - git_repo_2['dir_name'] = 'dir_name_2' - git_repo_2['branch'] = 'branch_2' - - git_repo_info_1 = {'url': 'url_1'} - git_repo_info_2 = {'url': 'url_2'} - get_git_repo_info_mock.side_effect = \ - lambda local_path: git_repo_info_1 if local_path == 'local_path_1' else git_repo_info_2 # pylint: disable=line-too-long - site_package_info = utils.checkout_git_repos([git_repo_1, git_repo_2], - False) - - self.assertEqual(2, len(site_package_info)) - self.assertEqual(git_repo_info_1, site_package_info['dir_name_1']) - self.assertEqual(git_repo_info_2, site_package_info['dir_name_2']) - - run_commands_mock.assert_has_calls(any_order=False, calls=[ - call(['git clone url_1 local_path_1']), - call(['git -C local_path_1 checkout branch_1']), - call(['git -C local_path_1 pull --rebase']), - call(['git -C local_path_1 reset --hard git_hash_1']), - call(['git clone url_2 local_path_2']), - call(['git -C local_path_2 checkout branch_2']) - ]) - - @patch('perfzero.utils.run_command') - def test_get_git_repo_info(self, run_command_mock): - run_command_mock.side_effect = [ - [0, 'git_url'], - [0, 'branch_name'], - [0, 'git_hash'] - ] - - git_repo_info = utils.get_git_repo_info('local_path_1') - self.assertEqual( - {'url': 'git_url', 'branch': 'branch_name', 'hash': 'git_hash'}, - git_repo_info) - run_command_mock.assert_has_calls(any_order=False, calls=[ - call('git -C local_path_1 config --get remote.origin.url'), - call('git -C local_path_1 rev-parse --abbrev-ref HEAD'), - call('git -C local_path_1 rev-parse HEAD') - ]) - - @patch('builtins.open') - @patch('perfzero.utils.make_dir_if_not_exist') - @patch('requests.get') - @patch('perfzero.utils.run_commands') - def test_download_data(self, run_commands_mock, requests_get_mock, - make_dir_mock, open_mock): # pylint: disable=unused-argument - get_mock = MagicMock() - get_mock.content = 'content' - requests_get_mock.return_value = get_mock - - download_info_1 = {'url': 'gs://remote_path_1/name_1', - 'local_path': 'local_path_1/modified_name_1'} - download_info_2 = {'url': 'http://remote_path_2/name_2', - 'local_path': 'local_path_2/modified_name_2'} - utils.download_data([download_info_1, download_info_2]) - - make_dir_mock.assert_has_calls(any_order=False, calls=[ - call('local_path_1'), - call('local_path_2') - ]) - requests_get_mock.assert_called_once_with('http://remote_path_2/name_2', - allow_redirects=True) - run_commands_mock.assert_has_calls(any_order=False, calls=[ - call([['gsutil', '-m', 'cp', '-r', '-n', - 'gs://remote_path_1/name_1', 'local_path_1']], - shell=False), - call(['mv local_path_1/name_1 local_path_1/modified_name_1']), - call(['mv local_path_2/name_2 local_path_2/modified_name_2']) - ]) - - def test_parse_data_downloads_str(self): - data_downloads_str = 'url_1;relative_path_1,url_2;relative_path_2' - download_infos = utils.parse_data_downloads_str('/root_data_dir', - data_downloads_str) - self.assertEqual(2, len(download_infos)) - self.assertEqual(download_infos[0], - {'url': 'url_1', - 'local_path': '/root_data_dir/relative_path_1'}) - self.assertEqual(download_infos[1], - {'url': 'url_2', - 'local_path': '/root_data_dir/relative_path_2'}) - - @patch('perfzero.utils.run_command') - def test_get_cpu_name(self, run_command_mock): - """Tests extract the cpu model name.""" - run_command_mock.return_value = [ - 0, 'model name : Intel(R) Xeon(R) CPU E5-1650 v2 @ 3.50GHz\n' - ] - cpu_name = utils.get_cpu_name() - self.assertEqual('Intel(R) Xeon(R) CPU E5-1650 v2 @ 3.50GHz', cpu_name) - - @patch('perfzero.utils.run_command') - def test_get_cpu_socket_count(self, run_command_mock): - """Tests get socket count.""" - run_command_mock.return_value = [0, '2\n'] - cpu_socket_count = utils.get_cpu_socket_count() - self.assertEqual(2, cpu_socket_count) - - @patch('perfzero.utils.run_command') - def test_get_gpu_model(self, run_command_mock): - # Tests get gpu info parses expected value into expected components. - run_command_mock.return_value = [ - 0, 'driver_version, name\n381.99, GTX 1080 \n' - ] - gpu_model = utils.get_gpu_info()['gpu_model'] - self.assertEqual('GTX 1080', gpu_model) - - # Tests gpu info returns second entry if first entry is a Quadro. - run_command_mock.return_value = [ - 0, 'blah\n200.99, Quadro K900 \n381.99, GTX 1080\n' - ] - gpu_model = utils.get_gpu_info()['gpu_model'] - self.assertEqual('GTX 1080', gpu_model) - - @patch('perfzero.utils.run_command') - def test_get_gpu_count(self, run_command_mock): - """Tests gpu info returns second entry if first entry is a Quadro.""" - run_command_mock.return_value = [ - 0, 'blah\n200.99, Quadro K900 \n381.99, GTX 1080\n' - ] - gpu_count = utils.get_gpu_info()['gpu_count'] - self.assertEqual(2, gpu_count) - - - - - - - - - - diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/setup.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/setup.py deleted file mode 100644 index 63406092e57d87c619362f2da37a1e183b7f9613..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/lib/setup.py +++ /dev/null @@ -1,197 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Checkout repository, download data and build docker image.""" -from __future__ import print_function - -import argparse -import json -import logging -import os -import shutil -import sys -import tempfile -import time - -import perfzero.device_utils as device_utils -import perfzero.perfzero_config as perfzero_config -import perfzero.utils as utils - - -def _temporary_file_name(parent_dir, base_name): - """Returns a temp name of the form //.""" - if not os.path.isdir(parent_dir): - os.makedirs(parent_dir) - temp_dir = tempfile.mkdtemp(dir=parent_dir) - return os.path.join(temp_dir, base_name) - - -def _load_docker_image(FLAGS, workspace_dir, setup_execution_time): - """Runs docker load --input_image . - - Fetches FLAGS.dockerfile_path to workspace_dir//local_docker first. - Runs docker load --input . - Deletes workspace_dir/ after the docker image is loaded. - - Args: - FLAGS: parser.parse_known_args object. - workspace_dir: String - The path to use for intermediate artifacts. - setup_execution_time: Map from string->double containing wall times for - different operations. This will have insertions describing the docker - setup time. - """ - load_docker_start_time = time.time() - local_docker_image_path = _temporary_file_name(workspace_dir, 'local_docker') - utils.download_data([{'url': FLAGS.dockerfile_path, - 'local_path': local_docker_image_path, - 'decompress': False}]) - - setup_execution_time['fetch_docker'] = time.time() - load_docker_start_time - - docker_load_cmd = 'docker load --input {}'.format(local_docker_image_path) - try: - utils.run_commands( - [docker_load_cmd, - 'docker images' # Print loaded image list. - ]) - setup_execution_time['load_docker'] = time.time() - load_docker_start_time - finally: - logging.info('removing parent dir of local docker image copy %s', - local_docker_image_path) - shutil.rmtree(os.path.dirname(local_docker_image_path)) - - -def _create_docker_image(FLAGS, project_dir, workspace_dir, - setup_execution_time): - """Creates a docker image. - - Args: - FLAGS: parser.parse_known_args object. - project_dir: String - The current project path. - workspace_dir: String - The path to use for intermediate artifacts. - setup_execution_time: Map from string->double containing wall times for - different operations. This will have insertions describing the docker - setup time. - """ - # Create docker image - docker_start_time = time.time() - docker_context = os.path.join(workspace_dir, 'resources') - # Necessary in case we don't have a local .whl file. - utils.create_empty_file(docker_context, 'EMPTY') - - # Download TensorFlow pip package from Google Cloud Storage and modify package - # path accordingly, if applicable - local_tensorflow_pip_spec = None - - if (FLAGS.tensorflow_pip_spec and - (FLAGS.tensorflow_pip_spec.startswith('gs://') or - FLAGS.tensorflow_pip_spec.startswith('file://'))): - local_pip_filename = os.path.basename(FLAGS.tensorflow_pip_spec) - local_pip_path = os.path.join(docker_context, local_pip_filename) - utils.download_data([{'url': FLAGS.tensorflow_pip_spec, - 'local_path': local_pip_path}]) - # Update path to pip wheel file for the Dockerfile. Note that this path has - # to be relative to the docker context (absolute path will not work). - FLAGS.tensorflow_pip_spec = local_pip_filename - local_tensorflow_pip_spec = local_pip_filename - else: - local_tensorflow_pip_spec = 'EMPTY' - - dockerfile_path = FLAGS.dockerfile_path - if not os.path.exists(dockerfile_path): - # Fall back to the deprecated approach if the user-specified - # dockerfile_path does not exist - dockerfile_path = os.path.join(project_dir, FLAGS.dockerfile_path) - extra_pip_specs = (FLAGS.extra_pip_specs or '').replace(';', '') - docker_base_cmd = 'docker build --no-cache --pull' - # FLAGS.extra_docker_build_args will be a list of strings (e.g. ['a', 'b=c']). - # We treat the strings directly as build-args: --build-arg a --build-arg b=c - # Empty strings are ignored. - extra_docker_build_args = ' '.join([ - '--build-arg %s' % arg for arg in FLAGS.extra_docker_build_args if arg]) - cmd = '{docker_base_cmd} -t {docker_tag}{tf_pip}{local_tf_pip}{extra_pip}{extra_docker_build_args} {suffix}'.format( - docker_base_cmd=docker_base_cmd, - docker_tag=FLAGS.docker_tag, - tf_pip=( - ' --build-arg tensorflow_pip_spec={}'.format( - FLAGS.tensorflow_pip_spec) if FLAGS.tensorflow_pip_spec else ''), - # local_tensorflow_pip_spec is either string 'EMPTY' or basename of - # local .whl file. - local_tf_pip=' --build-arg local_tensorflow_pip_spec={}'.format( - local_tensorflow_pip_spec), - extra_pip=' --build-arg extra_pip_specs=\'{}\''.format(extra_pip_specs), - extra_docker_build_args=' ' + extra_docker_build_args, - suffix=( - '-f {} {}'.format(dockerfile_path, docker_context) - if docker_context else '- < {}'.format(dockerfile_path)) - ) - - utils.run_commands([cmd]) - logging.info('Built docker image with tag %s', FLAGS.docker_tag) - setup_execution_time['build_docker'] = time.time() - docker_start_time - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - perfzero_config.add_setup_parser_arguments(parser) - FLAGS, unparsed = parser.parse_known_args() - - logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', - level=logging.DEBUG) - if unparsed: - logging.error('Arguments %s are not recognized', unparsed) - sys.exit(1) - - setup_execution_time = {} - project_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) - workspace_dir = os.path.join(project_dir, FLAGS.workspace) - site_package_dir = os.path.join(workspace_dir, 'site-packages') - utils.copy_and_rename_dirs(FLAGS.site_package_downloads, - site_package_dir) - - activate_gcloud = False - if FLAGS.dockerfile_path and FLAGS.dockerfile_path.startswith('gs://'): - # We might end up doing gsutil fetch later, so need to call - # active_gcloud_service(). - activate_gcloud = True - - if FLAGS.tensorflow_pip_spec and FLAGS.tensorflow_pip_spec.startswith('gs://'): - activate_gcloud = True - - # Download gcloud auth token. Remove this operation in the future when - # docker in Kokoro can accesss the GCP metadata server - start_time = time.time() - utils.active_gcloud_service(FLAGS.gcloud_key_file_url, - workspace_dir, download_only=not activate_gcloud) - setup_execution_time['download_token'] = time.time() - start_time - - # Set up the raid array. - start_time = time.time() - device_utils.create_drive_from_devices(FLAGS.root_data_dir, - FLAGS.gce_nvme_raid) - setup_execution_time['create_drive'] = time.time() - start_time - - if FLAGS.dockerfile_path: - if FLAGS.dockerfile_path.endswith('.tar.gz'): - logging.info('Assuming given file %s is a docker image to load', - FLAGS.dockerfile_path) - _load_docker_image(FLAGS, workspace_dir, - setup_execution_time) - else: - _create_docker_image(FLAGS, project_dir, workspace_dir, - setup_execution_time) - - logging.info('Setup time in seconds by operation:\n %s', - json.dumps(setup_execution_time, indent=2)) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/screenshots/profiling_overview.png b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/screenshots/profiling_overview.png deleted file mode 100644 index 8a21129232fddd840595e2d5bb9bbe5eb8d5c5d4..0000000000000000000000000000000000000000 Binary files a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/screenshots/profiling_overview.png and /dev/null differ diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/screenshots/profiling_trace_view.png b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/screenshots/profiling_trace_view.png deleted file mode 100644 index 78af206e13c1df31af360efe2832921da74788b2..0000000000000000000000000000000000000000 Binary files a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/screenshots/profiling_trace_view.png and /dev/null differ diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/scripts/create_big_table.txt b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/scripts/create_big_table.txt deleted file mode 100644 index 52b13b0290e3ee91d4a6ef1de6b2ab302f395365..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/scripts/create_big_table.txt +++ /dev/null @@ -1,43 +0,0 @@ -[ - { - "name": "execution_timestamp", - "type": "TIMESTAMP", - "mode": "REQUIRED" - }, - { - "name": "execution_id", - "type": "STRING", - "mode": "REQUIRED" - }, - - { - "name": "ml_framework_info", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "benchmark_result", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "benchmark_info", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "setup_info", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "system_info", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "process_info", - "type": "STRING", - "mode": "NULLABLE" - } -] diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/scripts/generate-readme-header.sh b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/scripts/generate-readme-header.sh deleted file mode 100644 index ab51a5fcb2d78804de4c22a2ea838a1dbc7a3a86..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/scripts/generate-readme-header.sh +++ /dev/null @@ -1,275 +0,0 @@ -#!/usr/bin/env bash - -# -# Steps: -# -# 1. Download corresponding html file for some README.md: -# curl -s $1 -# -# 2. Discard rows where no substring 'user-content-' (github's markup): -# awk '/user-content-/ { ... -# -# 3.1 Get last number in each row like ' ... sitemap.js.*<\/h/)+2, RLENGTH-5) -# -# 5. Find anchor and insert it inside "(...)": -# substr($0, match($0, "href=\"[^\"]+?\" ")+6, RLENGTH-8) -# - -gh_toc_version="0.6.0" - -gh_user_agent="gh-md-toc v$gh_toc_version" - -# -# Download rendered into html README.md by its url. -# -# -gh_toc_load() { - local gh_url=$1 - - if type curl &>/dev/null; then - curl --user-agent "$gh_user_agent" -s "$gh_url" - elif type wget &>/dev/null; then - wget --user-agent="$gh_user_agent" -qO- "$gh_url" - else - echo "Please, install 'curl' or 'wget' and try again." - exit 1 - fi -} - -# -# Converts local md file into html by GitHub -# -# ➥ curl -X POST --data '{"text": "Hello world github/linguist#1 **cool**, and #1!"}' https://api.github.com/markdown -#

Hello world github/linguist#1 cool, and #1!

'" -gh_toc_md2html() { - local gh_file_md=$1 - URL=https://api.github.com/markdown/raw - if [ -z "$GH_TOC_TOKEN" ]; then - TOKEN=$GH_TOC_TOKEN - else - TOKEN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/token.txt" - fi - if [ -f "$TOKEN" ]; then - URL="$URL?access_token=$(cat $TOKEN)" - fi - # echo $URL 1>&2 - OUTPUT="$(curl -s --user-agent "$gh_user_agent" \ - --data-binary @"$gh_file_md" -H "Content-Type:text/plain" \ - $URL)" - - if [ "$?" != "0" ]; then - echo "XXNetworkErrorXX" - fi - if [ "$(echo "${OUTPUT}" | awk '/API rate limit exceeded/')" != "" ]; then - echo "XXRateLimitXX" - else - echo "${OUTPUT}" - fi -} - - -# -# Is passed string url -# -gh_is_url() { - case $1 in - https* | http*) - echo "yes";; - *) - echo "no";; - esac -} - -# -# TOC generator -# -gh_toc(){ - local gh_src=$1 - local gh_src_copy=$1 - local gh_ttl_docs=$2 - local need_replace=$3 - - if [ "$gh_src" = "" ]; then - echo "Please, enter URL or local path for a README.md" - exit 1 - fi - - - # Show "TOC" string only if working with one document - if [ "$gh_ttl_docs" = "1" ]; then - - echo "Table of Contents" - echo "=================" - echo "" - gh_src_copy="" - - fi - - if [ "$(gh_is_url "$gh_src")" == "yes" ]; then - gh_toc_load "$gh_src" | gh_toc_grab "$gh_src_copy" - if [ "${PIPESTATUS[0]}" != "0" ]; then - echo "Could not load remote document." - echo "Please check your url or network connectivity" - exit 1 - fi - if [ "$need_replace" = "yes" ]; then - echo - echo "!! '$gh_src' is not a local file" - echo "!! Can't insert the TOC into it." - echo - fi - else - local rawhtml=$(gh_toc_md2html "$gh_src") - if [ "$rawhtml" == "XXNetworkErrorXX" ]; then - echo "Parsing local markdown file requires access to github API" - echo "Please make sure curl is installed and check your network connectivity" - exit 1 - fi - if [ "$rawhtml" == "XXRateLimitXX" ]; then - echo "Parsing local markdown file requires access to github API" - echo "Error: You exceeded the hourly limit. See: https://developer.github.com/v3/#rate-limiting" - TOKEN="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/token.txt" - echo "or place github auth token here: $TOKEN" - exit 1 - fi - local toc=`echo "$rawhtml" | gh_toc_grab "$gh_src_copy"` - echo "$toc" - if [ "$need_replace" = "yes" ]; then - local ts="<\!--ts-->" - local te="<\!--te-->" - local dt=`date +'%F_%H%M%S'` - local ext=".orig.${dt}" - local toc_path="${gh_src}.toc.${dt}" - local toc_footer="" - # http://fahdshariff.blogspot.ru/2012/12/sed-mutli-line-replacement-between-two.html - # clear old TOC - sed -i${ext} "/${ts}/,/${te}/{//!d;}" "$gh_src" - # create toc file - echo "${toc}" > "${toc_path}" - echo -e "\n${toc_footer}\n" >> "$toc_path" - # insert toc file - if [[ "`uname`" == "Darwin" ]]; then - sed -i "" "/${ts}/r ${toc_path}" "$gh_src" - else - sed -i "/${ts}/r ${toc_path}" "$gh_src" - fi - echo - echo "!! TOC was added into: '$gh_src'" - echo "!! Origin version of the file: '${gh_src}${ext}'" - echo "!! TOC added into a separate file: '${toc_path}'" - echo - fi - fi -} - -# -# Grabber of the TOC from rendered html -# -# $1 — a source url of document. -# It's need if TOC is generated for multiple documents. -# -gh_toc_grab() { - # if closed is on the new line, then move it on the prev line - # for example: - # was: The command foo1 - # - # became: The command foo1 - sed -e ':a' -e 'N' -e '$!ba' -e 's/\n<\/h/<\/h/g' | - # find strings that corresponds to template - grep -E -o '//g' | sed 's/<\/code>//g' | - # now all rows are like: - # ... .*<\/h/)+2, RLENGTH-5) - href = substr($0, match($0, "href=\"[^\"]+?\"")+6, RLENGTH-7) - print sprintf("%*s", level*3, " ") "* [" text "](" gh_url href ")" }' | - sed 'y/+/ /; s/%/\\x/g')" -} - -# -# Returns filename only from full path or url -# -gh_toc_get_filename() { - echo "${1##*/}" -} - -# -# Options hendlers -# -gh_toc_app() { - local app_name=$(basename $0) - local need_replace="no" - - if [ "$1" = '--help' ] || [ $# -eq 0 ] ; then - echo "GitHub TOC generator ($app_name): $gh_toc_version" - echo "" - echo "Usage:" - echo " $app_name [--insert] src [src] Create TOC for a README file (url or local path)" - echo " $app_name - Create TOC for markdown from STDIN" - echo " $app_name --help Show help" - echo " $app_name --version Show version" - return - fi - - if [ "$1" = '--version' ]; then - echo "$gh_toc_version" - echo - echo "os: `lsb_release -d | cut -f 2`" - echo "kernel: `cat /proc/version`" - echo "shell: `$SHELL --version`" - echo - for tool in curl wget grep awk sed; do - printf "%-5s: " $tool - echo `$tool --version | head -n 1` - done - return - fi - - if [ "$1" = "-" ]; then - if [ -z "$TMPDIR" ]; then - TMPDIR="/tmp" - elif [ -n "$TMPDIR" -a ! -d "$TMPDIR" ]; then - mkdir -p "$TMPDIR" - fi - local gh_tmp_md - gh_tmp_md=$(mktemp $TMPDIR/tmp.XXXXXX) - while read input; do - echo "$input" >> "$gh_tmp_md" - done - gh_toc_md2html "$gh_tmp_md" | gh_toc_grab "" - return - fi - - if [ "$1" = '--insert' ]; then - need_replace="yes" - shift - fi - - for md in "$@" - do - echo "" - gh_toc "$md" "$#" "$need_replace" - done - - echo "" - echo "Created by [gh-md-toc](https://github.com/ekalinin/github-markdown-toc)" -} - -# -# Entry point -# -gh_toc_app "$@" diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/scripts/plot_process_info.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/scripts/plot_process_info.py deleted file mode 100644 index 1d62a2b7230318800bf555b7072010f972019c12..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/perfzero/scripts/plot_process_info.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Plot graph showing process metric values over time""" - -from __future__ import print_function - -import argparse -import sys -import json -import matplotlib.pyplot as plt -import matplotlib.backends.backend_pdf as backend_pdf -import matplotlib.ticker as tick - -colors=['b', 'r', 'g', 'c', 'pink'] - -def visualize(file_path): - - entries = [] - with open(file_path) as f: - entries = [json.loads(line) for line in f.readlines() if line.strip()] - - if not entries: - print('There is no data in file {}'.format(file_path)) - return - - pdf = backend_pdf.PdfPages("process_info.pdf") - idx = 0 - names = [name for name in entries[0].keys() if name != 'time'] - times = [entry['time'] for entry in entries] - - for name in names: - values = [entry[name] for entry in entries] - fig = plt.figure() - ax = plt.gca() - ax.yaxis.set_major_formatter(tick.ScalarFormatter(useMathText=True)) - plt.ticklabel_format(style='sci', axis='y', scilimits=(-2,3)) - plt.plot(times, values, colors[idx % len(colors)], marker='x', label=name) - plt.xlabel('Time (sec)') - plt.ylabel(name) - plt.ylim(ymin=0) - plt.legend(loc = 'upper left') - pdf.savefig(fig) - idx += 1 - - plt.show() - pdf.close() - print('Generated process_info.pdf from {}'.format(file_path)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(usage='plot_process_info.py ' ) - parser.add_argument('file_path', type=str) - flags = parser.parse_args(sys.argv[1:]) - - - visualize(flags.file_path) - - - diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/README.md b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/README.md deleted file mode 100644 index e7b746487bcf0daad38d4522580a170ac58523f2..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/README.md +++ /dev/null @@ -1,88 +0,0 @@ -# tf_cnn_benchmarks: High performance benchmarks - -**Note: tf_cnn_benchmarks is no longer maintained.** - -tf_cnn_benchmarks contains TensorFlow 1 implementations of several popular -convolutional models, and is designed to be as fast as possible. -tf_cnn_benchmarks supports both running on a single machine or running in -distributed mode across multiple hosts. - -tf_cnn_benchmarks is no longer maintained. Although it will run with TensorFlow -2, it was written and optimized for TensorFlow 1, and has not been maintained -since TensorFlow 2 was released. For clean and easy-to-read TensorFlow 2 models, -please see the [TensorFlow Official -Models](https://github.com/tensorflow/models/tree/master/official). - -## Getting Started - -To run ResNet50 with synthetic data without distortions with a single GPU, run - -``` -python tf_cnn_benchmarks.py --num_gpus=1 --batch_size=32 --model=resnet50 --variable_update=parameter_server -``` - -Note that the master branch of tf_cnn_benchmarks occasionally requires the -latest nightly version of TensorFlow. You can install the nightly version by -running `pip install tf-nightly-gpu` in a clean environment, or by installing -TensorFlow from source. We sometimes will create a branch of tf_cnn_benchmarks, -in the form of cnn_tf_vX.Y_compatible, that is compatible with TensorFlow -version X.Y. For example, branch -[cnn_tf_v1.9_compatible](https://github.com/tensorflow/benchmarks/tree/cnn_tf_v1.9_compatible/scripts/tf_cnn_benchmarks) -works with TensorFlow 1.9. However, as tf_cnn_benchmarks is no longer -maintained, we will likely no longer create new branches. - -Some important flags are - -* model: Model to use, e.g. resnet50, inception3, vgg16, and alexnet. -* num_gpus: Number of GPUs to use. -* data_dir: Path to data to process. If not set, synthetic data is used. To - use Imagenet data use these - [instructions](https://github.com/tensorflow/models/tree/master/research/inception#getting-started) - as a starting point. -* batch_size: Batch size for each GPU. -* variable_update: The method for managing variables: parameter_server - ,replicated, distributed_replicated, independent -* local_parameter_device: Device to use as parameter server: cpu or gpu. - -To see the full list of flags, run `python tf_cnn_benchmarks.py --help`. - -To run ResNet50 with real data with 8 GPUs, run: - -``` -python tf_cnn_benchmarks.py --data_format=NCHW --batch_size=256 \ ---model=resnet50 --optimizer=momentum --variable_update=replicated \ ---nodistortions --gradient_repacking=8 --num_gpus=8 \ ---num_epochs=90 --weight_decay=1e-4 --data_dir=${DATA_DIR} --use_fp16 \ ---train_dir=${CKPT_DIR} -``` -This will train a ResNet-50 model on ImageNet with 2048 batch size on 8 -GPUs. The model should train to around 76% accuracy. - -## Running the tests - -To run the tests, run - -```bash -pip install portpicker -python run_tests.py && python run_tests.py --run_distributed_tests -``` - -Note the tests require portpicker. - -The command above runs a subset of tests that is both fast and fairly -comprehensive. Alternatively, all the tests can be run, but this will take a -long time: - -```bash -python run_tests.py --full_tests && python run_tests.py --full_tests --run_distributed_tests -``` - -We will run all tests on every PR before merging them, so it is not necessary -to pass `--full_tests` when running tests yourself. - -To run an individual test, such as method `testParameterServer` of test class -`TfCnnBenchmarksTest` of module `benchmark_cnn_test`, run - -```bash -python -m unittest -v benchmark_cnn_test.TfCnnBenchmarksTest.testParameterServer -``` diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/all_reduce_benchmark.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/all_reduce_benchmark.py deleted file mode 100644 index b61838d781890455d56365ed67bde57b853e14fb..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/all_reduce_benchmark.py +++ /dev/null @@ -1,290 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Benchmarks the all-reduce algorithms of tf_cnn_benchmarks. - -tf_cnn_benchmarks uses all-reduce to aggregate gradients. This benchmark is -useful for benchmarking the performance of just this gradient aggregation, -instead of the entire model. All the flags that tf_cnn_benchmarks accepts are -also accepted by this script, although many are silently ignored. - -The number and shapes of the tensors all-reduced are those of the variables of -the model specified by the --model flag. -TODO(reedwm): Allow custom sizes to be specified. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -import os -import time - -from absl import app -from absl import flags as absl_flags -import tensorflow.compat.v1 as tf - -from tensorflow.python.ops import control_flow_ops -import benchmark_cnn -import cnn_util -import flags -from cnn_util import log_fn - - -absl_flags.DEFINE_integer('iters_per_step', 5, - 'Number of iterations to run all-reduce for, per ' - 'step. Every step, a session will be run on a Graph ' - 'that contains this many copies of the all-reduce. ' - 'The copies are run sequentially. Setting this above ' - '1 is useful to lower the overhead of starting the ' - 'session run, running the VariableV2 ops at the ' - 'start of the step, etc.') - - -flags.define_flags() -for name in flags.param_specs.keys(): - absl_flags.declare_key_flag(name) - - -def get_var_shapes(model): - """Returns the list of variable shapes for a tf_cnn_benchmarks Model.""" - with tf.Graph().as_default(): - # The variable shapes do not depend on the batch size. - images = tf.placeholder(tf.float32, model.get_input_shapes('train')[0]) - model.build_network([images]) - return [[int(d) for d in v.shape.dims] for v in tf.trainable_variables()] - - -def all_reduce(all_device_tensors, variable_mgr): - """Performs a single batch all-reduce. - - Args: - all_device_tensors: List of lists of tensors. all_device_tensors[t][i] is - a tensor, where t is the tower the tensor is on and i is the index of - the tensor. - variable_mgr: The VariableMgr to perform the all-reduce. - Returns: - List of list of tensors in the same form as `all_device_tensors`, except the - tensors are aggregated across towers. - """ - tower_grads = [[(g, None) for g in device_tensors] for - device_tensors in all_device_tensors] - _, aggregated_tower_grads = variable_mgr.preprocess_device_grads(tower_grads) - return [ - [g for g, _ in agg_device_tensors] - for agg_device_tensors in aggregated_tower_grads] - - -def build_all_reduce_iterations(all_device_tensors, tower_devices, variable_mgr, - num_iters): - """Builds the all-reduce ops for multiple iterations to aggregate tensors. - - The tensors in `all_device_tensors` are aggregated `num_iters` times. Each - iteration aggregates the results from the previous iteration. The iterations - are run sequentially, so the aggregations for an iteration do not start - running until the previous iteration has completed. Each iteration after the - first is aggregating already-aggregated values, but it does not matter because - we are only aggregating for benchmarking purposes. - - Args: - all_device_tensors: List of lists of tensors. all_device_tensors[t][i] is - a tensor, where t is the tower the tensor is on and i is the index of - the tensor. - tower_devices: A list of device strings. tower_devices[t] is the device - of the tensors in all_device_tensors[t]. - variable_mgr: The VariableMgr to perform the all-reduce. - num_iters: Number of iterations to aggregate tensors for. - Returns: - An op that when run, causes the all-reduce ops to run. - """ - for i in range(num_iters): - with tf.name_scope('iteration_%d' % i): - # Step 1: Do the aggregation. - with tf.name_scope('tensor_aggregation'): - all_device_tensors = all_reduce(all_device_tensors, variable_mgr) - - # Step 2. Create identity ops, to bring the aggregated results back to - # each device. - new_all_device_tensors = [] - for device, device_tensors in zip(tower_devices, all_device_tensors): - with tf.device(device): - new_all_device_tensors.append([ - tf.identity(t, name='identity_after_allreduce') - for t in device_tensors - ]) - all_device_tensors = new_all_device_tensors - - # Step 3. Add control dependencies to delay the next iteration until this - # iteration is complete. To avoid extra overhead, we do not have any - # cross-device control dependencies, which means it's possible for two - # iterations to slightly overlap. - new_all_device_tensors = [] - for device_tensors in all_device_tensors: - new_all_device_tensors.append([ - control_flow_ops.with_dependencies( - device_tensors, t, name='identity_after_dependencies') - for t in device_tensors - ]) - all_device_tensors = new_all_device_tensors - - # To prevent the dependency optimizer from removing every op we created, - # we store the results in variables. - ops_to_run = [] - for device, device_tensors in zip(tower_devices, all_device_tensors): - with tf.device(device): - for t in device_tensors: - # The placeholder initial value is never run. - var = tf.Variable(tf.placeholder(tf.float32, t.shape), collections=[]) - ops_to_run.append(var.assign(t)) - return tf.group(*ops_to_run) - - -def build_graph(tower_devices, tensor_shapes, variable_mgr, num_iters): - """Builds the graph for the benchmark. - - Args: - tower_devices: A list of device strings of the devices to run the all-reduce - benchmark on. - tensor_shapes: A list of shapes of the tensors that will be aggregated for - the all-reduce. - variable_mgr: The VariableMgr to perform the all-reduce. - num_iters: Number of iterations to aggregate tensors for. - Returns: - An op that runs the benchmark. - """ - all_device_tensors = [] - for i, tower_device in enumerate(tower_devices): - with tf.device(tower_device): - device_tensors = [] - for j, shape in enumerate(tensor_shapes): - tensor = tf.Variable(tf.random_normal(shape, dtype=tf.float32), - name='tensor_%d_on_device_%d' % (j, i)) - device_tensors.append(tensor) - all_device_tensors.append(device_tensors) - - log_fn('Building all-reduce ops') - benchmark_op = build_all_reduce_iterations(all_device_tensors, tower_devices, - variable_mgr, num_iters) - log_fn('Done building all-reduce ops') - return benchmark_op - - -def run_graph(benchmark_op, bench_cnn, init_ops, dummy_loss_op): - """Runs the graph for the benchmark. - - Args: - benchmark_op: An op that runs the benchmark. - bench_cnn: The BenchmarkCNN where params and other attributes are obtained. - init_ops: A list of ops that are run before `benchmark_op` for - initialization. - dummy_loss_op: Any op. We must pass a loss op to - `benchmark_cnn.benchmark_one_step`, but the result of the op is never - actually used. - """ - config = benchmark_cnn.create_config_proto(bench_cnn.params) - with tf.Session(config=config) as sess: - for op in init_ops: - sess.run(op) - step_train_times = [] - fetches = {'average_loss': dummy_loss_op, 'benchmark_op': benchmark_op} - log_fn('Running warmup') - for i in range(-bench_cnn.num_warmup_batches, bench_cnn.num_batches): - if i == 0: - log_fn('Running all-reduce ops') - start = time.perf_counter() - if i > 0 and i % bench_cnn.params.display_every == 0: - log_fn('Iteration: %d. Average time per step so far: %s' % - (i, (time.perf_counter() - start) / i)) - # Call benchmark_one_step instead of directly calling sess.run(...), to - # potentially get a trace file, partitioned graphs, etc. - benchmark_cnn.benchmark_one_step( - sess=sess, - fetches=fetches, - step=i, - # The batch size is only used for the images/sec calculation, which is - # not actually calculated because we pass show_images_per_sec=False. - batch_size=None, - step_train_times=step_train_times, - trace_filename=bench_cnn.trace_filename, - partitioned_graph_file_prefix=( - bench_cnn.params.partitioned_graph_file_prefix), - profiler=None, - image_producer=None, - params=bench_cnn.params, - show_images_per_sec=False) - log_fn('Average time per step: %s' % - ((time.perf_counter() - start) / bench_cnn.num_batches)) - - -def run_benchmark(bench_cnn, num_iters): - """Runs the all-reduce benchmark. - - Args: - bench_cnn: The BenchmarkCNN where params, the variable manager, and other - attributes are obtained. - num_iters: Number of iterations to do all-reduce for for. - - Raises: - ValueError: Invalid params of bench_cnn. - """ - if bench_cnn.params.variable_update != 'replicated': - raise ValueError('--variable_update=replicated must be specified to use' - 'the all-reduce benchmark') - if bench_cnn.params.variable_consistency == 'relaxed': - raise ValueError('--variable_consistency=relaxed is not supported') - - benchmark_op = build_graph(bench_cnn.raw_devices, - get_var_shapes(bench_cnn.model), - bench_cnn.variable_mgr, num_iters) - init_ops = [ - tf.global_variables_initializer(), - bench_cnn.variable_mgr.get_post_init_ops() - ] - loss_op = tf.no_op() - - if bench_cnn.graph_file: - path, filename = os.path.split(bench_cnn.graph_file) - as_text = filename.endswith('txt') - log_fn('Writing GraphDef as %s to %s' % ( - 'text' if as_text else 'binary', bench_cnn.graph_file)) - tf.train.write_graph(tf.get_default_graph().as_graph_def(add_shapes=True), - path, filename, as_text) - - run_graph(benchmark_op, bench_cnn, init_ops, loss_op) - - -# TODO(reedwm): Reduce redundancy with tf_cnn_benchmarks -def main(positional_arguments): - # Command-line arguments like '--distortions False' are equivalent to - # '--distortions=True False', where False is a positional argument. To prevent - # this from silently running with distortions, we do not allow positional - # arguments. - assert len(positional_arguments) >= 1 - if len(positional_arguments) > 1: - raise ValueError('Received unknown positional arguments: %s' - % positional_arguments[1:]) - - params = benchmark_cnn.make_params_from_flags() - params = benchmark_cnn.setup(params) - bench = benchmark_cnn.BenchmarkCNN(params) - - tfversion = cnn_util.tensorflow_version_tuple() - log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) - - run_benchmark(bench, absl_flags.FLAGS.iters_per_step) - -if __name__ == '__main__': - tf.disable_v2_behavior() - app.run(main) # Raises error on invalid flags, unlike tf.app.run() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/all_reduce_benchmark_test.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/all_reduce_benchmark_test.py deleted file mode 100644 index c8efd53f421049e697a4eeea7486a758c5a52a6c..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/all_reduce_benchmark_test.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for all_reduce_benchmark.py.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow.compat.v1 as tf - -import all_reduce_benchmark -import benchmark_cnn -import test_util - - -class AllReduceBenchmarkTest(tf.test.TestCase): - """Tests the all-reduce benchmark.""" - - def _test_run_benchmark(self, params): - """Tests that run_benchmark() runs successfully with the params.""" - logs = [] - with test_util.monkey_patch(all_reduce_benchmark, - log_fn=test_util.print_and_add_to_list(logs)): - bench_cnn = benchmark_cnn.BenchmarkCNN(params) - all_reduce_benchmark.run_benchmark(bench_cnn, num_iters=5) - self.assertRegex(logs[-1], '^Average time per step: [0-9.]+$') - - def test_run_benchmark(self): - """Tests that run_benchmark() runs successfully.""" - params = benchmark_cnn.make_params(num_batches=10, - variable_update='replicated', - num_gpus=2) - self._test_run_benchmark(params) - params = params._replace(hierarchical_copy=True, gradient_repacking=8, - num_gpus=8) - self._test_run_benchmark(params) - -if __name__ == '__main__': - tf.disable_v2_behavior() - tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/allreduce.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/allreduce.py deleted file mode 100644 index 1056032869e586c7296d3f0f0893c284c8a09e05..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/allreduce.py +++ /dev/null @@ -1,649 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities for allreduce.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections as pycoll -import re - -from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow.compat.v1 as tf - -# pylint: disable=g-direct-tensorflow-import,g-import-not-at-top -try: - from tensorflow.python.distribute.v1 import all_reduce -except ImportError: - # Compatibility with TF 2.4 and below - from tensorflow.python.distribute import all_reduce -from tensorflow.python.framework import device as pydev -from tensorflow.python.framework import ops -from tensorflow.python.ops import collective_ops - -AllReduceSpecTuple = pycoll.namedtuple('AllReduceSpecTuple', 'alg shards limit') - - -def parse_general_int(s): - """Parse integer with power-of-2 suffix eg. 32k.""" - mo = re.match(r'(\d+)([KkMGT]?)$', s) - if mo: - i, suffix = mo.group(1, 2) - v = int(i) - if suffix: - if suffix == 'K' or suffix == 'k': - v *= 1024 - elif suffix == 'M': - v *= (1024 * 1024) - elif suffix == 'G': - v *= (1024 * 1024 * 1024) - elif suffix == 'T': - v *= (1024 * 1024 * 1024 * 1024) - else: - raise ValueError('invalid integer string %s' % s) - return v - else: - v = int(s) - return v - - -def parse_all_reduce_spec(all_reduce_spec): - """Parse all_reduce_spec. - - Args: - all_reduce_spec: a string specifying a combination of all-reduce - algorithms to apply for gradient reduction. - - Returns: - a list of AllReduceSpecTuple. - - Raises: - ValueError: all_reduce_spec is not well-formed. - - An all_reduce_spec has BNF form: - int ::= positive whole number - g_int ::= int[KkMGT]? - alg_spec ::= alg | alg#int - range_spec ::= alg_spec | alg_spec/alg_spec - spec ::= range_spec | range_spec:g_int:range_spec - - Not all syntactically correct specifications are supported. - Examples of supported all_reduce_spec strings, with semantics explained: - - 'collective' == apply tf.collective_reduce operator to all tensors. - 'collective#2' == apply tf.collective_reduce operator to all tensors, - requesting up to 2 simultaneous transfers at each node, if - feasible, by subdividing tensor by an additional factor of 2. - 'xring' == apply ring all-reduce to all tensors - 'xring#2' == apply ring all-reduce to all tensors, using two simultaneous - transfer rings, each operating on 1/2 of each tensor. - 'nccl' == apply NCCL all-reduce to all tensors (only works within - a single worker process where all devices are GPUs) - 'nccl/xring' == apply NCCL all-reduce to all tensors within each worker - to produce at least one full-reduced (locally) value, - then apply ring all-reduce to one such value from each - worker, then apply NCCL broadcast to propagate those globally - reduced values back to every device within each worker. - 'pscpu' == Shuffle reduce using worker CPUs as the gather devices: each - distributed tensor is reduced by copying all instances to - one of the worker CPUs, computing the reduction there, then - copying back to each participating device. Tensor reductions - are assigned to specific CPUs round-robin. - 'psgpu#4' == Arrange all GPUs across all workers into groups of 4. - Each distributed tensor is shuffle reduced against one - such group of 4 GPUs, selected round-robin. That is, each - tensor is split across 4 shards for the reduction. - 'pscpu:2k:pscpu#2:64k:xring' == Apply single-shard pscpu to - tensors of size <= 2048 elements, apply 2-shard pscpu to - tensors up to size 64k elements, apply xring to larger tensors. - 'pscpu/pscpu#2' == Use shuffle gather to locally reduce each tensor on - the worker's CPU, then use 2-shard shuffle to reduce those - locally reduced tensors across workers (on the worker CPUs), then - scatter the globally reduced values locally from each worker CPU. - """ - range_parts = all_reduce_spec.split(':') + ['-1'] - if len(range_parts) % 2: - raise ValueError('all_reduce_spec not well formed: %s' % all_reduce_spec) - limit = 0 - spec = [] - alg = None - shards = 1 - for i, range_part in enumerate(range_parts): - if i % 2 == 1: - try: - limit = parse_general_int(range_part) - spec.append(AllReduceSpecTuple(alg=alg, shards=shards, limit=limit)) - except ValueError: - raise ValueError('all_reduce_spec (%s) contains non-integer range %s' % - (all_reduce_spec, range_part)) - else: - alg = range_part - alg_parts = range_part.split('#') - alg = alg_parts[0] - if len(alg_parts) > 1: - try: - shards = int(alg_parts[1]) - except ValueError: - raise ValueError('all_reduce_spec (%s) contains non-integer ' - 'shards %s' % all_reduce_spec, alg_parts[1]) - else: - shards = 1 - if alg not in [ - 'nccl', 'nccl/xring', 'nccl/rechd', 'nccl/pscpu', 'xring', 'pscpu', - 'psgpu', 'pscpu/pscpu', 'collective' - ]: - raise ValueError('all_reduce_spec (%s) contains invalid alg %s' % - (all_reduce_spec, alg)) - return spec - - -def build_all_reduce_device_prefixes(job_name, num_tasks): - """Build list of device prefix names for all_reduce. - - Args: - job_name: 'worker', 'ps' or 'localhost'. - num_tasks: number of jobs across which device names should be generated. - - Returns: - A list of device name prefix strings. Each element spells out the full - host name without adding the device. - e.g. '/job:worker/task:0' - """ - if job_name != 'localhost': - return ['/job:%s/task:%d' % (job_name, d) for d in range(0, num_tasks)] - else: - assert num_tasks == 1 - return ['/job:%s' % job_name] - - -def group_device_names(devices, group_size): - """Group device names into groups of group_size. - - Args: - devices: list of strings naming devices. - group_size: int >= 1 - - Returns: - list of lists of devices, where each inner list is group_size long, - and each device appears at least once in an inner list. If - len(devices) % group_size = 0 then each device will appear - exactly once. - - Raises: - ValueError: group_size > len(devices) - """ - num_devices = len(devices) - if group_size > num_devices: - raise ValueError('only %d devices, but group_size=%d' % (num_devices, - group_size)) - num_groups = ( - num_devices // group_size + (1 if (num_devices % group_size != 0) else 0)) - groups = [[] for i in range(num_groups)] - for i in range(0, num_groups * group_size): - groups[i % num_groups].append(devices[i % num_devices]) - return groups - - -def split_grads_by_size(threshold_size, device_grads): - """Break gradients into two sets according to tensor size. - - Args: - threshold_size: int size cutoff for small vs large tensor. - device_grads: List of lists of (gradient, variable) tuples. The outer - list is over devices. The inner list is over individual gradients. - - Returns: - small_grads: Subset of device_grads where shape is <= theshold_size - elements. - large_grads: Subset of device_grads where shape is > threshold_size - elements. - """ - small_grads = [] - large_grads = [] - for dl in device_grads: - small_dl = [] - large_dl = [] - for (g, v) in dl: - tensor_size = g.get_shape().num_elements() - if tensor_size <= threshold_size: - small_dl.append([g, v]) - else: - large_dl.append([g, v]) - if small_dl: - small_grads.append(small_dl) - if large_dl: - large_grads.append(large_dl) - return small_grads, large_grads - - -_instance_key = 1 - - -def new_collective_instance_key(): - """Returns a new instance key for use in defining a collective op.""" - global _instance_key - v = _instance_key - _instance_key += 1 - return v - - -_group_key = 1 -_group_key_table = dict() - - -def collective_group_key(devices): - """Returns a group key for the set of devices. - - Args: - devices: list of strings naming devices in a collective group. - - Returns: - int key uniquely identifying the set of device names. - """ - global _group_key - global _group_key_table - parsed = [pydev.DeviceSpec.from_string(d) for d in devices] - names = sorted(['%s:%d' % (d.device_type, d.device_index) for d in parsed]) - concat = ','.join(names) - if concat not in _group_key_table.keys(): - new_key = _group_key - _group_key += 1 - _group_key_table[concat] = new_key - rv = _group_key_table[concat] - return rv - - -def build_collective_reduce(input_tensors, num_workers, num_shards, - red_op='Add', un_op='Id'): - """Build a subgraph that does one full all-reduce, using the collective Op. - - Args: - input_tensors: tensors within a single worker graph that are to be reduced - together; must be one per device. - num_workers: total number of workers with identical independent graphs that - will be doing this same reduction. The reduction will actually include - the corresponding tensors at all these workers. - num_shards: number of shards into which to divide each per-tick chunk, - normally 1 but could be higher on multi-data-path architectures. - red_op: string naming the reduction op - un_op: string naming the unary final op - - Returns: - An array of final tensors, one per device, computed by the full reduction. - - Raises: - ValueError: There must be at least two tensors over all the workers. - """ - group_size = len(input_tensors) * num_workers - if group_size < 2: - raise ValueError('num_workers * len(input_tensors) must be 2 or greater') - devices = [t.device for t in input_tensors] - num_devices = len(devices) - group_key = collective_group_key(devices) - instance_key = new_collective_instance_key() - out_tensors = [] - if num_shards == 1: - subdiv_offsets = [0] - elif num_shards == 2: - if num_devices > 1: - subdiv_offsets = [0, -(num_devices // 2)] - else: - subdiv_offsets = [0] - else: - raise ValueError('Unsupported num_shards %d' % num_shards) - for d in range(num_devices): - with ops.device(devices[d]): - reduce_op = collective_ops.all_reduce(input_tensors[d], - group_size, group_key, instance_key, - red_op, un_op, - subdiv_offsets) - out_tensors.append(reduce_op) - return out_tensors - - -def broadcast_send(t, shape, dtype, group_size, group_key, instance_key): - return collective_ops.broadcast_send(t, shape, dtype, group_size, group_key, - instance_key) - - -def broadcast_recv(shape, dtype, group_size, group_key, instance_key): - return collective_ops.broadcast_recv(shape, dtype, group_size, group_key, - instance_key) - - -def sum_grad_and_var_all_reduce(single_session, - grad_and_vars, - num_workers, - alg, - gpu_indices, - aux_devices=None, - num_shards=1): - """Apply all-reduce algorithm over specified gradient tensors.""" - scaled_grads = [g for g, _ in grad_and_vars] - if alg == 'collective': - assert not single_session - summed_grads = build_collective_reduce( - scaled_grads, num_workers, num_shards, 'Add', 'Id') - else: - with tf.name_scope('allreduce'): - # Note that each grad_and_vars looks like the following: - # ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN)) - if alg == 'nccl': - summed_grads = all_reduce.build_nccl_all_reduce(scaled_grads, tf.add) - elif alg == 'xring': - summed_grads = all_reduce.build_ring_all_reduce( - scaled_grads, num_workers, num_shards, gpu_indices, tf.add) - elif alg == 'nccl/xring': - summed_grads = all_reduce.build_nccl_then_ring(scaled_grads, num_shards, - tf.add) - elif alg == 'nccl/rechd': - summed_grads = all_reduce.build_nccl_then_recursive_hd( - scaled_grads, tf.add) - elif alg == 'nccl/pscpu': - summed_grads = all_reduce.build_nccl_then_shuffle( - scaled_grads, aux_devices, tf.add, tf.add_n) - elif alg == 'pscpu/pscpu': - summed_grads = all_reduce.build_shuffle_then_shuffle( - scaled_grads, - aux_devices, - # TODO(tucker): devise a way of better specifying the device set - # for the second level. - [aux_devices[0]], - tf.add_n) - elif alg in ['pscpu', 'psgpu']: - summed_grads = all_reduce.build_shuffle_all_reduce( - scaled_grads, aux_devices, tf.add_n) - else: - raise ValueError('unsupported all_reduce alg: ', alg) - - result = [] - for (_, v), g in zip(grad_and_vars, summed_grads): - result.append([g, v]) - return result - - -def contains_any(haystack, needles): - """Tests if any needle is a substring of haystack. - - Args: - haystack: a string - needles: list of strings - - Returns: - True if any element of needles is a substring of haystack, - False otherwise. - """ - for n in needles: - if n in haystack: - return True - return False - - -def sum_gradients_all_reduce(single_session, - dev_prefixes, - tower_grads, - num_workers, - alg, - num_shards, - gpu_indices, - agg_small_grads_max_bytes=0, - agg_small_grads_max_group=10, - allreduce_merge_scope=1): - """Apply all-reduce algorithm over specified gradient tensors. - - Args: - single_session: true if reduction is applied to one graph across - all workers, false if ths application is to a single-worker graph only. - dev_prefixes: list of prefix strings to use to generate PS device names. - tower_grads: the gradients to reduce. - num_workers: number of worker processes across entire job. - alg: the all-reduce algorithm to apply. - num_shards: alg-specific sharding factor. - gpu_indices: indices of local GPUs in order usable for ring-reduce. - agg_small_grads_max_bytes: largest tensor eligible for aggregation, - in number of bytes. - agg_small_grads_max_group: largest permitted aggregation of small - tensors. - allreduce_merge_scope: size of groups into which to partition consecutive - gradients grouped under a common 'allreduce' name scope for application - of ScopedAllocator optimization. - - Returns: - list of reduced tensors - """ - alg_contains_shuffle = contains_any(alg, ['pscpu', 'psgpu']) - is_hierarchical = '/' in alg - if 'pscpu' in alg: - aux_devices = [prefix + '/cpu:0' for prefix in dev_prefixes] - elif 'psgpu' in alg: - aux_devices = [ - prefix + '/gpu:%d' % i - for i in range(len(gpu_indices)) - for prefix in dev_prefixes - ] - else: - aux_devices = ['/job:localhost/cpu:0'] - aux_device_groups = group_device_names( - aux_devices, - num_shards if (alg != 'collective' and alg_contains_shuffle) else 1) - group_index = 0 - if agg_small_grads_max_bytes > 0 and agg_small_grads_max_group > 0: - tower_grads, packing = pack_small_tensors( - tower_grads, - max_bytes=agg_small_grads_max_bytes, - max_group=agg_small_grads_max_group) - else: - packing = None - reduced_gv_list = [] - gv = list(zip(*tower_grads)) - merge_scope = allreduce_merge_scope if allreduce_merge_scope > 0 else 1 - chunked_gv = [gv[x:x + merge_scope] - for x in xrange(0, len(gv), merge_scope)] - for chunk in chunked_gv: - with tf.name_scope('allreduce'): - for grad_and_vars in chunk: - reduced_gv_list.append(sum_grad_and_var_all_reduce( - single_session, - grad_and_vars, num_workers, alg, gpu_indices, - (aux_devices if is_hierarchical - else aux_device_groups[group_index]), - num_shards)) - group_index = (group_index + 1) % len(aux_device_groups) - new_tower_grads = [list(x) for x in zip(*reduced_gv_list)] - if packing: - new_tower_grads = unpack_small_tensors(new_tower_grads, packing) - return new_tower_grads - - -def extract_ranges(index_list, range_size_limit=32): - """Extract consecutive ranges and singles from index_list. - - Args: - index_list: List of monotone increasing non-negative integers. - range_size_limit: Largest size range to return. If a larger - consecutive range exists it will be returned as multiple - ranges. - - Returns: - ranges, singles where ranges is a list of [first, last] pairs of - consecutive elements in index_list, and singles is all of the - other elements, in original order. - """ - if not index_list: - return [], [] - first = index_list[0] - last = first - ranges = [] - singles = [] - for i in index_list[1:]: - if i == last + 1 and (last - first) <= range_size_limit: - last = i - else: - if last > first: - ranges.append([first, last]) - else: - singles.append(first) - first = i - last = i - if last > first: - ranges.append([first, last]) - else: - singles.append(first) - return ranges, singles - - -GradPackTuple = pycoll.namedtuple('GradPackTuple', 'indices vars shapes') - - -def pack_range(key, packing, grad_vars, rng): - """Form the concatenation of a specified range of gradient tensors. - - Args: - key: Value under which to store meta-data in packing that will be used - later to restore the grad_var list structure. - packing: Dict holding data describing packed ranges of small tensors. - grad_vars: List of (grad, var) pairs for one tower. - rng: A pair of integers giving the first, last indices of a consecutive - range of tensors to be packed. - - Returns: - A tensor that is the concatenation of all the specified small tensors. - """ - to_pack = grad_vars[rng[0]:rng[1] + 1] - members = [] - variables = [] - restore_shapes = [] - with tf.name_scope('pack'): - for g, v in to_pack: - variables.append(v) - restore_shapes.append(g.shape) - with tf.device(g.device): - members.append(tf.reshape(g, [-1])) - packing[key] = GradPackTuple( - indices=range(rng[0], rng[1] + 1), - vars=variables, - shapes=restore_shapes) - with tf.device(members[0].device): - return tf.concat(members, 0) - - -def unpack_grad_tuple(gv, gpt): - """Unpack a previously packed collection of gradient tensors. - - Args: - gv: A (grad, var) pair to be unpacked. - gpt: A GradPackTuple describing the packing operation that produced gv. - - Returns: - A list of (grad, var) pairs corresponding to the values that were - originally packed into gv, maybe following subsequent operations like - reduction. - """ - elt_widths = [x.num_elements() for x in gpt.shapes] - with tf.device(gv[0][0].device): - with tf.name_scope('unpack'): - splits = tf.split(gv[0], elt_widths) - unpacked_gv = [] - for idx, s in enumerate(splits): - unpacked_gv.append((tf.reshape(s, gpt.shapes[idx]), gpt.vars[idx])) - return unpacked_gv - - -def pack_small_tensors(tower_grads, max_bytes=0, max_group=0): - """Concatenate small gradient tensors together for reduction. - - Args: - tower_grads: List of lists of (gradient, variable) tuples. - max_bytes: Int giving max number of bytes in a tensor that - may be considered small. - max_group: Int giving max number of small tensors that may be - concatenated into one new tensor. - - Returns: - new_tower_grads, packing where new_tower_grads is identical to - tower_grads except that all feasible small_tensors have been removed - from their places and concatenated into larger tensors that are - now in the front of the list for each tower, and packing contains - the data necessary to restore the tower_grads structure. - - Look through the first tower for gradients of the same type (float), - and small size, that are all sequential. For each such group, - replace by a new tensor that is a flattened concatenation. Note - that the corresponding variable will be absent, which doesn't matter - because it isn't used during all-reduce. - - Requires: - Every gv_list in towers must have isomorphic structure including identical - tensor sizes and types. - """ - small_indices = [] - large_indices = [] - for idx, (g, _) in enumerate(tower_grads[0]): - if g.dtype == tf.float32 and (4 * g.shape.num_elements()) <= max_bytes: - small_indices.append(idx) - else: - large_indices.append(idx) - small_ranges, small_singles = extract_ranges( - small_indices, range_size_limit=max_group) - large_indices = sorted(large_indices + small_singles) - num_gv = len(tower_grads[0]) - packing = {} - if small_ranges: - new_tower_grads = [] - for dev_idx, gv_list in enumerate(tower_grads): - assert len(gv_list) == num_gv - new_gv_list = [] - for r in small_ranges: - key = '%d:%d' % (dev_idx, len(new_gv_list)) - new_gv_list.append((pack_range(key, packing, gv_list, r), - 'packing_var_placeholder')) - for i in large_indices: - new_gv_list.append(gv_list[i]) - new_tower_grads.append(new_gv_list) - return new_tower_grads, packing - else: - return tower_grads, None - - -def unpack_small_tensors(tower_grads, packing): - """Undo the structure alterations to tower_grads done by pack_small_tensors. - - Args: - tower_grads: List of List of (grad, var) tuples. - packing: A dict generated by pack_small_tensors describing the changes - it made to tower_grads. - - Returns: - new_tower_grads: identical to tower_grads except that concatentations - of small tensors have been split apart and returned to their original - positions, paired with their original variables. - """ - if not packing: - return tower_grads - new_tower_grads = [] - num_devices = len(tower_grads) - num_packed = len(packing.keys()) // num_devices - for dev_idx, gv_list in enumerate(tower_grads): - new_gv_list = gv_list[num_packed:] - for i in xrange(0, num_packed): - k = '%d:%d' % (dev_idx, i) - gpt = packing[k] - gv = unpack_grad_tuple(gv_list[i], gpt) - for gi, idx in enumerate(gpt.indices): - assert idx == gpt.indices[gi] - new_gv_list.insert(idx, gv[gi]) - new_tower_grads.append(new_gv_list) - return new_tower_grads diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/allreduce_test.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/allreduce_test.py deleted file mode 100644 index c414e48f2b302a516b66d424400c16caaec908be..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/allreduce_test.py +++ /dev/null @@ -1,448 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tests for tf_cnn_benchmark.allreduce.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections as pycoll - -import numpy as np -import tensorflow.compat.v1 as tf -from tensorflow.python.framework import ops -from tensorflow.python.framework import test_util -from tensorflow.python.ops import variables -import allreduce - - -class AllReduceTest(tf.test.TestCase): - - def testGroupKey(self): - d0 = ['/job:worker/replica:0/task:0/device:GPU:1', - '/job:worker/replica:0/task:0/device:GPU:0', - '/job:worker/replica:0/task:0/device:GPU:3',] - d1 = ['/job:worker/replica:0/task:1/device:GPU:1', - '/job:worker/replica:0/task:1/device:GPU:0', - '/job:worker/replica:0/task:1/device:GPU:3',] - d2 = ['/job:worker/replica:0/task:1/device:GPU:1', - '/job:worker/replica:0/task:1/device:GPU:3', - '/job:worker/replica:0/task:1/device:GPU:0',] - d3 = ['/job:worker/replica:0/task:1/device:GPU:1', - '/job:worker/replica:0/task:1/device:GPU:3', - '/job:worker/replica:0/task:1/device:GPU:2',] - d4 = ['/job:worker/task:0/device:GPU:1', - '/job:worker/task:0/device:GPU:2', - '/job:worker/task:0/device:GPU:3',] - d5 = ['/job:worker/task:0/device:CPU:1', - '/job:worker/task:0/device:CPU:2'] - d6 = ['/job:worker/task:0/device:CPU:2', - '/job:worker/task:0/device:CPU:1'] - g0 = allreduce.collective_group_key(d0) - g1 = allreduce.collective_group_key(d1) - g2 = allreduce.collective_group_key(d2) - g3 = allreduce.collective_group_key(d3) - g4 = allreduce.collective_group_key(d4) - g5 = allreduce.collective_group_key(d5) - g6 = allreduce.collective_group_key(d6) - self.assertEqual(g0, g1) - self.assertEqual(g0, g2) - self.assertNotEqual(g0, g3) - self.assertEqual(g3, g4) - self.assertEqual(g5, g6) - self.assertNotEqual(g4, g5) - - def testExtractRanges(self): - x = [] - expected_ranges = [] - expected_singles = [] - ranges, singles = allreduce.extract_ranges(x) - self.assertEqual(expected_ranges, ranges) - self.assertEqual(expected_singles, singles) - x = [1, 3, 4, 6, 7, 8, 9] - expected_ranges = [[3, 4], [6, 9]] - expected_singles = [1] - ranges, singles = allreduce.extract_ranges(x) - self.assertEqual(expected_ranges, ranges) - self.assertEqual(expected_singles, singles) - x = [1, 2, 3, 4, 6, 7, 8, 9] - expected_ranges = [[1, 4], [6, 9]] - expected_singles = [] - ranges, singles = allreduce.extract_ranges(x) - self.assertEqual(expected_ranges, ranges) - self.assertEqual(expected_singles, singles) - x = [1, 3, 4, 6, 7, 9] - expected_ranges = [[3, 4], [6, 7]] - expected_singles = [1, 9] - ranges, singles = allreduce.extract_ranges(x) - self.assertEqual(expected_ranges, ranges) - self.assertEqual(expected_singles, singles) - x = [1, 3, 6, 9] - expected_ranges = [] - expected_singles = [1, 3, 6, 9] - ranges, singles = allreduce.extract_ranges(x) - self.assertEqual(expected_ranges, ranges) - self.assertEqual(expected_singles, singles) - - def testPackRange(self): - packing = {} - t0 = tf.constant([0, 1, 2, 3], dtype=tf.float32) - t1 = tf.constant([4, 5, 6, 7], dtype=tf.float32) - - gv = [(t0, 'v0'), (t1, 'v1')] - new_t = allreduce.pack_range('0:0', packing, gv, [0, 1]) - self.assertEqual(1, new_t.shape.ndims) - self.assertEqual(8, new_t.shape.dims[0]) - self.assertEqual( - packing, { - '0:0': - allreduce.GradPackTuple( - indices=range(2), - vars=['v0', 'v1'], - shapes=[tf.TensorShape([4]), - tf.TensorShape([4])]) - }) - - t2 = tf.constant([[0, 1, 2], [3, 4, 5], [6, 7, 8]], dtype=tf.float32) - t3 = tf.constant([[0, 1, 2], [3, 4, 5], [6, 7, 8]], dtype=tf.float32) - gv = [(t0, 'v0'), (t1, 'v1'), (t2, 'v2'), (t3, 'v3')] - packing = {} - new_t = allreduce.pack_range('1:0', packing, gv, [0, 3]) - self.assertEqual(1, new_t.shape.ndims) - self.assertEqual(26, new_t.shape.dims[0]) - self.assertEqual( - packing, { - '1:0': - allreduce.GradPackTuple( - indices=range(4), - vars=['v0', 'v1', 'v2', 'v3'], - shapes=[ - tf.TensorShape([4]), - tf.TensorShape([4]), - tf.TensorShape([3, 3]), - tf.TensorShape([3, 3]) - ]) - }) - - def testUnpackGradTuple(self): - packing = { - '0:0': - allreduce.GradPackTuple( - indices=range(4), - vars=['v0', 'v1', 'v2', 'v3'], - shapes=[ - tf.TensorShape([4]), - tf.TensorShape([4]), - tf.TensorShape([3, 3]), - tf.TensorShape([3, 3]) - ]) - } - tc = tf.constant([0, 1, 2, 3, 4, 5, 6, 7, - 0, 1, 2, 3, 4, 5, 6, 7, 8, - 0, 1, 2, 3, 4, 5, 6, 7, 8], dtype=tf.float32) - packed_gv = [tc, 'packing_var_placeholder'] - gv = allreduce.unpack_grad_tuple(packed_gv, packing['0:0']) - self.assertLen(gv, 4) - self.assertEqual('v0', gv[0][1]) - self.assertEqual('v1', gv[1][1]) - self.assertEqual('v2', gv[2][1]) - self.assertEqual('v3', gv[3][1]) - self.assertEqual(1, gv[0][0].shape.ndims) - self.assertEqual(4, gv[0][0].shape.dims[0]) - self.assertEqual(1, gv[1][0].shape.ndims) - self.assertEqual(4, gv[1][0].shape.dims[0]) - self.assertEqual(2, gv[2][0].shape.ndims) - self.assertEqual(3, gv[2][0].shape.dims[0]) - self.assertEqual(3, gv[2][0].shape.dims[1]) - - def testPackSmallTensors(self): - t0 = tf.constant([0, 1, 2, 3], dtype=tf.float32) - t1 = tf.constant([4, 5, 6, 7], dtype=tf.float32) - t2 = tf.constant([[0, 1, 2], [3, 4, 5], [6, 7, 8]], dtype=tf.float32) - t3 = tf.constant([[0, 1, 2], [3, 4, 5], [6, 7, 8]], dtype=tf.float32) - tower_grads = [] - for d in range(0, 3): - gv = [(t0, 'v_%d_0' % d), (t1, 'v_%d_1' %d), (t2, 'v_%d_2' %d), - (t3, 'v_%d_3' % d)] - tower_grads.append(gv) - - # 1) Set the size limit so small that nothing gets concatenated. - new_tower_grads, packing = allreduce.pack_small_tensors( - tower_grads, max_bytes=12, - max_group=10) - self.assertEqual(tower_grads, new_tower_grads) - self.assertIs(packing, None) - - # 2) Set the size limit so only the first two tensors get concatenated - new_tower_grads, packing = allreduce.pack_small_tensors( - tower_grads, max_bytes=16, # 16 bytes == 4 elements - max_group=10) - self.assertLen(new_tower_grads, 3) - self.assertLen(tower_grads[0], 4) - first_tower = new_tower_grads[0] - self.assertLen(first_tower, 3) - self.assertEqual(1, first_tower[0][0].shape.ndims) - self.assertEqual(8, first_tower[0][0].shape.dims[0]) - self.assertEqual(packing, - {'0:0': allreduce.GradPackTuple( - indices=range(2), - vars=['v_0_0', 'v_0_1'], - shapes=[tf.TensorShape([4]), - tf.TensorShape([4])]), - '1:0': allreduce.GradPackTuple( - indices=range(2), - vars=['v_1_0', 'v_1_1'], - shapes=[tf.TensorShape([4]), - tf.TensorShape([4])]), - '2:0': allreduce.GradPackTuple( - indices=range(2), - vars=['v_2_0', 'v_2_1'], - shapes=[tf.TensorShape([4]), - tf.TensorShape([4])])}) - - # 3) Set the size limit so all tensors get concatenated - new_tower_grads, packing = allreduce.pack_small_tensors( - tower_grads, max_bytes=256, # bytes = 64 elements - max_group=10) - self.assertLen(new_tower_grads, 3) - self.assertLen(tower_grads[0], 4) - self.assertLen(new_tower_grads[0], 1) - first_tower = new_tower_grads[0] - self.assertEqual(1, first_tower[0][0].shape.ndims) - self.assertEqual(26, first_tower[0][0].shape.dims[0]) - self.assertEqual(packing, - {'0:0': allreduce.GradPackTuple( - indices=range(4), - vars=['v_0_0', 'v_0_1', 'v_0_2', 'v_0_3'], - shapes=[tf.TensorShape([4]), - tf.TensorShape([4]), - tf.TensorShape([3, 3,]), - tf.TensorShape([3, 3,])]), - '1:0': allreduce.GradPackTuple( - indices=range(4), - vars=['v_1_0', 'v_1_1', 'v_1_2', 'v_1_3'], - shapes=[tf.TensorShape([4]), - tf.TensorShape([4]), - tf.TensorShape([3, 3,]), - tf.TensorShape([3, 3,])]), - '2:0': allreduce.GradPackTuple( - indices=range(4), - vars=['v_2_0', 'v_2_1', 'v_2_2', 'v_2_3'], - shapes=[tf.TensorShape([4]), - tf.TensorShape([4]), - tf.TensorShape([3, 3,]), - tf.TensorShape([3, 3,])])}) - - def testUnpackSmallTensors(self): - packing = {'0:0': allreduce.GradPackTuple(indices=range(2), - vars=['v_0_0', 'v_0_1'], - shapes=[tf.TensorShape([4]), - tf.TensorShape([4])]), - '0:1': allreduce.GradPackTuple(indices=range(3, 5), - vars=['v_0_3', 'v_0_4'], - shapes=[tf.TensorShape([3, 3,]), - tf.TensorShape([3, 3,])]), - '1:0': allreduce.GradPackTuple(indices=range(2), - vars=['v_1_0', 'v_1_1'], - shapes=[tf.TensorShape([4]), - tf.TensorShape([4])]), - '1:1': allreduce.GradPackTuple(indices=range(3, 5), - vars=['v_1_3', 'v_1_4'], - shapes=[tf.TensorShape([3, 3,]), - tf.TensorShape([3, 3,])])} - t0 = tf.constant([0, 1, 2, 3, 4, 5, 6, 7], dtype=tf.float32) - t1 = tf.constant([17, 17], dtype=tf.float32) - t2 = tf.constant([0, 1, 2, 3, 4, 5, 6, 7, 8, - 0, 1, 2, 3, 4, 5, 6, 7, 8], dtype=tf.float32) - t3 = tf.constant([0], dtype=tf.float32) - tower_grads = [] - for d in range(0, 2): - one_tower = [(t0, 'packing_var_placeholder'), - (t2, 'packing_var_placeholder'), - (t1, 'v_%d_2' % d), (t3, 'v_%d_5' %d)] - tower_grads.append(one_tower) - new_tower_grads = allreduce.unpack_small_tensors(tower_grads, packing) - self.assertLen(new_tower_grads, 2) - for d, tg in enumerate(new_tower_grads): - self.assertLen(tg, 6) - self.assertEqual('v_%d_0' % d, tg[0][1]) - self.assertEqual('v_%d_1' % d, tg[1][1]) - self.assertEqual('v_%d_2' % d, tg[2][1]) - self.assertEqual('v_%d_3' % d, tg[3][1]) - self.assertEqual('v_%d_4' % d, tg[4][1]) - self.assertEqual('v_%d_5' % d, tg[5][1]) - self.assertEqual(1, tg[0][0].shape.ndims) - self.assertEqual(4, tg[0][0].shape.dims[0]) - self.assertEqual(1, tg[1][0].shape.ndims) - self.assertEqual(4, tg[1][0].shape.dims[0]) - self.assertEqual(1, tg[2][0].shape.ndims) - self.assertEqual(2, tg[2][0].shape.dims[0]) - self.assertEqual(2, tg[3][0].shape.ndims) - self.assertEqual(3, tg[3][0].shape.dims[0]) - self.assertEqual(3, tg[3][0].shape.dims[1]) - self.assertEqual(2, tg[4][0].shape.ndims) - self.assertEqual(3, tg[4][0].shape.dims[0]) - self.assertEqual(3, tg[4][0].shape.dims[1]) - self.assertEqual(1, tg[5][0].shape.ndims) - self.assertEqual(1, tg[5][0].shape.dims[0]) - - -class DynamicPackingTest(test_util.TensorFlowTestCase): - """Packing/Unpacking tests that require executing a TensorFlow session.""" - - def _init_tensors(self, num_towers, tensor_shapes): - """Construct a collection of tensors across multiple devices.""" - num_tensors = len(tensor_shapes) - consts = [] - tensors = [] - vrbls = [] - tower_grads = [] - tf.Variable([-1], dtype=tf.int32, name='packing_var_placeholder') - for dev_idx in range(0, num_towers): - devname = '/job:localhost/device:GPU:%d' % dev_idx - consts.append([]) - tensors.append([]) - vrbls.append([]) - with tf.device(devname): - base_value = 0 - gv_tuples = [] - for t_idx in range(0, num_tensors): - shape = tensor_shapes[t_idx] - num_elts = 0 - for d in shape: - num_elts = (num_elts or 1) * d - c = np.fromiter(range(base_value, base_value + num_elts), - dtype=np.float32).reshape(shape) - base_value += num_elts - consts[dev_idx].append(c) - tensors[dev_idx].append(tf.constant(c)) - vrbls[dev_idx].append( - tf.Variable(c, name='v_d%d_t%d' % (dev_idx, t_idx))) - gv_tuples.append((tensors[dev_idx][-1], vrbls[dev_idx][-1])) - tower_grads.append(gv_tuples) - return tower_grads, consts, tensors, vrbls - - _test_tuple = pycoll.namedtuple('_test_tuple', - 'num_devices, in_shapes out_shapes out_i') - - def _do_pack_unpack_test(self, tt): - """Do a single pack-unpack test. - - Args: - tt: A _test_tuple defining the parameters of the test to do. - - This test executes a graph that performs a pack of tower_grads - followed by an unpack and verifies that the shapes and values - of gradient tensors are unchanged, along with paired variables. - """ - with ops.Graph().as_default(): - tower_grads, consts, _, vrbls = self._init_tensors( - tt.num_devices, tt.in_shapes) - packed_tg, packing = allreduce.pack_small_tensors( - tower_grads, max_bytes=40, max_group=10) - unpacked_tg = allreduce.unpack_small_tensors(packed_tg, packing) - with self.test_session() as sess: - sess.run(variables.global_variables_initializer()) - packed = sess.run(packed_tg) - for d in range(0, tt.num_devices): - for t in range(0, len(tt.out_shapes)): - num_elts = 0 - for dim in tt.out_shapes[t]: - num_elts = (num_elts or 1) * dim - self.assertTrue(np.array_equal( - np.array(range(tt.out_i[t], tt.out_i[t] + num_elts), - dtype=np.float32).reshape(tt.out_shapes[t]), - packed[d][t][0])) - unpacked = sess.run(unpacked_tg) - for d in range(0, tt.num_devices): - for t in range(0, len(tt.in_shapes)): - self.assertTrue(np.array_equal(consts[d][t], unpacked[d][t][0])) - self.assertEqual(vrbls[d][t], unpacked_tg[d][t][1]) - - def testPackUnpack0(self): - self._do_pack_unpack_test( - self._test_tuple(num_devices=3, - in_shapes=[[8], [3, 3], [12], [5, 5, 5]], - out_shapes=[[17], [12], [5, 5, 5]], - out_i=[0, 17, 29])) - - def testPackUnpack1(self): - self._do_pack_unpack_test( - self._test_tuple(num_devices=4, - in_shapes=[[5, 5, 5], [2, 3], [5]], - out_shapes=[[11], [5, 5, 5]], - out_i=[125, 0])) - - def testPackUnpack2(self): - self._do_pack_unpack_test( - self._test_tuple(num_devices=2, - in_shapes=[[5, 5, 5], [2, 3], [1, 5], [7], [100]], - out_shapes=[[18], [5, 5, 5], [100]], - out_i=[125, 0, 143])) - - def _do_all_reduce_pack_test(self, tt): - """Test that all-reduce results are the same with or without packing.""" - with ops.Graph().as_default(): - tower_grads, consts, _, _ = self._init_tensors( - tt.num_devices, tt.in_shapes) - dev_prefixes = ['/job:localhost'] - num_workers = 1 - alg = 'xring' - shards = 1 - single_session = True - gpu_indices = range(0, tt.num_devices) - assert len(gpu_indices) == len(tower_grads) - no_pack_all_reduce = allreduce.sum_gradients_all_reduce( - single_session, - dev_prefixes, tower_grads, num_workers, alg, shards, - gpu_indices, - agg_small_grads_max_bytes=0, agg_small_grads_max_group=1) - packed_tg, packing = allreduce.pack_small_tensors(tower_grads, 100, 100) - packed_all_reduce = allreduce.sum_gradients_all_reduce( - single_session, - dev_prefixes, packed_tg, num_workers, alg, shards, - gpu_indices, - agg_small_grads_max_bytes=0, agg_small_grads_max_group=1) - unpacked_tg = allreduce.unpack_small_tensors(packed_all_reduce, packing) - with self.test_session() as sess: - sess.run(variables.global_variables_initializer()) - no_pack_values = sess.run(no_pack_all_reduce) - pack_unpack_values = sess.run(unpacked_tg) - for d in range(1, tt.num_devices): - for t in range(0, len(tt.in_shapes)): - self.assertTrue(np.allclose(no_pack_values[d][t][0], - tt.num_devices * consts[0][t])) - self.assertTrue(np.array_equal(no_pack_values[d][t][0], - pack_unpack_values[d][t][0])) - - def testAllReducePacked0(self): - self._do_all_reduce_pack_test( - self._test_tuple(num_devices=3, - in_shapes=[[8], [3, 3], [12], [5, 5, 5]], - out_shapes=[[17], [12], [5, 5, 5]], - out_i=[0, 17, 29])) - - def testAllReducePacked1(self): - self._do_all_reduce_pack_test( - self._test_tuple(num_devices=2, - in_shapes=[[8], [3, 3], [12], [5, 5, 5], [3], [4]], - out_shapes=[[17], [7], [12], [5, 5, 5]], - out_i=[0, 17, 29, 154, 157])) - - -if __name__ == '__main__': - tf.disable_v2_behavior() - tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/batch_allreduce.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/batch_allreduce.py deleted file mode 100644 index a2cbe68da0f66dabced033a4f3e33a5d54462764..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/batch_allreduce.py +++ /dev/null @@ -1,628 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains classes and functions for doing a single-machine batch all-reduce. - -An all-reduce is taking the reduction (typically a sum) of a list of tensors, -each on a different device. The result must end up back on each device, which is -where the word "all" comes from. In summary, each device starts with a single -tensor, and ends up with the reduction of all tensors. - -A batch all-reduce is doing several independent all-reduces. When doing a batch -all-reduce, care is taken to evenly distribute the reduction computations -across devices and inter-device tensor transfers across device links. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# TODO(reedwm): Support distributed all-reduces in this file. -# TODO(reedwm): Merge this code with allreduce.py, which contains some batch -# all-reduce code that this file calls. allreduce.py also supports distributed -# batch-reduce while this file only supports single-machine all-reduce. - -import abc - -import six -import tensorflow.compat.v1 as tf - -from tensorflow.python.ops import data_flow_ops -import allreduce -import constants - - -def _all_reduce_using_copy(tensors_across_devices, use_mean): - """Does an all-reduce of a list of tensors by copying to the current device. - - The tensors are copied to the current device and then reduced. - - Args: - tensors_across_devices: A list of tensors, each on a different device. - use_mean: Whether to take the mean of the tensors instead of a sum: - Returns: - A reduced tensor on the current device. - """ - reduced_tensor = tf.add_n(tensors_across_devices) - if use_mean: - reduced_tensor *= 1 / len(tensors_across_devices) - return reduced_tensor - - -@six.add_metaclass(abc.ABCMeta) -class BatchAllReduceAlgorithm(object): - """Represents an algorithm for performing a batch all-reduce operation.""" - - def batch_all_reduce(self, - all_device_tensors, - num_splits, - compact_tensors, - defer_tensors, - xla_compile=False): - """Performs a batch all-reduce. - - The reduction done is a sum. - - `all_device_tensors` is a list of list of tensors that will be batch - all-reduced. All tensors within a single inner list must be on the same - device. The nth element in each list, for any n, will be reduced together. - The return value is in the same form as `all_device_tensors`, except that - each tensor is reduced. - - For example, if `all_device_tensors` is: - [[ A, B ], # A and B are on GPU 0 - [ C, D ]] # C and D are on GPU 1 - - Then the return value will be: - [[ A+C, B+D ], # These two tensors are on GPU 0 - [ A+C, B+D ]] # These two tensors are on GPU 1 - - Args: - all_device_tensors: A list of list of tensors. `all_device_tensors[i][j]` - is a tensor where `i` is the device index and `j` is the tensor index. - num_splits: If not None, tensors will be concatenated and split into this - many pieces during the all-reduce, then split back into their original - shapes afterwards. Has no impact on correctness and can improve - performance. Requires all tensors to be the same type. - compact_tensors: If True, tensors are casted to fp16 before being all- - reduced. Improves performance, but hurts numerical stability. - defer_tensors: If True, every time the return value - `reduced_all_device_tensors` is evaluated, the result will be the - reduced tensors values of `all_device_tensors` from the previous session - run instead of the current session run, or zero on the first session - run. This can improve performance. When training neural networks, - deferring gradients often does not harm training, so this can be used to - improve performance. - xla_compile: If True, use XLA to compile gradients packing and unpacking - ops. - - Returns: - reduced_all_device_tensors: A list in the same form as - `all_device_tensors`, except each tensor has been reduced. - warmup_ops: A list of ops needed to be run once before the all-reduce can - occur. - """ - - # Before all-reducing tensors, we do several preprocessing functions that - # can speed up the all-reduce. We undo these functions after all-reducing - # the tensors. - - # all_device_packed_tensors is a 2-d list of tensors indexed by - # [device_id][tensor_id], holding packed tensors from all devices involved - # in all-reduce. - all_device_packed_tensors = [] - - # all_device_warmup_ops is a 2-d list of ops indexed by - # [device_id][tensor_id], holding warmup_ops that need to be run once before - # all-reduce can occur. - all_device_warmup_ops = [] - - # all_device_put_ops is a 2-d list of ops indexed by - # [device_id][tensor_id], holding put ops for deferred tensors. They will be - # called in each all-reduce step automatically due to control dependency. - all_device_put_ops = [] - - # packers is a list of _TensorPacker, one for each device involved in - # all-reduce. - packers = [ - _TensorPacker(num_splits, compact_tensors) for _ in all_device_tensors - ] - - for packer, device_tensors in zip(packers, all_device_tensors): - - def pack_single_device_tensors(packer=packer, - device_tensors=device_tensors): - """Pack gradient tensors of a device.""" - packed_tensors = packer.maybe_concat_tensors(device_tensors) - packed_tensors = packer.maybe_compact_tensors(packed_tensors) - # When xla_compile=False, defer tensors after concat for better - # performance. - if defer_tensors and not xla_compile: - packed_tensors, put_ops, warmup_ops = defer_single_device_tensors( - packed_tensors) - all_device_put_ops.append(put_ops) - all_device_warmup_ops.append(warmup_ops) - packed_tensors = packer.maybe_split_tensors(packed_tensors) - return packed_tensors - - with tf.device(device_tensors[0].device): - if xla_compile: - packed_tensors = tf.xla.experimental.compile( - pack_single_device_tensors) - # When xla_compile=True, intermediate tensors in packing process are - # not materialized. Thus, we defer tensors after packing process is - # completed instead of in the middle of it. - if defer_tensors: - packed_tensors, put_ops, warmup_ops = defer_single_device_tensors( - packed_tensors) - all_device_put_ops.append(put_ops) - all_device_warmup_ops.append(warmup_ops) - else: - packed_tensors = pack_single_device_tensors() - - all_device_packed_tensors.append(packed_tensors) - - # Perform all-reduce on packed tensors. - all_device_tensors = self._do_batch_all_reduce(all_device_packed_tensors) - - all_device_unpacked_tensors = [] - for packer, device_tensors in zip(packers, all_device_tensors): - - def unpack_single_device_tensors(packer=packer, - device_tensors=device_tensors): - """Unpack gradient tensors of a device.""" - unpacked_tensors = packer.undo_maybe_split_tensors(device_tensors) - unpacked_tensors = packer.undo_maybe_compact_tensors(unpacked_tensors) - unpacked_tensors = packer.undo_maybe_concat_tensors(unpacked_tensors) - return unpacked_tensors - - with tf.device(device_tensors[0].device): - if xla_compile: - unpacked_device_tensor = tf.xla.experimental.compile( - unpack_single_device_tensors) - else: - unpacked_device_tensor = unpack_single_device_tensors() - - all_device_unpacked_tensors.append(unpacked_device_tensor) - - # Note: There is no undo operation for deferring tensors. But we do need to - # call _add_put_op_control_deps at the end if we deferred the tensors. - if defer_tensors: - all_device_unpacked_tensors = _add_put_op_control_deps( - all_device_unpacked_tensors, num_splits, all_device_put_ops) - - return all_device_unpacked_tensors, all_device_warmup_ops - - @abc.abstractmethod - def _do_batch_all_reduce(self, all_device_tensors): - """Performs a batch all-reduce. - - Unlike `self.batch_all_reduce`, this does not do any preprocessing of the - tensors. - - Args: - all_device_tensors: A list of list of tensors. `all_device_tensors[i][j]` - is a tensor where `i` is the device index and `j` is the tensor index. - Returns: - reduced_all_device_tensors: A list in the same form as - `all_device_tensors`, except each tensor has been reduced. - """ - pass - - -class CopyToDeviceAlgorithm(BatchAllReduceAlgorithm): - """An algorithm that copies tensors to be reduced to a specific device.""" - - def __init__(self, devices_to_reduce_on, use_mean=False): - self._devices = devices_to_reduce_on - self._use_mean = use_mean - - def _do_batch_all_reduce(self, all_device_tensors): - reduced_tensors = [] - for i, tensors_across_devices in enumerate(zip(*all_device_tensors)): - with tf.device(self._devices[i % len(self._devices)]): - reduced_tensor = _all_reduce_using_copy(tensors_across_devices, - self._use_mean) - reduced_tensors.append(reduced_tensor) - # The tensors will be brought back to each device once they are used. - return [reduced_tensors] * len(all_device_tensors) - - -class HierarchicalCopyAlgorithm(BatchAllReduceAlgorithm): - """An algorithm that uses hierarchical copies. This is only optimized for - eight devices connected in NetworkTopology.DGX1 or NetworkTopology.GCP_V100 - topology. - """ - - def __init__(self, network_topology): - """Initializer for HierarchicalCopyAlgorithm. - - Args: - network_topology: An instance of Enum class constants.NetworkTopology. - """ - self._network_topology = network_topology - - def _do_batch_all_reduce(self, all_device_tensors): - avail_devices = [device_tensors[0].device - for device_tensors in all_device_tensors] - reduced_tensors = [] - num_devices = len(avail_devices) - group_size = num_devices // 2 - for i, tensors_across_devices in enumerate(zip(*all_device_tensors)): - group_0_main_device, group_1_main_device = self.__get_main_devices( - i, num_devices) - if group_0_main_device < group_size: - group_0_begin = 0 - group_1_begin = group_size - else: - group_0_begin = group_size - group_1_begin = 0 - - # Reduce the first group. - group_0_tensors = tensors_across_devices[group_0_begin: - group_0_begin + group_size] - with tf.device(avail_devices[group_0_main_device]): - group_0_reduced_tensor = _all_reduce_using_copy(group_0_tensors, False) - - # Reduce the second group. - group_1_tensors = tensors_across_devices[group_1_begin: - group_1_begin + group_size] - with tf.device(avail_devices[group_1_main_device]): - group_1_reduced_tensor = _all_reduce_using_copy(group_1_tensors, False) - - # Reduce between the groups. - with tf.device(avail_devices[group_0_main_device]): - total_reduced_tensor = _all_reduce_using_copy( - [group_0_reduced_tensor, group_1_reduced_tensor], False) - - # Broadcast the result back into the root of each group. - with tf.device(avail_devices[group_0_main_device]): - group_0_reduced_tensor_bcast = tf.identity(total_reduced_tensor) - with tf.device(avail_devices[group_1_main_device]): - group_1_reduced_tensor_bcast = tf.identity(total_reduced_tensor) - - reduced_tensors_bcast = [] - for j in range(len(tensors_across_devices)): - with tf.device(avail_devices[j]): - # Broadcast the result back to each member in the group from the root. - if (group_0_main_device < group_size) == (j < group_size): - src_device_tensor = group_0_reduced_tensor_bcast - else: - src_device_tensor = group_1_reduced_tensor_bcast - reduced_tensors_bcast.append(tf.identity(src_device_tensor)) - - reduced_tensors.append(reduced_tensors_bcast) - - reduced_tensors = list(zip(*reduced_tensors)) - return reduced_tensors - - def __get_main_devices(self, tensor_index, num_devices): - """Returns the pair of main devices to use for initial reduction. - - Args: - tensor_index: Index of the current tensor in the list of tensors to copy. - num_devices: Total number of devices. - - Returns: - A tuple containing pair of main device indices for the initial - reduction. Then, the first element of the tuple should be used for the - final reduction. - - Raises: - ValueError: Invalid input arguments. - """ - if self._network_topology == constants.NetworkTopology.DGX1: - return tensor_index % num_devices, (tensor_index + - (num_devices // 2)) % num_devices - elif self._network_topology == constants.NetworkTopology.GCP_V100: - if num_devices != 8: - raise ValueError('HierarchicalCopy only supports eight devices in %s.' % - self._network_topology) - # TODO(hinsu): Generalize main device indices to handle any other - # isomorphic connection graph that connects two cliques using connections - # other than 0-5 and 2-7. - main_device_pairs = [(0, 5), (2, 7), (5, 0), (7, 2)] - return main_device_pairs[tensor_index % len(main_device_pairs)] - else: - # TODO(reedwm): make this logic more general for arbitrary topology. - raise ValueError( - 'HierarchicalCopy is not supported for %s network topology.' % - self._network_topology) - - -class AllReduceSpecAlgorithm(BatchAllReduceAlgorithm): - """An algorithm that uses an all reduce spec.""" - - def __init__(self, all_reduce_spec, gpu_indices, agg_small_grads_max_bytes, - agg_small_grads_max_group): - spec = allreduce.parse_all_reduce_spec(all_reduce_spec) - if len(spec) != 1: - raise ValueError( - 'Replicated mode does not support hybrid all-reduce strategies') - self._all_reduce_spec = spec[0] - self._gpu_indices = gpu_indices - self._agg_small_grads_max_bytes = agg_small_grads_max_bytes - self._agg_small_grads_max_group = agg_small_grads_max_group - - def _do_batch_all_reduce(self, all_device_tensors): - # TODO(reedwm): Merge allreduce.sum_gradients_all_reduce with the other - # gradient aggregation code, since gradient aggregation is doing an all - # reduce. Currently, we do gradient repacking in two different places. - # TODO(reedwm): Change the allreduce code to reduce tensors instead of - # tower_grads. - tower_grads = [[(t, None) for t in device_tensors] - for device_tensors in all_device_tensors] - aggregated_device_grads = allreduce.sum_gradients_all_reduce( - False, # single_session - ['/job:localhost'], - tower_grads, - 1, - self._all_reduce_spec.alg, - self._all_reduce_spec.shards, - self._gpu_indices, - agg_small_grads_max_bytes=self._agg_small_grads_max_bytes, - agg_small_grads_max_group=self._agg_small_grads_max_group) - return [[t for t, _ in grad_vars] for grad_vars in aggregated_device_grads] - - -def algorithm_from_params(params): - """Returns a BatchAllReduceAlgorithm from a Params tuple.""" - if params.all_reduce_spec: - if params.gpu_indices: - gpu_indices = [int(x) for x in params.gpu_indices.split(',')] - else: - gpu_indices = [x for x in range(params.num_gpus)] - return AllReduceSpecAlgorithm(params.all_reduce_spec, gpu_indices, - params.agg_small_grads_max_bytes, - params.agg_small_grads_max_group) - elif params.hierarchical_copy: - return HierarchicalCopyAlgorithm(params.network_topology) - else: - if params.local_parameter_device == 'gpu': - devices_to_reduce_on = ['/gpu:%d' % i for i in range(params.num_gpus)] - else: - devices_to_reduce_on = ['/cpu:0'] - return CopyToDeviceAlgorithm(devices_to_reduce_on) - - -def _apply_to_all_device_tensors(all_device_tensors, apply_func, colocate=True): - """Applies a function to each tensor in `all_device_tensors`. - - A new list of lists of tensors is returned, where every tensor in - `all_device_tensors` has had `apply_func` called on it. `all_device_tensors` - is not modified. - - Args: - all_device_tensors: A list of list of tensors. `all_device_tensors[i][j]` is - a tensor where `i` is the device index and `j` is the tensor index. - apply_func: A function taking in three arguments: tensor, device_index, - tensor_index, and returning a modified tensor. - `tensor` is `all_device_tensors[device_index][tensor_index]`. - colocate: If True, apply_func will be run under context manager colocated - with it's input tensor. - Returns: - A list in the same form as `all_device_tensors`, except each tensor has had - `apply_func` called on it. - """ - new_all_device_tensors = [] - for device_index, device_tensors in enumerate(all_device_tensors): - new_device_tensors = [] - for tensor_index, t in enumerate(device_tensors): - if colocate: - with tf.colocate_with(t): - new_t = apply_func(t, device_index, tensor_index) - else: - new_t = apply_func(t, device_index, tensor_index) - new_device_tensors.append(new_t) - new_all_device_tensors.append(new_device_tensors) - return new_all_device_tensors - - -def _defer_tensor(tensor): - """Defers the retrieval of a tensor. - - The tensor is put into a StagingArea, and the return value is the - retrieval of the tensor from the StagingArea. The effect is that the - tensor returned from this function is the tensor that was put in the - StagingArea for the previous Session.run() call. - - Args: - tensor: The tensor to defer for one step. - - Returns: - deferred_tensor: The tensor deferred for one step. - put_op: An op to put `tensor` in the StagingArea. Must be run every step - that `deferred_tensor` is run. - warmup_op: A warmup op that should be called before the first step. Puts - a zero tensor into the StagingArea. - """ - tensor_stage = data_flow_ops.StagingArea([tensor.dtype], [tensor.shape]) - put_op = tensor_stage.put([tensor]) - warmup_op = tensor_stage.put([tf.zeros(tensor.shape, dtype=tensor.dtype)]) - - # Fetch the next tensor to use. - (tensor,) = tensor_stage.get() - return tensor, put_op, warmup_op - - -def defer_single_device_tensors(device_tensors): - """Defer tensors (gradients in this case) from a single device. - - Args: - device_tensors: A list of gradients tensors from a single device to defer. - - Returns: - deferred_tensors: A list of tensors deferred for one step. - put_ops: A list of ops that put `tensors` in the StagingAreas. Must be run - every step that `deferred_tensors` is run. - warmup_ops: Warmup ops that should be called before the first step. Puts - zero tensors into the StagingArea. - """ - put_ops = [] - warmup_ops = [] - deferred_tensors = [] - - for tensor in device_tensors: - deferred_tensor, put_op, warmup_op = _defer_tensor(tensor) - deferred_tensors.append(deferred_tensor) - put_ops.append(put_op) - warmup_ops.append(warmup_op) - - return deferred_tensors, put_ops, warmup_ops - - -def _add_put_op_control_deps(all_device_tensors, num_splits, put_ops): - """Add control dependencies from `put_ops` to `all_device_tensors`. - - This should only be called when deferred tensors are being used. - - The control dependencies are added so that the put ops are run whenever - `all_device_tensors` is run. That way, the caller does not have to explicitly - run the put ops. - - Args: - all_device_tensors: A list of list of tensors. `all_device_tensors[i][j]` is - a tensor where `i` is the device index and `j` is the tensor index. - num_splits: The number of splits that were used for the all-reduce. - put_ops: A list of put ops from deferring the tensors. - Returns: - A list in the same form as `all_device_tensors`, except each tensor has a - control dependency on an op in `put_ops`. - - """ - def apply_func(tensor, device_index, tensor_index): - if num_splits == 0: - deps = [put_ops[device_index][tensor_index]] - else: - deps = put_ops[device_index] - assert len(deps) == 1 - with tf.control_dependencies(deps): - return tf.identity(tensor, name='control_dependency') - return _apply_to_all_device_tensors(all_device_tensors, apply_func) - - -class _TensorPacker(object): - """Packs and unpacks tensors into groups. - - This class first concatenates a set of tensors, then split the concatenated - tensor into a small number of chunks. This is useful for all-reducing tensors, - as doing a small number of all-reduces on large tensors can be faster than - doing a large number of all-reduces on small tensors. - - It also provides option to compact tensors by casting them to fp16, for better - all-reduce performance. - - This class maintains states of processed tensors like shapes and types. So - each packer can only be used to pack and unpack one list of tensors. If you - need to pack multiple lists of tensors (say from multiple devices), then you - need multiple _TensorPacker object, one for each device. - """ - - def __init__(self, num_splits, compact): - """Initializes the _TensorPacker. - - Args: - num_splits: The number of tensors to split the concatenated tensor into. - The batch all-reduce will consist of `num_splits` all-reduces. if None - or zero, tensors are not split or concatenated. - compact: If True, tensors are casted to fp16 during packing and casted - back to their original dtypes during unpacking. - """ - self._num_splits = num_splits - self._compact = compact - self._before_compact_dtypes = [] - - def maybe_concat_tensors(self, device_tensors): - """Concatenate tensors into a single tensor.""" - if not self._num_splits: - return device_tensors - - flat_tensors = [tf.reshape(t, [-1]) for t in device_tensors] - self._orig_shapes = [t.shape for t in device_tensors] - self._orig_sizes = [s.num_elements() for s in self._orig_shapes] - # All shapes must be fully defined. - assert None not in self._orig_sizes - concatenated_grad = tf.concat(flat_tensors, 0) - return [concatenated_grad] - - def maybe_split_tensors(self, concatenated_tensor): - """Split concatenated tensor into `num_splits` pieces.""" - if not self._num_splits: - return concatenated_tensor - - if len(concatenated_tensor) != 1: - raise RuntimeError('tensors must be concatenated via ' - 'maybe_concat_tensors() before splitting') - - concatenated_tensor = concatenated_tensor[0] - total_tensor_size = concatenated_tensor.shape.num_elements() - split_size = total_tensor_size // self._num_splits - split_size_last = total_tensor_size - split_size * (self._num_splits - 1) - split_sizes = [split_size] * (self._num_splits - 1) + [split_size_last] - tensor_packs = tf.split(concatenated_tensor, split_sizes) - return tensor_packs - - def undo_maybe_split_tensors(self, tensor_packs): - """Undo maybe_split_tensors().""" - if not self._num_splits: - return tensor_packs - - return [tf.concat(tensor_packs, 0)] - - def undo_maybe_concat_tensors(self, concatenated_tensor): - """Undo maybe_concat_tensors().""" - if not self._num_splits: - return concatenated_tensor - - if len(concatenated_tensor) != 1: - raise RuntimeError( - 'undo_maybe_split_tensors() must be called before ' - 'undo_maybe_concat_tensors when num_splits is greater than 1') - concatenated_tensor = concatenated_tensor[0] - - tensors_with_sizes = tf.split(concatenated_tensor, - self._orig_sizes) - tensors_with_shapes = [ - tf.reshape(grad, shape) for grad, shape in zip( - tensors_with_sizes, self._orig_shapes) - ] - return tensors_with_shapes - - def maybe_compact_tensors(self, device_tensors): - """Cast tensors to fp16 and store their original types.""" - if not self._compact: - return device_tensors - - if self._before_compact_dtypes: - raise RuntimeError('maybe_compact_tensors can only be called once.') - - self._before_compact_dtypes = [t.dtype for t in device_tensors] - compact_tensors = [tf.cast(t, tf.float16) for t in device_tensors] - - return compact_tensors - - def undo_maybe_compact_tensors(self, compact_tensors): - """Undo maybe_compact_tensors().""" - if not self._compact: - return compact_tensors - - if not self._before_compact_dtypes: - raise RuntimeError('maybe_compact_tensors() must be called before ' - 'undo_maybe_compact_tensors()') - - device_tensors = [ - tf.cast(t, dtype) - for t, dtype in zip(compact_tensors, self._before_compact_dtypes) - ] - return device_tensors diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/benchmark_cnn.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/benchmark_cnn.py deleted file mode 100644 index 6cec0a3be880b330d18696fd7084015fe5e2f6ba..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/benchmark_cnn.py +++ /dev/null @@ -1,3548 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""TensorFlow benchmark library. - -See the README for more information. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import argparse -from collections import namedtuple -import contextlib -import math -import multiprocessing -import os -import re -import threading -import time -import traceback - -from absl import flags as absl_flags -import numpy as np - -import six -from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow.compat.v1 as tf - -# pylint: disable=g-direct-tensorflow-import -import cnn_util -import constants -import datasets -import flags -import mlperf -import variable_mgr -import variable_mgr_util -from cnn_util import log_fn -from models import model_config -from platforms import util as platforms_util -from google.protobuf import text_format -from tensorflow.core.protobuf import rewriter_config_pb2 -from tensorflow.python import debug as tf_debug -from tensorflow.python.client import timeline -from tensorflow.python.framework import graph_util -from tensorflow.python.framework import graph_util_impl -from tensorflow.python.framework import importer -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.platform import gfile -from tensorflow.python.util import nest - - -_DEFAULT_NUM_BATCHES = 100 - - -# GraphInfo encapsulates the tensors/ops that we care about after building a -# graph. We use them to benchmark the graph. -GraphInfo = namedtuple( # pylint: disable=invalid-name - 'GraphInfo', - [ - # Ops that produce the input batches (before preprocessing). - 'input_producer_op', - # Ops that adds the preprocessed images to the staging areas - 'enqueue_ops', - # Fetches of sess.run() - 'fetches', - # Op that performs synchronization in distributed mode - 'execution_barrier', - # The global step variable - 'global_step', - # Group of ops that perform per-device initialization work - 'local_var_init_op_group', - # Op to produce summaries - 'summary_op' - ]) - - -# InputProcessingInfo contains various sources of inputs which will be later fed -# into the model. If synthetic data is used, all three fields are None. -InputProcessingInfo = namedtuple( - 'InputProcessingInfo', - [ - # The first two fields are non-None iff datasets prefetching is not - # used. - - # Ops that produce the input batches. - 'input_producer_op', - # A list of StagingArea for each device. - 'input_producer_stages', - - # Input produced using multi device iterator. Non-None iff datasets - # prefetching is used - 'multi_device_iterator_input' - ]) - - -# TODO(reedwm): add upper_bound and lower_bound to appropriate integer and -# float flags, and change certain string flags to enum flags. - -flags.DEFINE_string('model', 'trivial', - 'Name of the model to run, the list of supported models ' - 'are defined in models/model.py') -# The code will first check if it's running under benchmarking mode -# or evaluation mode, depending on 'eval': -# Under the evaluation mode, this script will read a saved model, -# and compute the accuracy of the model against a validation dataset. -# Additional ops for accuracy and top_k predictors are only used under -# this mode. -# Under the benchmarking mode, user can specify whether nor not to use -# the forward-only option, which will only compute the loss function. -# forward-only cannot be enabled with eval at the same time. -flags.DEFINE_boolean('eval', False, 'whether use eval or benchmarking') -flags.DEFINE_integer('eval_interval_secs', 0, - 'How often to run eval on saved checkpoints. Usually the ' - 'same as save_model_secs from the corresponding training ' - 'run. Pass 0 to eval only once.') -flags.DEFINE_integer('eval_during_training_every_n_steps', None, - 'Every n steps during training, pause training, run ' - 'evaluation, then resume training. Must not be used with ' - '--eval, as unlike --eval, this option causes both ' - 'training and eval to be done. This may take slightly ' - 'more GPU memory than running just training or evaluation ' - 'alone. It also may slightly slow down training, even ' - 'when not taking into account the additional time to ' - 'evaluate.', lower_bound=1) -flags.DEFINE_float('eval_during_training_every_n_epochs', None, - 'After every n training epochs, pause training, run ' - 'evaluation, then resume training. See ' - '--eval_during_training_every_n_steps for more information.') -flags.DEFINE_list('eval_during_training_at_specified_steps', [], - 'Specify a list of training steps, pause training at each of ' - 'these steps, run evaluation, then resume training. See ' - '--eval_during_training_every_n_steps for more information.') -flags.DEFINE_list('eval_during_training_at_specified_epochs', [], - 'Specify a list of training epochs, pause training after ' - 'each of these epochs, run evaluation, then resume training. ' - 'See --eval_during_training_every_n_steps for more ' - 'information.') -flags.DEFINE_boolean('forward_only', False, - 'whether use forward-only or training for benchmarking') -flags.DEFINE_boolean('freeze_when_forward_only', False, - 'whether to freeze the graph when in forward-only mode.') -flags.DEFINE_boolean('print_training_accuracy', False, - 'whether to calculate and print training accuracy during ' - 'training') -flags.DEFINE_integer('batch_size', 0, 'batch size per compute device') -flags.DEFINE_integer('eval_batch_size', 0, 'eval batch size per compute device') -flags.DEFINE_integer('batch_group_size', 1, - 'number of groups of batches processed in the image ' - 'producer.') -flags.DEFINE_integer('num_batches', None, 'number of batches to run, excluding ' - 'warmup. Defaults to %d' % _DEFAULT_NUM_BATCHES) -flags.DEFINE_integer('num_eval_batches', None, - 'number of eval batches to run, excluding warmup. ' - 'Defaults to --num_batches') -flags.DEFINE_float('num_epochs', None, - 'number of epochs to run, excluding warmup. ' - 'This and --num_batches cannot both be specified.') -flags.DEFINE_float('num_eval_epochs', None, - 'number of eval epochs to run, excluding warmup. ' - 'Defaults to --num_epochs') -flags.DEFINE_float('stop_at_top_1_accuracy', None, - 'If set, stops training after the evaluation accuracy hits ' - 'this number. Can only be used with one of the ' - '--eval_during_training_* flags.') -flags.DEFINE_boolean('collect_eval_results_async', False, - 'If True, start a separate process to postprocess eval ' - 'results asynchronously. This currently only works with ' - 'the SSD model.') -flags.DEFINE_integer('num_warmup_batches', None, - 'number of batches to run before timing') -flags.DEFINE_integer('autotune_threshold', None, - 'The autotune threshold for the models') -# TODO(tucker): change num_gpus to num_devices -flags.DEFINE_integer('num_gpus', 1, 'the number of GPUs to run on') -flags.DEFINE_string('gpu_indices', '', 'indices of worker GPUs in ring order') -flags.DEFINE_integer('display_every', 10, - 'Number of local steps after which progress is printed ' - 'out') -flags.DEFINE_float('display_perf_ewma', None, - 'If set, display numbers of images/sec using exponentially ' - 'weighted moving avearge with the specified weight, which ' - 'defines how much current value contributes to the reported ' - 'average. Increasing weight makes the reported performance ' - 'number reflect more about the real-time speed instead of ' - 'the entire history', lower_bound=0, upper_bound=1) -flags.DEFINE_string('data_dir', None, - 'Path to dataset in TFRecord format (aka Example ' - 'protobufs). If not specified, synthetic data will be ' - 'used.') -flags.DEFINE_string('data_name', None, - 'Name of dataset: imagenet or cifar10. If not specified, ' - 'it is automatically guessed based on data_dir.') -flags.DEFINE_string('resize_method', 'bilinear', - 'Method for resizing input images: crop, nearest, ' - 'bilinear, bicubic, area, or round_robin. The `crop` mode ' - 'requires source images to be at least as large as the ' - 'network input size. The `round_robin` mode applies ' - 'different resize methods based on position in a batch in ' - 'a round-robin fashion. Other modes support any sizes and ' - 'apply random bbox distortions before resizing (even with ' - 'distortions=False).') -flags.DEFINE_boolean('distortions', False, - 'Enable/disable distortions during image preprocessing. ' - 'These include bbox and color distortions.') -flags.DEFINE_boolean('use_datasets', True, - 'Enable use of datasets for input pipeline') -flags.DEFINE_string('input_preprocessor', 'default', - 'Name of input preprocessor. The list of supported input ' - 'preprocessors are defined in preprocessing.py.') -flags.DEFINE_string('gpu_thread_mode', 'gpu_private', - 'Methods to assign GPU host work to threads. ' - 'global: all GPUs and CPUs share the same global threads; ' - 'gpu_private: a private threadpool for each GPU; ' - 'gpu_shared: all GPUs share the same threadpool.') -flags.DEFINE_integer('per_gpu_thread_count', 0, - 'The number of threads to use for GPU. Only valid when ' - 'gpu_thread_mode is not global.') -flags.DEFINE_boolean('hierarchical_copy', False, - 'Use hierarchical copies. Currently only optimized for ' - 'use on a DGX-1 with 8 GPUs and may perform poorly on ' - 'other hardware. Requires --num_gpus > 1, and only ' - 'recommended when --num_gpus=8') -# TODO(hinsu): Support auto-detection of the network topology while still -# retaining the ability to specify a particular topology for debugging. -flags.DEFINE_enum( - 'network_topology', constants.NetworkTopology.DGX1, - (constants.NetworkTopology.DGX1, constants.NetworkTopology.GCP_V100), - 'Network topology specifies the topology used to connect multiple devices. ' - 'Network topology is used to decide the hierarchy to use for the ' - 'hierarchical_copy.') -flags.DEFINE_integer('gradient_repacking', 0, 'Use gradient repacking. It ' - 'currently only works with replicated mode. At the end of ' - 'each step, it repacks the gradients for more efficient ' - 'cross-device transportation. A non-zero value specifies ' - 'the number of split packs that will be formed.', - lower_bound=0) -flags.DEFINE_boolean('compact_gradient_transfer', True, 'Compact gradient' - 'as much as possible for cross-device transfer and ' - 'aggregation.') -flags.DEFINE_enum('variable_consistency', 'strong', ('strong', 'relaxed'), - 'The data consistency for trainable variables. With strong ' - 'consistency, the variable always have the updates from ' - 'previous step. With relaxed consistency, all the updates ' - 'will eventually show up in the variables. Likely one step ' - 'behind.') -flags.DEFINE_boolean('datasets_repeat_cached_sample', False, - 'Enable use of a special datasets pipeline that reads a ' - 'single TFRecord into memory and repeats it infinitely ' - 'many times. The purpose of this flag is to make it ' - 'possible to write regression tests that are not ' - 'bottlenecked by CNS throughput. ' - 'Use datasets_use_caching to cache input data.') -flags.DEFINE_enum('local_parameter_device', 'gpu', ('cpu', 'gpu', 'CPU', 'GPU'), - 'Device to use as parameter server: cpu or gpu. For ' - 'distributed training, it can affect where caching of ' - 'variables happens.') -flags.DEFINE_enum('device', 'gpu', ('cpu', 'gpu', 'CPU', 'GPU'), - 'Device to use for computation: cpu or gpu') -flags.DEFINE_enum('data_format', 'NCHW', ('NHWC', 'NCHW'), - 'Data layout to use: NHWC (TF native) or NCHW (cuDNN ' - 'native, requires GPU).') -flags.DEFINE_integer('num_intra_threads', None, - 'Number of threads to use for intra-op parallelism. If ' - 'set to 0, the system will pick an appropriate number. ' - 'None is the same as 0 except that it disables intra-op ' - 'parallelism on a GPU.') -flags.DEFINE_integer('num_inter_threads', 0, - 'Number of threads to use for inter-op parallelism. If ' - 'set to 0, the system will pick an appropriate number.') -flags.DEFINE_boolean('use_numa_affinity', False, - 'Whether to turn on NUMA affinity for CPU devices. ' - 'This is probably only useful when --device=cpu.') -flags.DEFINE_string('trace_file', '', - 'Enable TensorFlow tracing and write trace to this file.') -flags.DEFINE_boolean('use_chrome_trace_format', True, - 'If True, the trace_file, if specified, will be in a ' - 'Chrome trace format. If False, then it will be a ' - 'StepStats raw proto.') -_NUM_STEPS_TO_PROFILE = 10 -_NUM_OPS_TO_PRINT = 20 -flags.DEFINE_string('tfprof_file', None, - 'If specified, write a tfprof ProfileProto to this file. ' - 'The performance and other aspects of the model can then ' - 'be analyzed with tfprof. See ' - 'https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/profiler/g3doc/command_line.md ' # pylint: disable=line-too-long - 'for more info on how to do this. The first %d steps ' - 'are profiled. Additionally, the top %d most time ' - 'consuming ops will be printed.\n' - 'Note: profiling with tfprof is very slow, but most of the ' - 'overhead is spent between steps. So, profiling results ' - 'are more accurate than the slowdown would suggest.' % - (_NUM_STEPS_TO_PROFILE, _NUM_OPS_TO_PRINT)) -flags.DEFINE_string('graph_file', None, - 'Write the model\'s graph definition to this file. ' - 'Defaults to binary format unless filename ends in "txt".') -flags.DEFINE_string('partitioned_graph_file_prefix', None, - 'If specified, after the graph has been partitioned and ' - 'optimized, write out each partitioned graph to a file ' - 'with the given prefix.') -flags.DEFINE_enum('optimizer', 'sgd', ('momentum', 'sgd', 'rmsprop', 'adam'), - 'Optimizer to use') -flags.DEFINE_float('init_learning_rate', None, - 'Initial learning rate for training.') -flags.DEFINE_string('piecewise_learning_rate_schedule', None, - 'Specifies a piecewise learning rate schedule based on the ' - 'number of epochs. This is the form LR0;E1;LR1;...;En;LRn, ' - 'where each LRi is a learning rate and each Ei is an epoch ' - 'indexed from 0. The learning rate is LRi if the ' - 'E(i-1) <= current_epoch < Ei. For example, if this ' - 'paramater is 0.3;10;0.2;25;0.1, the learning rate is 0.3 ' - 'for the first 10 epochs, then is 0.2 for the next 15 ' - 'epochs, then is 0.1 until training ends.') -flags.DEFINE_float('num_epochs_per_decay', 0, - 'Steps after which learning rate decays. If 0, the learning ' - 'rate does not decay.') -flags.DEFINE_float('learning_rate_decay_factor', 0, - 'Learning rate decay factor. Decay by this factor every ' - '`num_epochs_per_decay` epochs. If 0, learning rate does ' - 'not decay.') -flags.DEFINE_float('num_learning_rate_warmup_epochs', 0, - 'Slowly increase to the initial learning rate in the first ' - 'num_learning_rate_warmup_epochs linearly.') -flags.DEFINE_float('minimum_learning_rate', 0, - 'The minimum learning rate. The learning rate will ' - 'never decay past this value. Requires `learning_rate`, ' - '`num_epochs_per_decay` and `learning_rate_decay_factor` to ' - 'be set.') -flags.DEFINE_float('resnet_base_lr', None, "Base learning rate at bs=256. Only " - "relevant when training ResNet and utilizing the model's " - "learning rate heuristic (get_learning_rate).") -flags.DEFINE_float('momentum', 0.9, 'Momentum for training.') -flags.DEFINE_float('rmsprop_decay', 0.9, 'Decay term for RMSProp.') -flags.DEFINE_float('rmsprop_momentum', 0.9, 'Momentum in RMSProp.') -flags.DEFINE_float('rmsprop_epsilon', 1.0, 'Epsilon term for RMSProp.') -flags.DEFINE_float('adam_beta1', 0.9, 'Beta2 term for the Adam optimizer') -flags.DEFINE_float('adam_beta2', 0.999, 'Beta2 term for the Adam optimizer') -flags.DEFINE_float('adam_epsilon', 1e-8, 'Epsilon term for the Adam optimizer') -flags.DEFINE_float('gradient_clip', None, - 'Gradient clipping magnitude. Disabled by default.') -flags.DEFINE_float('weight_decay', 0.00004, - 'Weight decay factor for training.') -flags.DEFINE_float('gpu_memory_frac_for_testing', 0, - 'If non-zero, the fraction of GPU memory that will be used. ' - 'Useful for testing the benchmark script, as this allows ' - 'distributed mode to be run on a single machine. For ' - 'example, if there are two tasks, each can be allocated ' - '~40 percent of the memory on a single machine. This is ' - 'also useful for using unified memory, as this can be set ' - 'above 1 to oversubscribe the GPU using unified memory.', - lower_bound=0.) -flags.DEFINE_boolean('use_unified_memory', None, - 'If True, allocate unified memory enabling larger models ' - 'to fit in available device RAM.') -flags.DEFINE_boolean('timestamped_allocator', False, - 'If True marks free BFCAllocator::Chunks with time ' - 'at which they are freed which can allow more efficient ' - 'memory allocation in cases like RDMA networking.') -flags.DEFINE_integer('gpu_kt_max_interval', 0, - 'If > 0, the maximum number of GPU Ops that may be queued ' - 'in a row without also queuing a tracking event.') -flags.DEFINE_integer('gpu_kt_max_bytes', 0, - 'If > 0, the maximum number of bytes ' - 'of GPU memory that may be allocated by sequential ' - 'GPU Ops without queuing a tracking event.') -flags.DEFINE_integer('gpu_kt_max_pending', 0, - 'If > 0 no more than this many GPU tracking events may be ' - 'outstanding at any time. When this limit is reached ' - 'launch of additional kernels will stall until an ' - 'outstanding event completes.') -flags.DEFINE_boolean('use_tf_layers', True, - 'If True, use tf.layers for neural network layers. This ' - 'should not affect performance or accuracy in any way.') -flags.DEFINE_integer('tf_random_seed', 1234, - 'The TensorFlow random seed. Useful for debugging NaNs, ' - 'as this can be set to various values to see if the NaNs ' - 'depend on the seed.') -flags.DEFINE_string('debugger', None, - 'If set, use the TensorFlow debugger. If set to "cli", use ' - 'the local CLI debugger. Otherwise, this must be in the ' - 'form hostname:port (e.g., localhost:7007) in which case ' - 'the experimental TensorBoard debugger will be used') -flags.DEFINE_boolean('use_python32_barrier', False, - 'When on, use threading.Barrier at Python 3.2.') - -flags.DEFINE_boolean('ml_perf', False, - 'When True, change how the Imagenet input pipeline works ' - 'slightly to meet the MLPerf compliance rules. This slows ' - 'down the input pipeline. Without this option, at the end ' - 'of the input pipeline, the image is divided by 127.5, ' - 'then 1.0 is subtracted from it, bringing the image ' - 'values from [0, 255] to [-1.0, 1.0]. With this option, ' - 'each of the three channels (red, green, blue) have the ' - 'average channel value among all image subtracted from ' - 'it, and no division is done.') - -flags.DEFINE_boolean('datasets_use_prefetch', True, - 'Enable use of prefetched datasets for input pipeline. ' - 'This option is meaningless if use_datasets=False.') -flags.DEFINE_integer('datasets_prefetch_buffer_size', 1, - 'Prefetching op buffer size per compute device.') -flags.DEFINE_integer('datasets_num_private_threads', None, - 'Number of threads for a private threadpool created for ' - 'all datasets computation. By default, we pick an ' - 'appropriate number. If set to 0, we use the default ' - 'tf-Compute threads for dataset operations.') -flags.DEFINE_boolean('datasets_use_caching', False, - 'Cache the compressed input data in memory. This improves ' - 'the data input performance, at the cost of additional ' - 'memory.') -flags.DEFINE_integer('datasets_parallel_interleave_cycle_length', None, - 'Number of parallel file readers interleaving input data.') -flags.DEFINE_boolean('datasets_sloppy_parallel_interleave', False, - 'Allow parallel interleave to depart from deterministic ' - 'ordering, by temporarily skipping over files whose ' - 'elements are not readily available. This can increase ' - 'througput in particular in the presence of stragglers.') -flags.DEFINE_integer('datasets_parallel_interleave_prefetch', None, - 'The number of input elements to fetch before they are ' - 'needed for interleaving.') - -flags.DEFINE_integer( - 'multi_device_iterator_max_buffer_size', 1, - 'Configuration parameter for the MultiDeviceIterator that ' - ' specifies the host side buffer size for each device.') - -# Performance tuning parameters. -flags.DEFINE_boolean('winograd_nonfused', True, - 'Enable/disable using the Winograd non-fused algorithms.') -flags.DEFINE_boolean( - 'batchnorm_persistent', True, - 'Enable/disable using the CUDNN_BATCHNORM_SPATIAL_PERSISTENT ' - 'mode for batchnorm.') -flags.DEFINE_boolean('sync_on_finish', False, - 'Enable/disable whether the devices are synced after each ' - 'step.') -flags.DEFINE_boolean('staged_vars', False, - 'whether the variables are staged from the main ' - 'computation') -flags.DEFINE_boolean('force_gpu_compatible', False, - 'whether to enable force_gpu_compatible in GPU_Options') -flags.DEFINE_boolean('allow_growth', None, - 'whether to enable allow_growth in GPU_Options') -flags.DEFINE_boolean('xla', False, 'whether to enable XLA auto-jit compilation') -flags.DEFINE_boolean('xla_compile', False, - 'Enable xla to compile the graph. Uncompilable ops will ' - 'result in fatal errors.') -flags.DEFINE_boolean('fuse_decode_and_crop', True, - 'Fuse decode_and_crop for image preprocessing.') -flags.DEFINE_boolean('distort_color_in_yiq', True, - 'Distort color of input images in YIQ space.') -flags.DEFINE_boolean('enable_optimizations', True, - 'Whether to enable grappler and other optimizations.') -flags.DEFINE_string('rewriter_config', None, - 'Config for graph optimizers, described as a ' - 'RewriterConfig proto buffer.') -flags.DEFINE_enum('loss_type_to_report', 'total_loss', - ('base_loss', 'total_loss'), - 'Which type of loss to output and to write summaries for. ' - 'The total loss includes L2 loss while the base loss does ' - 'not. Note that the total loss is always used while ' - 'computing gradients during training if weight_decay > 0, ' - 'but explicitly computing the total loss, instead of just ' - 'computing its gradients, can have a performance impact.') -flags.DEFINE_boolean('single_l2_loss_op', False, - 'If True, instead of using an L2 loss op per variable, ' - 'concatenate the variables into a single tensor and do a ' - 'single L2 loss on the concatenated tensor.') -flags.DEFINE_boolean('use_resource_vars', False, - 'Use resource variables instead of normal variables. ' - 'Resource variables are slower, but this option is useful ' - 'for debugging their performance.') -flags.DEFINE_boolean('compute_lr_on_cpu', False, - 'If True, do computations related to learning rate on the ' - 'CPU instead of the GPU. This will significantly improve ' - 'XLA performance in some cases.') -flags.DEFINE_boolean('sparse_to_dense_grads', False, - 'If True, convert all sparse gradients to dense gradients ' - 'before passing them to the optimizer to update ' - 'variables. Only affects models with sparse gradients, ' - 'which currently is only the NCF model.') -# Performance tuning specific to MKL. -flags.DEFINE_boolean('mkl', False, 'If true, set MKL environment variables.') -flags.DEFINE_integer('kmp_blocktime', 0, - 'The time, in milliseconds, that a thread should wait, ' - 'after completing the execution of a parallel region, ' - 'before sleeping') -flags.DEFINE_string('kmp_affinity', 'granularity=fine,verbose,compact,1,0', - 'Restricts execution of certain threads (virtual execution ' - 'units) to a subset of the physical processing units in a ' - 'multiprocessor computer.') -flags.DEFINE_integer('kmp_settings', 1, - 'If set to 1, MKL settings will be printed.') - -# fp16 parameters. If use_fp16=False, no other fp16 parameters apply. -flags.DEFINE_boolean('use_fp16', False, - 'Use 16-bit floats for certain tensors instead of 32-bit ' - 'floats. This is currently experimental.') -# TODO(reedwm): The default loss scale of 128 causes most models to diverge -# on the second step with synthetic data. Changing the tf.set_random_seed -# call to tf.set_random_seed(1235) or most other seed values causes the -# issue not to occur. -flags.DEFINE_float('fp16_loss_scale', None, - 'If fp16 is enabled, the loss is multiplied by this amount ' - 'right before gradients are computed, then each gradient ' - 'is divided by this amount. Mathematically, this has no ' - 'effect, but it helps avoid fp16 underflow. Set to 1 to ' - 'effectively disable. Ignored during eval.') -flags.DEFINE_boolean('fp16_vars', False, - 'If fp16 is enabled, also use fp16 for variables. If ' - 'False, the variables are stored in fp32 and casted to ' - 'fp16 when retrieved. Recommended to leave as False.') -flags.DEFINE_boolean('fp16_enable_auto_loss_scale', False, - 'If True and use_fp16 is True, automatically adjust the ' - 'loss scale during training.') -flags.DEFINE_integer('fp16_inc_loss_scale_every_n', 1000, - 'If fp16 is enabled and fp16_enable_auto_loss_scale is ' - 'True, increase the loss scale every n steps.') - -# The method for managing variables: -# parameter_server: variables are stored on a parameter server that holds -# the master copy of the variable. In local execution, a local device -# acts as the parameter server for each variable; in distributed -# execution, the parameter servers are separate processes in the -# cluster. -# For each step, each tower gets a copy of the variables from the -# parameter server, and sends its gradients to the param server. -# replicated: each GPU has its own copy of the variables. To apply -# gradients, an all_reduce algorithm or or regular cross-device -# aggregation is used to replicate the combined gradients to all -# towers (depending on all_reduce_spec parameter setting). -# independent: each GPU has its own copy of the variables, and gradients -# are not shared between towers. This can be used to check performance -# when no data is moved between GPUs. -# distributed_replicated: Distributed training only. Each GPU has a copy -# of the variables, and updates its copy after the parameter servers -# are all updated with the gradients from all servers. Only works with -# cross_replica_sync=true. Unlike 'replicated', currently never uses -# nccl all-reduce for replicating within a server. -# distributed_all_reduce: Distributed training where all replicas run -# in a single session, using all-reduce to mutally reduce the -# gradients. Uses no parameter servers. When there is only one -# worker, this is the same as replicated. -# collective_all_reduce: Distributed training where all replicas run -# independepently except for variable initialization and for -# gradient reduction which is done via collective all-reduce. -# NOTE: collective_all_reduce in conjunction with use_fp16 can -# lead to NaNs in some models (resnet50). TODO(tucker): fix it. -# horovod: Distributed training using Horovod library. Runs workers using -# an MPI framework (e.g. Open MPI). Each worker runs training on -# single GPU, and averages gradients using NCCL or MPI all-reduce. -# See https://github.com/uber/horovod for more details. -flags.DEFINE_enum('variable_update', 'parameter_server', - ('parameter_server', 'replicated', 'distributed_replicated', - 'independent', 'distributed_all_reduce', - 'collective_all_reduce', 'horovod'), - 'The method for managing variables: parameter_server, ' - 'replicated, distributed_replicated, independent, ' - 'distributed_all_reduce, collective_all_reduce, horovod') -flags.DEFINE_string('all_reduce_spec', None, - 'A specification of the all_reduce algorithm to be used ' - 'for reducing gradients. For more details, see ' - 'parse_all_reduce_spec in variable_mgr.py. An ' - 'all_reduce_spec has BNF form:\n' - 'int ::= positive whole number\n' - 'g_int ::= int[KkMGT]?\n' - 'alg_spec ::= alg | alg#int\n' - 'range_spec ::= alg_spec | alg_spec/alg_spec\n' - 'spec ::= range_spec | range_spec:g_int:range_spec\n' - 'NOTE: not all syntactically correct constructs are ' - 'supported.\n\n' - 'Examples:\n ' - '"xring" == use one global ring reduction for all ' - 'tensors\n' - '"pscpu" == use CPU at worker 0 to reduce all tensors\n' - '"nccl" == use NCCL to locally reduce all tensors. ' - 'Limited to 1 worker.\n' - '"nccl/xring" == locally (to one worker) reduce values ' - 'using NCCL then ring reduce across workers.\n' - '"pscpu:32k:xring" == use pscpu algorithm for tensors of ' - 'size up to 32kB, then xring for larger tensors.') - -# If variable_update==distributed_all_reduce then it may be advantageous -# to aggregate small tensors into one prior to reduction. These parameters -# control that aggregation. -flags.DEFINE_integer('agg_small_grads_max_bytes', 0, - 'If > 0, try to aggregate tensors of less than this ' - 'number of bytes prior to all-reduce.') -flags.DEFINE_integer('agg_small_grads_max_group', 10, - 'When aggregating small tensors for all-reduce do not ' - 'aggregate more than this many into one new tensor.') -flags.DEFINE_integer('allreduce_merge_scope', 1, - 'Establish a name scope around this many ' - 'gradients prior to creating the all-reduce operations. ' - 'It may affect the ability of the backend to merge ' - 'parallel ops.') - -# Distributed training parameters. -flags.DEFINE_enum('job_name', '', ('ps', 'worker', 'controller', ''), - 'One of "ps", "worker", "controller", "". Empty for local ' - 'training') -flags.DEFINE_string('ps_hosts', '', 'Comma-separated list of target hosts') -flags.DEFINE_string('worker_hosts', '', 'Comma-separated list of target hosts') -flags.DEFINE_string('controller_host', None, 'optional controller host') -flags.DEFINE_integer('task_index', 0, 'Index of task within the job') -flags.DEFINE_string('server_protocol', 'grpc', 'protocol for servers') -flags.DEFINE_boolean('cross_replica_sync', True, '') -flags.DEFINE_string('horovod_device', '', 'Device to do Horovod all-reduce on: ' - 'empty (default), cpu or gpu. Default with utilize GPU if ' - 'Horovod was compiled with the HOROVOD_GPU_ALLREDUCE ' - 'option, and CPU otherwise.') - -# Summary and Save & load checkpoints. -flags.DEFINE_integer('summary_verbosity', 0, 'Verbosity level for summary ops. ' - 'level 0: disable any summary.\n' - 'level 1: small and fast ops, e.g.: learning_rate, ' - 'total_loss.\n' - 'level 2: medium-cost ops, e.g. histogram of all ' - 'gradients.\n' - 'level 3: expensive ops: images and histogram of each ' - 'gradient.\n') -flags.DEFINE_integer('save_summaries_steps', 0, - 'How often to save summaries for trained models. Pass 0 ' - 'to disable summaries.') -flags.DEFINE_integer('save_model_secs', 0, - 'How often to save trained models. Pass 0 to disable ' - 'saving checkpoints every N seconds. A checkpoint is ' - 'saved after training completes regardless of this ' - 'option.') -flags.DEFINE_integer('save_model_steps', None, - 'How often to save trained models. If specified, ' - 'save_model_secs must not be specified.') -flags.DEFINE_integer('max_ckpts_to_keep', 5, - 'Max number of checkpoints to keep.') -flags.DEFINE_string('train_dir', None, - 'Path to session checkpoints. Pass None to disable saving ' - 'checkpoint at the end.') -flags.DEFINE_string('eval_dir', '/tmp/tf_cnn_benchmarks/eval', - 'Directory where to write eval event logs.') -flags.DEFINE_string('backbone_model_path', None, - 'Path to pretrained backbone model checkpoint. Pass None ' - 'if not using a backbone model.') -flags.DEFINE_enum('trt_mode', '', ['', 'FP32', 'FP16', 'INT8'], - 'If this is specified in forward_only mode and ' - 'freeze_when_forward_only is set to True, use TensorRT to ' - 'optimize the graph before execution.') -flags.DEFINE_integer('trt_max_workspace_size_bytes', 4 << 30, - 'Max workspace size bytes used by the TensorRT optimizer.') - -# Benchmark logging for model garden metric -flags.DEFINE_string('benchmark_log_dir', None, - 'The directory to place the log files containing the ' - 'results of benchmark. The logs are created by ' - 'BenchmarkFileLogger. Requires the root of the Tensorflow ' - 'models repository to be in $PYTHTONPATH.') -flags.DEFINE_string('benchmark_test_id', None, - 'The unique test ID of the benchmark run. It could be the ' - 'combination of key parameters. It is hardware independent ' - 'and could be used compare the performance between ' - 'different test runs. This flag is designed for human ' - 'consumption, and does not have any impact within the ' - 'system.') - -platforms_util.define_platform_params() - - -class GlobalStepWatcher(threading.Thread): - """A helper class for global_step. - - Polls for changes in the global_step of the model, and finishes when the - number of steps for the global run are done. - """ - - def __init__(self, sess, global_step_op, start_at_global_step, - end_at_global_step): - threading.Thread.__init__(self) - self.sess = sess - self.global_step_op = global_step_op - self.start_at_global_step = start_at_global_step - self.end_at_global_step = end_at_global_step - - self.start_time = 0 - self.start_step = 0 - self.finish_time = 0 - self.finish_step = 0 - - def run(self): - while self.finish_time == 0: - time.sleep(.25) - global_step_val, = self.sess.run([self.global_step_op]) - if self.start_time == 0 and global_step_val >= self.start_at_global_step: - # Use tf.logging.info instead of log_fn, since print (which is log_fn) - # is not thread safe and may interleave the outputs from two parallel - # calls to print, which can break tests. - tf.logging.info('Starting real work at step %s at time %s' % - (global_step_val, time.ctime())) - self.start_time = time.perf_counter() - self.start_step = global_step_val - if self.finish_time == 0 and global_step_val >= self.end_at_global_step: - tf.logging.info('Finishing real work at step %s at time %s' % - (global_step_val, time.ctime())) - self.finish_time = time.perf_counter() - self.finish_step = global_step_val - - def done(self): - return self.finish_time > 0 - - def num_steps(self): - return self.finish_step - self.start_step - - def elapsed_time(self): - return self.finish_time - self.start_time - - -class CheckpointNotFoundException(Exception): - pass - - -def create_config_proto(params): - """Returns session config proto. - - Args: - params: Params tuple, typically created by make_params or - make_params_from_flags. - """ - config = tf.ConfigProto() - config.allow_soft_placement = True - if params.num_intra_threads is None: - if params.device == 'gpu': - config.intra_op_parallelism_threads = 1 - else: - config.intra_op_parallelism_threads = params.num_intra_threads - config.inter_op_parallelism_threads = params.num_inter_threads - config.experimental.collective_group_leader = '/job:worker/replica:0/task:0' - config.gpu_options.experimental.collective_ring_order = params.gpu_indices - config.gpu_options.force_gpu_compatible = params.force_gpu_compatible - config.experimental.use_numa_affinity = params.use_numa_affinity - if params.device == 'cpu': - # TODO(tucker): change num_gpus to num_devices - config.device_count['CPU'] = params.num_gpus - if params.allow_growth is not None: - config.gpu_options.allow_growth = params.allow_growth - if params.gpu_memory_frac_for_testing > 0: - config.gpu_options.per_process_gpu_memory_fraction = ( - params.gpu_memory_frac_for_testing) - if params.use_unified_memory: - config.gpu_options.experimental.use_unified_memory = ( - params.use_unified_memory) - if params.timestamped_allocator: - config.gpu_options.experimental.timestamped_allocator = ( - params.timestamped_allocator) - if params.gpu_kt_max_interval > 0: - config.gpu_options.experimental.kernel_tracker_max_interval = ( - params.gpu_kt_max_interval) - if params.gpu_kt_max_bytes > 0: - config.gpu_options.experimental.kernel_tracker_max_bytes = ( - params.gpu_kt_max_bytes) - if params.gpu_kt_max_pending > 0: - config.gpu_options.experimental.kernel_tracker_max_pending = ( - params.gpu_kt_max_pending) - if params.xla: - config.graph_options.optimizer_options.global_jit_level = ( - tf.OptimizerOptions.ON_1) - if params.rewriter_config: - rewriter_config = rewriter_config_pb2.RewriterConfig() - text_format.Merge(params.rewriter_config, rewriter_config) - config.graph_options.rewrite_options.CopyFrom(rewriter_config) - elif not params.enable_optimizations: - config.graph_options.optimizer_options.opt_level = tf.OptimizerOptions.L0 - config.graph_options.rewrite_options.disable_meta_optimizer = True - elif params.variable_update == 'collective_all_reduce': - rewrite_options = config.graph_options.rewrite_options - rewrite_options.scoped_allocator_optimization = ( - rewriter_config_pb2.RewriterConfig.ON) - rewrite_options.scoped_allocator_opts.enable_op.append('CollectiveReduce') - if params.variable_update == 'horovod': - import horovod.tensorflow as hvd # pylint: disable=g-import-not-at-top - config.gpu_options.visible_device_list = str(hvd.local_rank()) - # For collective_all_reduce, ignore all devices except current worker. - if params.variable_update == 'collective_all_reduce': - del config.device_filters[:] - config.device_filters.append( - '/job:%s/replica:0/task:%d' % (params.job_name, params.task_index)) - - # TODO(b/117324590): Re-enable PinToHostOptimizer when b/117324590 is fixed. - # Currently we have to disable PinToHostOptimizer w/ XLA since it causes - # OOM/perf cliffs. - config.graph_options.rewrite_options.pin_to_host_optimization = ( - rewriter_config_pb2.RewriterConfig.OFF) - return config - - -def get_mode_from_params(params): - """Returns the mode in which this script is running. - - Args: - params: Params tuple, typically created by make_params or - make_params_from_flags. - Raises: - ValueError: Unsupported params settings. - """ - if params.forward_only and params.eval: - raise ValueError('Only one of forward_only and eval parameters is true') - - if params.eval: - return constants.BenchmarkMode.EVAL - elif params.forward_only: - return constants.BenchmarkMode.FORWARD_ONLY - elif (params.eval_during_training_every_n_steps or - params.eval_during_training_every_n_epochs or - params.eval_during_training_at_specified_steps or - params.eval_during_training_at_specified_epochs): - return constants.BenchmarkMode.TRAIN_AND_EVAL - else: - return constants.BenchmarkMode.TRAIN - - -# How many digits to show for the loss and accuracies during training. -LOSS_AND_ACCURACY_DIGITS_TO_SHOW = 3 - - -def benchmark_one_step(sess, - fetches, - step, - batch_size, - step_train_times, - trace_filename, - partitioned_graph_file_prefix, - profiler, - image_producer, - params, - summary_op=None, - show_images_per_sec=True, - benchmark_logger=None, - collective_graph_key=0, - should_output_files=True): - """Advance one step of benchmarking.""" - should_profile = profiler and 0 <= step < _NUM_STEPS_TO_PROFILE - need_options_and_metadata = ( - should_profile or collective_graph_key > 0 or - ((trace_filename or partitioned_graph_file_prefix) and step == -2) - ) - if need_options_and_metadata: - run_options = tf.RunOptions() - if (trace_filename and step == -2) or should_profile: - run_options.trace_level = tf.RunOptions.FULL_TRACE - if partitioned_graph_file_prefix and step == -2: - run_options.output_partition_graphs = True - if collective_graph_key > 0: - run_options.experimental.collective_graph_key = collective_graph_key - run_metadata = tf.RunMetadata() - else: - run_options = None - run_metadata = None - summary_str = None - start_time = time.perf_counter() - if summary_op is None: - results = sess.run(fetches, options=run_options, run_metadata=run_metadata) - else: - (results, summary_str) = sess.run( - [fetches, summary_op], options=run_options, run_metadata=run_metadata) - - if not params.forward_only: - lossval = results['average_loss'] - else: - lossval = 0. - if image_producer is not None: - image_producer.notify_image_consumption() - train_time = time.perf_counter() - start_time - step_train_times.append(train_time) - if (show_images_per_sec and step >= 0 and - (step == 0 or (step + 1) % params.display_every == 0)): - speed_mean, speed_uncertainty, speed_jitter = get_perf_timing( - batch_size, step_train_times, params.display_perf_ewma) - log_str = '%i\t%s\t%.*f' % ( - step + 1, - get_perf_timing_str(speed_mean, speed_uncertainty, speed_jitter), - LOSS_AND_ACCURACY_DIGITS_TO_SHOW, lossval) - if 'top_1_accuracy' in results: - log_str += '\t%.*f\t%.*f' % ( - LOSS_AND_ACCURACY_DIGITS_TO_SHOW, results['top_1_accuracy'], - LOSS_AND_ACCURACY_DIGITS_TO_SHOW, results['top_5_accuracy']) - log_fn(log_str) - if benchmark_logger: - benchmark_logger.log_metric( - 'current_examples_per_sec', speed_mean, global_step=step + 1) - if 'top_1_accuracy' in results: - benchmark_logger.log_metric( - 'top_1_accuracy', results['top_1_accuracy'], global_step=step + 1) - benchmark_logger.log_metric( - 'top_5_accuracy', results['top_5_accuracy'], global_step=step + 1) - if need_options_and_metadata: - if should_profile: - profiler.add_step(step, run_metadata) - if trace_filename and step == -2 and should_output_files: - log_fn('Dumping trace to %s' % trace_filename) - trace_dir = os.path.dirname(trace_filename) - if not gfile.Exists(trace_dir): - gfile.MakeDirs(trace_dir) - with gfile.Open(trace_filename, 'w') as trace_file: - if params.use_chrome_trace_format: - trace = timeline.Timeline(step_stats=run_metadata.step_stats) - trace_file.write(trace.generate_chrome_trace_format(show_memory=True)) - else: - trace_file.write(str(run_metadata.step_stats)) - if partitioned_graph_file_prefix and step == -2 and should_output_files: - path, filename = os.path.split(partitioned_graph_file_prefix) - if '.' in filename: - base_filename, ext = filename.rsplit('.', 1) - ext = '.' + ext - else: - base_filename, ext = filename, '' - as_text = filename.endswith('txt') - for graph_def in run_metadata.partition_graphs: - device = graph_def.node[0].device.replace('/', '_').replace(':', '_') - graph_filename = '%s%s%s' % (base_filename, device, ext) - log_fn('Writing partitioned GraphDef as %s to %s' % ( - 'text' if as_text else 'binary', - os.path.join(path, graph_filename))) - tf.train.write_graph(graph_def, path, graph_filename, as_text) - return (summary_str, lossval) - - -def get_perf_timing_str(speed_mean, speed_uncertainty, speed_jitter, scale=1): - if scale == 1: - # TODO(laigd): rename 'images' to maybe 'inputs', same below. - return ('images/sec: %.1f +/- %.1f (jitter = %.1f)' % - (speed_mean, speed_uncertainty, speed_jitter)) - else: - return 'images/sec: %.1f' % speed_mean - - -def get_perf_timing(batch_size, step_train_times, ewma_alpha=None, scale=1): - """Calculate benchmark processing speed.""" - times = np.array(step_train_times) - speeds = batch_size / times - if ewma_alpha: - weights = np.logspace(len(times)-1, 0, len(times), base=1-ewma_alpha) - time_mean = np.average(times, weights=weights) - else: - time_mean = np.mean(times) - speed_mean = scale * batch_size / time_mean - speed_uncertainty = np.std(speeds) / np.sqrt(float(len(speeds))) - speed_jitter = 1.4826 * np.median(np.abs(speeds - np.median(speeds))) - return speed_mean, speed_uncertainty, speed_jitter - - -def load_checkpoint(saver, sess, ckpt_dir): - """Loads checkpoint from provided directory or full path. - - Args: - saver: Saver used to restore the checkpoint. - sess: TensorFlow session. - ckpt_dir: Path to a folder of checkpoints or full path to a checkpoint. - - Returns: - Global step. - """ - model_checkpoint_path = _get_checkpoint_to_load(ckpt_dir) - global_step = model_checkpoint_path.split('/')[-1].split('-')[-1] - if not global_step.isdigit(): - global_step = 0 - else: - global_step = int(global_step) - saver.restore(sess, model_checkpoint_path) - log_fn('Successfully loaded model from %s.' % model_checkpoint_path) - return global_step - - -def _get_checkpoint_to_load(ckpt_dir): - """Returns which checkpoint to load. - - Args: - ckpt_dir: Path to a folder of checkpoints or full path to a checkpoint. - - Returns: - Full path to checkpoint to load. - - Raises: - CheckpointNotFoundException: If checkpoint is not found. - """ - p = re.compile(r'ckpt-\d+$') - if p.search(ckpt_dir): - model_checkpoint_path = ckpt_dir - else: - # Finds latest checkpoint in directory provided - ckpt = tf.train.get_checkpoint_state(ckpt_dir) - if ckpt and ckpt.model_checkpoint_path: - model_checkpoint_path = ckpt.model_checkpoint_path - else: - raise CheckpointNotFoundException('No checkpoint file found in dir:{}'. - format(ckpt_dir)) - return model_checkpoint_path - - -# Params are passed to BenchmarkCNN's constructor. Params is a map from name -# to value, with one field per key in flags.param_specs. -# -# Call make_params() or make_params_from_flags() below to construct a Params -# tuple with default values from flags.param_specs, rather than constructing -# Params directly. -Params = namedtuple('Params', flags.param_specs.keys()) # pylint: disable=invalid-name - - -def validate_params(params): - """Validates that the Params tuple had valid values. - - When command-line flags are defined for each ParamSpec by calling - flags.define_flags(), calling this function is unnecessary because absl - already does flag validation. Otherwise, this function should be called. - - Args: - params: A Params tuple. - Raises: - ValueError: An element of params had an invalid value. - """ - for name, value in params._asdict().items(): - param_spec = flags.param_specs[name] - if param_spec.flag_type in ('integer', 'float'): - if (value is not None and param_spec.kwargs['lower_bound'] is not None and - value < param_spec.kwargs['lower_bound']): - raise ValueError('Param %s value of %s is lower than the lower bound ' - 'of %s' % - (name, value, param_spec.kwargs['lower_bound'])) - if (value is not None and param_spec.kwargs['upper_bound'] is not None and - param_spec.kwargs['upper_bound'] < value): - raise ValueError('Param %s value of %s is higher than the upper bound ' - 'of %s' % - (name, value, param_spec.kwargs['upper_bound'])) - elif (value is not None and param_spec.flag_type == 'enum' and - value not in param_spec.kwargs['enum_values']): - raise ValueError('Param %s of value %s is not in %s'% - (name, value, param_spec.kwargs['enum_values'])) - - -def make_params(**kwargs): - """Create a Params tuple for BenchmarkCNN from kwargs. - - Default values are filled in from flags.param_specs. - - Args: - **kwargs: kwarg values will override the default values. - Returns: - Params namedtuple for constructing BenchmarkCNN. - """ - # Create a (name: default_value) map from flags.param_specs. - default_kwargs = { - name: flags.param_specs[name].default_value - for name in flags.param_specs - } - params = Params(**default_kwargs)._replace(**kwargs) - validate_params(params) - return params - - -def make_params_from_flags(): - """Create a Params tuple for BenchmarkCNN from absl_flags.FLAGS. - - Returns: - Params namedtuple for constructing BenchmarkCNN. - """ - # Collect (name: value) pairs for absl_flags.FLAGS with matching names in - # flags.param_specs. - flag_values = {name: getattr(absl_flags.FLAGS, name) - for name in flags.param_specs.keys()} - return Params(**flag_values) - - -def remove_param_fields(params, fields_to_remove): - """Remove fields from a Params namedtuple.""" - params_dict = params._asdict() - for field in fields_to_remove: - assert field in params_dict, 'Invalid Params field: ' + field - params_dict = {k: v for k, v in params_dict.items() - if k not in fields_to_remove} - new_params_type = namedtuple('Params', params_dict.keys()) - return new_params_type(**params_dict) - - -def get_num_batches_and_epochs(params, batch_size, num_examples_per_epoch): - """Returns the number of batches and epochs to run for. - - Args: - params: Params tuple, typically created by make_params or - make_params_from_flags. - batch_size: The number of images per step. - num_examples_per_epoch: The number of images in a single epoch. - - Returns: - num_batches: The number of batches to run for. - num_epochs: The number of epochs to run for. This might be slightly - smaller than params.num_epochs if specified, because the number of batches - must be an integer. - - Raises: - ValueError: Invalid or unsupported params. - """ - if params.num_batches and params.num_epochs: - raise ValueError('At most one of --num_batches and --num_epochs may be ' - 'specified.') - if params.num_epochs: - num_batches = int(params.num_epochs * num_examples_per_epoch + - batch_size - 1) // batch_size - else: - num_batches = params.num_batches or _DEFAULT_NUM_BATCHES - num_epochs = num_batches * batch_size / num_examples_per_epoch - return (num_batches, num_epochs) - - -def get_piecewise_learning_rate(piecewise_learning_rate_schedule, - global_step, num_batches_per_epoch): - """Returns a piecewise learning rate tensor. - - Args: - piecewise_learning_rate_schedule: The --piecewise_learning_rate_schedule - parameter - global_step: Scalar tensor representing the global step. - num_batches_per_epoch: float indicating the number of batches per epoch. - - Returns: - A scalar float tensor, representing the learning rate. - - Raises: - ValueError: piecewise_learning_rate_schedule is not formatted correctly. - """ - pieces = piecewise_learning_rate_schedule.split(';') - if len(pieces) % 2 == 0: - raise ValueError('--piecewise_learning_rate_schedule must have an odd ' - 'number of components') - values = [] - boundaries = [] - for i, piece in enumerate(pieces): - if i % 2 == 0: - try: - values.append(float(piece)) - except ValueError: - raise ValueError('Invalid learning rate: ' + piece) - else: - try: - boundaries.append(int(int(piece) * num_batches_per_epoch) - 1) - except ValueError: - raise ValueError('Invalid epoch: ' + piece) - return tf.train.piecewise_constant(global_step, boundaries, values, - name='piecewise_learning_rate') - - -def get_learning_rate(params, global_step, num_examples_per_epoch, model, - batch_size): - """Returns a learning rate tensor based on global_step. - - Args: - params: Params tuple, typically created by make_params or - make_params_from_flags. - global_step: Scalar tensor representing the global step. - num_examples_per_epoch: The number of examples per epoch. - model: The model.Model object to obtain the default learning rate from if no - learning rate is specified. - batch_size: Number of examples per step - - Returns: - A scalar float tensor, representing the learning rate. When evaluated, the - learning rate depends on the current value of global_step. - - Raises: - ValueError: Invalid or unsupported params. - """ - with tf.name_scope('learning_rate'): - num_batches_per_epoch = num_examples_per_epoch / batch_size - - if params.piecewise_learning_rate_schedule: - if (params.init_learning_rate is not None or - params.learning_rate_decay_factor or - params.minimum_learning_rate or params.num_epochs_per_decay): - raise ValueError('No other learning rate-related flags can be ' - 'specified if --piecewise_learning_rate_schedule is ' - 'specified') - learning_rate = get_piecewise_learning_rate( - params.piecewise_learning_rate_schedule, - global_step, num_batches_per_epoch) - elif params.init_learning_rate is not None: - learning_rate = params.init_learning_rate - if (params.num_epochs_per_decay > 0 and - params.learning_rate_decay_factor > 0): - decay_steps = int(num_batches_per_epoch * params.num_epochs_per_decay) - - # Decay the learning rate exponentially based on the number of steps. - learning_rate = tf.train.exponential_decay( - params.init_learning_rate, - global_step, - decay_steps, - params.learning_rate_decay_factor, - staircase=True) - - if params.minimum_learning_rate != 0.: - learning_rate = tf.maximum(learning_rate, - params.minimum_learning_rate) - else: - learning_rate = model.get_learning_rate(global_step, batch_size) - if params.num_learning_rate_warmup_epochs > 0 and ( - params.init_learning_rate is not None or - params.piecewise_learning_rate_schedule): - warmup_steps = int(num_batches_per_epoch * - params.num_learning_rate_warmup_epochs) - init_lr = params.init_learning_rate - if init_lr is None: - init_lr = float(params.piecewise_learning_rate_schedule.split(';')[0]) - warmup_lr = init_lr * tf.cast(global_step, tf.float32) / tf.cast( - warmup_steps, tf.float32) - learning_rate = tf.cond(global_step < warmup_steps, - lambda: warmup_lr, lambda: learning_rate) - - learning_rate = mlperf.logger.log_deferred_tensor_value( - mlperf.tags.OPT_LR, learning_rate, global_step, every_n=100) - return learning_rate - - -def get_optimizer(params, learning_rate): - """Returns the optimizer that should be used based on params.""" - if params.optimizer == 'momentum': - mlperf.logger.log(key=mlperf.tags.OPT_NAME, - value=mlperf.tags.SGD_WITH_MOMENTUM) - mlperf.logger.log(key=mlperf.tags.OPT_MOMENTUM, value=params.momentum) - opt = tf.train.MomentumOptimizer( - learning_rate, params.momentum, use_nesterov=True) - elif params.optimizer == 'sgd': - mlperf.logger.log(key=mlperf.tags.OPT_NAME, value=mlperf.tags.SGD) - opt = tf.train.GradientDescentOptimizer(learning_rate) - elif params.optimizer == 'rmsprop': - opt = tf.train.RMSPropOptimizer( - learning_rate, - params.rmsprop_decay, - momentum=params.rmsprop_momentum, - epsilon=params.rmsprop_epsilon) - elif params.optimizer == 'adam': - opt = tf.train.AdamOptimizer(learning_rate, params.adam_beta1, - params.adam_beta2, params.adam_epsilon) - else: - raise ValueError('Optimizer "{}" was not recognized'. - format(params.optimizer)) - return opt - - -def generate_tfprof_profile(profiler, tfprof_file): - """Generates a tfprof profile, writing it to a file and printing top ops. - - Args: - profiler: A tf.profiler.Profiler. `profiler.add_step` must have already been - called. - tfprof_file: The filename to write the ProfileProto to. - """ - profile_proto = profiler.serialize_to_string() - log_fn('Dumping ProfileProto to %s' % tfprof_file) - with gfile.Open(tfprof_file, 'wb') as f: - f.write(profile_proto) - - # Print out the execution times of the top operations. Note this - # information can also be obtained with the dumped ProfileProto, but - # printing it means tfprof doesn't have to be used if all the user wants - # is the top ops. - options = tf.profiler.ProfileOptionBuilder.time_and_memory() - options['max_depth'] = _NUM_OPS_TO_PRINT - options['order_by'] = 'accelerator_micros' - profiler.profile_operations(options) - - -class BenchmarkCNN(object): - """Class for benchmarking a cnn network.""" - - def __init__(self, params, dataset=None, model=None): - """Initialize BenchmarkCNN. - - Args: - params: Params tuple, typically created by make_params or - make_params_from_flags. - dataset: If not None, the dataset to use. Otherwise, params is used to - obtain the dataset. - model: If not None, the model to use. Otherwise, params is used to obtain - the model. - Raises: - ValueError: Unsupported params settings. - """ - mlperf.logger.log(key=mlperf.tags.RUN_START) - self.params = params - if params.eval: - self._doing_eval = True - else: - # Note self._doing_eval can later switch to True in self._do_eval() if - # self.params.eval_during_training_* is specified. - self._doing_eval = False - self.dataset = dataset or datasets.create_dataset(self.params.data_dir, - self.params.data_name) - self.model = model or model_config.get_model_config( - self.params.model, self.dataset, self.params) - self.trace_filename = self.params.trace_file - self.rewriter_config = self.params.rewriter_config - autotune_threshold = self.params.autotune_threshold if ( - self.params.autotune_threshold) else 1 - min_autotune_warmup = 5 * autotune_threshold * autotune_threshold - self.num_warmup_batches = self.params.num_warmup_batches if ( - self.params.num_warmup_batches is not None) else max( - 10, min_autotune_warmup) - self.graph_file = self.params.graph_file - self.resize_method = self.params.resize_method - self.sync_queue_counter = 0 - self.num_gpus = self.params.num_gpus - if self.params.gpu_indices: - self.gpu_indices = [int(x) for x in self.params.gpu_indices.split(',')] - else: - self.gpu_indices = [x for x in range(self.num_gpus)] - - if (self.params.device == 'cpu' and self.params.data_format == 'NCHW' and - not self.params.mkl): - raise ValueError('device=cpu requires that data_format=NHWC') - - if ((self.params.num_epochs_per_decay or - self.params.learning_rate_decay_factor) and - not (self.params.init_learning_rate is not None and - self.params.num_epochs_per_decay - and self.params.learning_rate_decay_factor)): - raise ValueError('If one of num_epochs_per_decay or ' - 'learning_rate_decay_factor is set, both must be set' - 'and learning_rate must be set') - if (self.params.minimum_learning_rate and - not (self.params.init_learning_rate is not None and - self.params.num_epochs_per_decay and - self.params.learning_rate_decay_factor)): - raise ValueError('minimum_learning_rate requires learning_rate,' - 'num_epochs_per_decay, and ' - 'learning_rate_decay_factor to be set') - - if (self.params.use_fp16 and self.params.fp16_vars and - 'replicated' in self.params.variable_update and - self.params.all_reduce_spec and 'nccl' in self.params.all_reduce_spec): - raise ValueError('fp16 variables are not supported with NCCL') - if (self.params.use_fp16 and self.params.fp16_vars and - self.params.gradient_repacking): - raise ValueError('--fp16_vars cannot be used with --gradient_repacking') - - if self.params.variable_update == 'horovod' and self.params.num_gpus > 1: - raise ValueError('Horovod benchmarks require num_gpus=1 on each worker') - - if self.params.variable_update == 'horovod' and self.params.job_name: - raise ValueError('job_name should not be specified for Horovod.') - - if self.params.use_fp16 and self.params.fp16_enable_auto_loss_scale: - if self.params.all_reduce_spec and 'nccl' in self.params.all_reduce_spec: - raise ValueError('Automatic loss scaling is not supported with NCCL.') - if self.params.variable_update not in ('parameter_server', 'replicated', - 'independent'): - raise ValueError('Automatic loss scaling is not supported with ' - 'variable_update=%s.' % self.params.variable_update) - if self.params.staged_vars: - raise ValueError('Automatic loss scaling is not supported with' - 'staged_vars.') - - if (self.params.debugger is not None and self.params.debugger != 'cli' and - ':' not in self.params.debugger): - raise ValueError('--debugger must be "cli" or in the form ' - 'host:port') - - if self.params.hierarchical_copy and self.params.num_gpus <= 1: - raise ValueError('--hierarchical_copy requires --num_gpus to be greater ' - 'than 1') - - if params.save_model_secs and params.save_model_steps: - raise ValueError('At most one of --save_model_secs and ' - '--save_model_steps can be specified') - - eval_during_training_flags = list(map(bool, [ - params.eval_during_training_every_n_steps, - params.eval_during_training_every_n_epochs, - params.eval_during_training_at_specified_steps, - params.eval_during_training_at_specified_epochs, - ])) - - if eval_during_training_flags.count(True) > 1: - raise ValueError('At most one flag with --eval_during_training_* prefix ' - 'must be specified.') - - eval_during_training_enabled = any(eval_during_training_flags) - - if eval_during_training_enabled: - if params.eval: - raise ValueError('At most one of --eval and --eval_during_training_* ' - 'must be specified') - if params.forward_only: - raise ValueError('At most one of --forward_only and ' - '--eval_during_training_* must be specified') - if params.job_name: - raise ValueError('--eval_during_training_* is not yet supported in ' - 'distributed mode.') - if params.staged_vars: - raise ValueError('--eval_during_training_* is not currently compatible ' - 'with --staged_vars') - - if params.stop_at_top_1_accuracy and not eval_during_training_enabled: - raise ValueError('--stop_at_top_1_accuracy is only supported with ' - '--eval_during_training_*') - if params.collect_eval_results_async and params.model != 'ssd300': - raise ValueError('--collect_eval_results_async only works with ssd300 ' - 'model currently.') - if self.params.forward_only and self.params.freeze_when_forward_only: - if self.params.train_dir is not None: - raise ValueError('In forward_only mode, when --freeze_when_forward_only' - ' is True, --train_dir should not be specified') - if self.params.data_dir and not self.params.datasets_use_prefetch: - raise ValueError('In forward_only mode, when --freeze_when_forward_only' - ' is True and --data_dir is set, ' - '--datasets_use_prefetch should be set to True') - if self.params.job_name: - raise ValueError('In forward_only mode, when --freeze_when_forward_only' - ' is True, --job_name should not be specified and ' - 'distributed running is not supported') - self.forward_only_and_freeze = True - else: - self.forward_only_and_freeze = False - if self.params.trt_mode: - raise ValueError('--trt_mode should not be specified if one of ' - '--forward_only and --freeze_when_forward_only is set ' - 'to False') - - self.mode = get_mode_from_params(self.params) - - # Use the batch size from the command line if specified, otherwise use the - # model's default batch size. Scale the benchmark's batch size by the - # number of GPUs. - if self.params.batch_size > 0: - self.model.set_batch_size(self.params.batch_size) - self.batch_size = self.model.get_batch_size() * self.num_gpus - if self.mode in (constants.BenchmarkMode.TRAIN, - constants.BenchmarkMode.TRAIN_AND_EVAL): - self.train_batch_size = self.batch_size - else: - self.train_batch_size = None - if self.mode in (constants.BenchmarkMode.EVAL, - constants.BenchmarkMode.TRAIN_AND_EVAL): - if self.params.eval_batch_size > 0: - self.eval_batch_size = self.params.eval_batch_size * self.num_gpus - else: - self.eval_batch_size = self.batch_size - else: - self.eval_batch_size = None - self.batch_group_size = self.params.batch_group_size - self.enable_auto_loss_scale = ( - self.params.use_fp16 and self.params.fp16_enable_auto_loss_scale) - self.loss_scale = None - self.loss_scale_normal_steps = None - - self.job_name = self.params.job_name # "" for local training - - # PS server is used for distributed jobs not using all-reduce. - use_ps_server = self.job_name and (self.params.variable_update != - 'distributed_all_reduce' and - self.params.variable_update != - 'collective_all_reduce') - # controller is used for distributed_all_reduce with > 1 worker. - use_controller = ( - self.params.variable_update == 'distributed_all_reduce' and - self.job_name) - if use_controller and not params.controller_host: - raise ValueError('When variable_update==distributed_all_reduce ' - 'controller_host must also be specified.') - self.single_session = ( - self.params.variable_update == 'distributed_all_reduce') - # collective_all_reduce doesn't need a controller or ps - self.distributed_collective = ( - self.params.variable_update == 'collective_all_reduce' and - self.job_name) - - self.local_parameter_device_flag = self.params.local_parameter_device - if self.job_name: - self.task_index = self.params.task_index - self.cluster_manager = platforms_util.get_cluster_manager( - params, create_config_proto(params)) - assert isinstance(self.cluster_manager, cnn_util.BaseClusterManager) - - worker_prefix = '/job:worker/replica:0/task:%s' % self.task_index - if use_ps_server: - self.param_server_device = tf.train.replica_device_setter( - worker_device=worker_prefix + '/cpu:0', - cluster=self.cluster_manager.get_cluster_spec()) - # This device on which the queues for managing synchronization between - # servers should be stored. - self.sync_queue_devices = [ - '/job:ps/replica:0/task:%s/cpu:0' % i - for i in range(self.cluster_manager.num_ps()) - ] - else: - self.sync_queue_devices = ['/job:worker/replica:0/task:0/cpu:0'] - else: - self.task_index = 0 - self.cluster_manager = None - worker_prefix = '' - self.param_server_device = '/%s:0' % self.params.local_parameter_device - self.sync_queue_devices = [self.param_server_device] - - if self.cluster_manager: - self.num_workers = self.cluster_manager.num_workers() - elif self.params.variable_update == 'horovod': - import horovod.tensorflow as hvd # pylint: disable=g-import-not-at-top - self.num_workers = hvd.size() - else: - self.num_workers = 1 - self.num_ps = self.cluster_manager.num_ps() if self.cluster_manager else 0 - - if self.num_workers > 1 and self.params.all_reduce_spec == 'nccl': - raise ValueError('--all_reduce_spec=nccl is invalid in a ' - 'multi-worker job') - - # Device to use for ops that need to always run on the local worker's CPU. - self.cpu_device = '%s/cpu:0' % worker_prefix - - # Device to use for ops that need to always run on the local worker's - # compute device, and never on a parameter server device. - self.raw_devices = [ - '%s/%s:%i' % (worker_prefix, self.params.device, i) - for i in xrange(self.num_gpus) - ] - - subset = 'validation' if params.eval else 'train' - self.num_batches, self.num_epochs = get_num_batches_and_epochs( - params, self.batch_size * self.num_workers, - self.dataset.num_examples_per_epoch(subset)) - if self.mode in (constants.BenchmarkMode.EVAL, - constants.BenchmarkMode.TRAIN_AND_EVAL): - # TODO(reedwm): Currently we do extra eval logic for num_eval_batches and - # the preprocessor. We should encapsulate this logic into a shared - # function or class. - if params.num_eval_batches is None and params.num_eval_epochs is None: - eval_params = self.params - else: - eval_params = self.params._replace( - num_batches=self.params.num_eval_batches, - num_epochs=self.params.num_eval_epochs) - self.num_eval_batches, self.num_eval_epochs = get_num_batches_and_epochs( - eval_params, self.eval_batch_size * self.num_workers, - self.dataset.num_examples_per_epoch('validation')) - else: - self.num_eval_batches, self.num_eval_epochs = None, None - - num_train_examples_per_epoch = self.dataset.num_examples_per_epoch('train') - if self.params.eval_during_training_every_n_epochs: - n_epochs = self.params.eval_during_training_every_n_epochs - self.eval_during_training_at_specified_steps = { - (int(e * num_train_examples_per_epoch + self.batch_size - 1) // - self.batch_size) - for e in np.arange(n_epochs, self.num_epochs, n_epochs)} - - if self.params.eval_during_training_at_specified_steps: - try: - self.eval_during_training_at_specified_steps = set(map( - int, self.params.eval_during_training_at_specified_steps)) - except ValueError: - raise ValueError('Param eval_during_training_at_specified_steps value ' - 'of %s cannot be converted to a list of integers.' % - (self.params.eval_during_training_at_specified_steps)) - - if self.params.eval_during_training_at_specified_epochs: - try: - n_epochs = list(map( - float, self.params.eval_during_training_at_specified_epochs)) - offset = n_epochs[0] - 1 - if offset.is_integer(): - offset = int(offset) - mlperf.logger.log(key=mlperf.tags.EVAL_EPOCH_OFFSET, value=offset) - self.eval_during_training_at_specified_steps = { - (int(e * num_train_examples_per_epoch + self.batch_size - 1) // - self.batch_size) - for e in n_epochs} - except ValueError: - raise ValueError('Param eval_during_training_at_specified_epochs value ' - 'of %s cannot be converted to a list of floats.' % - (self.params.eval_during_training_at_specified_epochs)) - - if params.eval_during_training_every_n_epochs: - offset = params.eval_during_training_every_n_epochs - 1 - if offset.is_integer(): - offset = int(offset) - mlperf.logger.log(key=mlperf.tags.EVAL_EPOCH_OFFSET, value=offset) - - if (self.params.staged_vars and - self.params.variable_update != 'parameter_server'): - raise ValueError('staged_vars for now is only supported with ' - 'variable_update=parameter_server') - - if self.params.variable_update == 'parameter_server': - if self.job_name: - if not self.params.staged_vars: - self.variable_mgr = variable_mgr.VariableMgrDistributedFetchFromPS( - self) - else: - self.variable_mgr = ( - variable_mgr.VariableMgrDistributedFetchFromStagedPS(self)) - else: - if not self.params.staged_vars: - self.variable_mgr = variable_mgr.VariableMgrLocalFetchFromPS(self) - else: - self.variable_mgr = variable_mgr.VariableMgrLocalFetchFromStagedPS( - self) - elif self.params.variable_update == 'replicated': - if self.job_name: - raise ValueError('Invalid variable_update in distributed mode: %s' % - self.params.variable_update) - self.variable_mgr = variable_mgr.VariableMgrLocalReplicated( - self, self.params.all_reduce_spec, - self.params.agg_small_grads_max_bytes, - self.params.agg_small_grads_max_group, - self.params.allreduce_merge_scope) - elif self.params.variable_update == 'distributed_all_reduce': - assert self.params.cross_replica_sync - self.variable_mgr = variable_mgr.VariableMgrDistributedAllReduce( - self, self.params.all_reduce_spec, - ('worker' if self.num_workers > 1 else 'localhost'), - self.num_workers, self.params.agg_small_grads_max_bytes, - self.params.agg_small_grads_max_group, - self.params.allreduce_merge_scope) - elif self.params.variable_update == 'collective_all_reduce': - assert self.params.cross_replica_sync - self.variable_mgr = variable_mgr.VariableMgrCollectiveAllReduce( - self, self.params.all_reduce_spec, - self.num_workers, self.num_gpus, self.task_index, - self.params.allreduce_merge_scope) - elif self.params.variable_update == 'distributed_replicated': - assert self.params.cross_replica_sync - if not self.job_name: - raise ValueError('Invalid variable_update in local mode: %s' % - self.params.variable_update) - self.variable_mgr = variable_mgr.VariableMgrDistributedReplicated(self) - elif self.params.variable_update in ('independent', 'horovod'): - if self.job_name: - raise ValueError('Invalid variable_update in distributed mode: %s' % - self.params.variable_update) - self.variable_mgr = variable_mgr.VariableMgrIndependent(self) - else: - raise ValueError( - 'Invalid variable_update: %s' % self.params.variable_update) - - # Device to use for running on the local worker's compute device, but - # with variables assigned to parameter server devices. - self.devices = self.variable_mgr.get_devices() - if self.job_name: - if use_ps_server: - self.global_step_device = self.param_server_device - elif self.params.variable_update == 'collective_all_reduce': - self.global_step_device = self.cpu_device - else: - self.global_step_device = '/job:worker/replica:0/task:0/cpu:0' - else: - self.global_step_device = self.cpu_device - - self.input_preprocessor = None - self.eval_input_preprocessor = None - if not self.dataset.use_synthetic_gpu_inputs(): - if not self.params.eval: - self.input_preprocessor = self.get_input_preprocessor() - if self.mode in (constants.BenchmarkMode.EVAL, - constants.BenchmarkMode.TRAIN_AND_EVAL): - with self._do_eval(): - self.eval_input_preprocessor = self.get_input_preprocessor() - self.datasets_use_prefetch = ( - self.params.datasets_use_prefetch and - # TODO(rohanj): Figure out why --datasets_use_prefetch freezes on the - # CPU. - self.params.device.lower() != 'cpu' and - self.input_preprocessor and - self.input_preprocessor.supports_datasets()) - self.init_global_step = 0 - - self._config_benchmark_logger() - - if self.mode == constants.BenchmarkMode.TRAIN_AND_EVAL: - # Remove "eval" from params so it is not accidentally used. Since eval can - # still occur despite params.eval being False, params.eval should never - # be used. We cannot yet remove this unconditionally, because the SSD - # model still uses params.eval, and hence does not work properly with - # --eval_during_training_*. - # TODO(b/116627045): We should also remove fields that have an eval - # equivalent, like num_batches and num_eval_batches. - self.params = remove_param_fields(self.params, {'eval'}) - - @contextlib.contextmanager - def _do_eval(self): - """Context manager to switches BenchmarkCNN to eval mode. - - Any evaluation code should be put under this context manager. This context - manager switches self._doing_eval to True. It also switches certain - attributes, like self.num_batches and self.num_epochs, to be the number of - batches and epochs for evaluation respectively - - Yields: - Nothing. - """ - # TODO(b/116627045): Find a more general way of switching attributes to the - # eval equivalents. - old_doing_eval = self._doing_eval - old_num_batches = self.num_batches - old_num_epochs = self.num_epochs - old_batch_size = self.batch_size - try: - self._doing_eval = True - self.num_batches = self.num_eval_batches - self.num_epochs = self.num_eval_epochs - self.batch_size = self.eval_batch_size - self.model.set_batch_size(self.eval_batch_size // self.num_gpus) - yield - finally: - self._doing_eval = old_doing_eval - self.num_batches = old_num_batches - self.num_epochs = old_num_epochs - self.batch_size = old_batch_size - self.model.set_batch_size(old_batch_size // self.num_gpus) - - def _config_benchmark_logger(self): - """Config the model garden benchmark logger.""" - model_benchmark_logger = None - if self.params.benchmark_log_dir is not None: - try: - from official.r1.utils.logs import logger as models_logger # pylint: disable=g-import-not-at-top - except ImportError: - tf.logging.fatal('Please include tensorflow/models to the PYTHONPATH ' - 'in order to use BenchmarkLogger. Configured ' - 'benchmark_log_dir: %s' - % self.params.benchmark_log_dir) - raise - model_benchmark_logger = models_logger.BenchmarkFileLogger( - self.params.benchmark_log_dir) - self.benchmark_logger = model_benchmark_logger - - # TODO(laigd): this changes the global device list which is used everywhere, - # consider refactoring it. - def reset_devices_for_task(self, task_num, is_local=False): - """Used to imitate another task when building a distributed graph.""" - worker_prefix = ('/job:localhost' if is_local else - '/job:worker/replica:0/task:%s' % task_num) - self.cpu_device = '%s/cpu:0' % worker_prefix - self.raw_devices = [ - '%s/%s:%i' % (worker_prefix, self.params.device, i) - for i in xrange(self.num_gpus) - ] - self.devices = self.variable_mgr.get_devices() - - def raw_devices_across_tasks(self, is_local=False): - """Returns list of raw device names across all tasks.""" - if is_local: - assert self.num_workers == 1 - return self.raw_devices - else: - return [ - 'job:worker/replica:0/task%s/%s:%i' % (t, self.params.device, i) - for t in xrange(self.num_workers) - for i in xrange(self.num_gpus) - ] - - def print_info(self): - """Print basic information.""" - benchmark_info = self._get_params_info() - log_fn('Model: %s' % self.model.get_model_name()) - log_fn('Dataset: %s' % benchmark_info['dataset_name']) - log_fn('Mode: %s' % self.mode) - log_fn('SingleSess: %s' % benchmark_info['single_session']) - log_fn('Batch size: %s global' % (self.batch_size * self.num_workers)) - log_fn(' %s per device' % (self.batch_size // - len(self.raw_devices))) - if self.batch_group_size > 1: - log_fn(' %d batches per prepocessing group' % - self.batch_group_size) - log_fn('Num batches: %d' % self.num_batches) - log_fn('Num epochs: %.2f' % self.num_epochs) - log_fn('Devices: %s' % benchmark_info['device_list']) - log_fn('NUMA bind: %s' % self.params.use_numa_affinity) - log_fn('Data format: %s' % self.params.data_format) - if self.rewriter_config: - log_fn('RewriterConfig: %s' % self.rewriter_config) - log_fn('Optimizer: %s' % self.params.optimizer) - log_fn('Variables: %s' % self.params.variable_update) - if (self.params.variable_update == 'replicated' or - self.params.variable_update == 'distributed_all_reduce' - or self.params.variable_update == 'collective_all_reduce'): - log_fn('AllReduce: %s' % self.params.all_reduce_spec) - if self.job_name: - log_fn('Sync: %s' % self.params.cross_replica_sync) - if self.params.staged_vars: - log_fn('Staged vars: %s' % self.params.staged_vars) - if self.params.variable_update == 'horovod' and self.params.horovod_device: - log_fn('Horovod on: %s' % self.params.horovod_device) - log_fn('==========') - - def _get_params_info(self): - """Get the common parameters info for the benchmark run. - - Returns: - A dict of processed parameters. - """ - dataset_name = self.dataset.name - if self.dataset.use_synthetic_gpu_inputs(): - dataset_name += ' (synthetic)' - single_session = self.params.variable_update == 'distributed_all_reduce' - if single_session: - device_list = self.raw_devices_across_tasks() - elif self.params.variable_update == 'horovod': - device_list = ['horovod/%s:%d' % (self.params.device, idx) - for idx in range(self.num_workers)] - else: - device_list = self.raw_devices - return { - 'dataset_name': dataset_name, - 'single_session': single_session, - 'device_list': device_list,} - - def _log_benchmark_run(self): - """Log the benchmark info to the logger. - - The info logged here should be similar to print_info(), but in a structured - JSON format. - """ - if self.benchmark_logger: - benchmark_info = self._get_params_info() - - run_param = { - 'model': self.model.get_model_name(), - 'dataset': benchmark_info['dataset_name'], - 'mode': self.mode, - 'single_sess': benchmark_info['single_session'], - 'devices': benchmark_info['device_list'], - 'batch_size': self.batch_size, - 'batch_size_per_device': self.batch_size // len(self.raw_devices), - 'num_batches': self.num_batches, - 'num_epochs': self.num_epochs, - 'data_format': self.params.data_format, - 'rewrite_config': self.rewriter_config, - 'optimizer': self.params.optimizer, - 'session_config': create_config_proto(self.params), - } - # TODO(scottzhu): tf_cnn_benchmark might execute several times with - # different param setting on the same box. This will cause the run file to - # only contain the latest info. The benchmark_log_dir should be updated - # for every new run. - self.benchmark_logger.log_run_info( - self.model.get_model_name(), benchmark_info['dataset_name'], - run_param, test_id=self.params.benchmark_test_id) - - def run(self): - """Run the benchmark task assigned to this process. - - Returns: - Dictionary of statistics for training or eval. - Raises: - ValueError: unrecognized job name. - """ - if self.params.job_name == 'ps': - log_fn('Running parameter server %s' % self.task_index) - self.cluster_manager.join_server() - return {} - - # For distributed_all_reduce with multiple workers, drive - # from a separate controller process. - if self.params.variable_update == 'distributed_all_reduce': - if self.params.job_name == 'worker': - log_fn('Starting worker %s' % self.task_index) - self.cluster_manager.join_server() - return - elif self.params.job_name and self.params.job_name != 'controller': - raise ValueError('unrecognized job name: %s' % self.params.job_name) - - self._log_benchmark_run() - if self._doing_eval: - with tf.Graph().as_default(): - # TODO(laigd): freeze the graph in eval mode. - return self._run_eval() - else: - return self._benchmark_train() - - def _run_eval(self): - """Evaluate a model every self.params.eval_interval_secs. - - Returns: - Dictionary containing eval statistics. Currently returns an empty - dictionary. - - Raises: - ValueError: If self.params.train_dir is unspecified. - """ - if self.params.train_dir is None: - raise ValueError('Trained model directory not specified') - graph_info = self._build_eval_graph() - saver = tf.train.Saver(self.variable_mgr.savable_variables()) - summary_writer = tf.summary.FileWriter(self.params.eval_dir, - tf.get_default_graph()) - target = '' - # TODO(huangyp): Check if checkpoints haven't updated for hours and abort. - while True: - with tf.Session( - target=target, config=create_config_proto(self.params)) as sess: - image_producer = None - try: - global_step = load_checkpoint(saver, sess, self.params.train_dir) - image_producer = self._initialize_eval_graph( - graph_info.enqueue_ops, graph_info.input_producer_op, - graph_info.local_var_init_op_group, sess) - except CheckpointNotFoundException: - log_fn('Checkpoint not found in %s' % self.params.train_dir) - else: # Only executes if an exception was not thrown - self._eval_once(sess, summary_writer, graph_info.fetches, - graph_info.summary_op, image_producer, global_step) - if image_producer is not None: - image_producer.done() - if self.params.eval_interval_secs <= 0: - break - time.sleep(self.params.eval_interval_secs) - return {} - - def _build_eval_graph(self, scope_name=None): - """Build the evaluation graph. - - Args: - scope_name: String to filter what summaries are collected. Only summary - ops whose name contains `scope_name` will be added, which is useful for - only including evaluation ops. - - Returns: - A GraphInfo named_tuple containing various useful ops and tensors of the - evaluation grpah. - """ - with self._do_eval(): - input_producer_op, enqueue_ops, fetches = self._build_model() - local_var_init_op = tf.local_variables_initializer() - table_init_ops = tf.tables_initializer() - variable_mgr_init_ops = [local_var_init_op] - if table_init_ops: - variable_mgr_init_ops.extend([table_init_ops]) - with tf.control_dependencies([local_var_init_op]): - variable_mgr_init_ops.extend(self.variable_mgr.get_post_init_ops()) - local_var_init_op_group = tf.group(*variable_mgr_init_ops) - - summary_op = tf.summary.merge_all(scope=scope_name) - # The eval graph has no execution barrier because it doesn't run in - # distributed mode. - execution_barrier = None - # We do not use the global step during evaluation. - global_step = None - return GraphInfo(input_producer_op, enqueue_ops, fetches, - execution_barrier, global_step, local_var_init_op_group, - summary_op) - - # TODO(reedwm): For consistency, we should have a similar - # "_initialize_train_graph" function. They can likely be the same function. - def _initialize_eval_graph(self, enqueue_ops, input_producer_op, - local_var_init_op_group, sess): - """Initializes the evaluation graph. - - Args: - enqueue_ops: Ops that adds the preprocessed images to the staging areas. - input_producer_op: Op that produce the input batches (before - preprocessing). - local_var_init_op_group: Group of ops that perform per-device - initialization work. - sess: The session to initialize the eval graph with. - - Returns: - An ImageProducer, or None if an ImageProducer isn't being used. - """ - with self._do_eval(): - if local_var_init_op_group is not None: - # We might reinitialize local variables if they were already initialized - # during training. This is OK. - sess.run(local_var_init_op_group) - if self.dataset.queue_runner_required(): - tf.train.start_queue_runners(sess=sess) - image_producer = None - if input_producer_op is not None: - image_producer = cnn_util.ImageProducer( - sess, input_producer_op, self.batch_group_size, - self.params.use_python32_barrier) - image_producer.start() - if enqueue_ops: - for i in xrange(len(enqueue_ops)): - sess.run(enqueue_ops[:(i + 1)]) - if image_producer is not None: - image_producer.notify_image_consumption() - return image_producer - - def _eval_once(self, sess, summary_writer, fetches, summary_op, - image_producer, global_step): - """Evaluate the model using the validation dataset.""" - with self._do_eval(): - mlperf.logger.log_eval_epoch( - mlperf.tags.EVAL_START, global_step, self.batch_size) - loop_start_time = start_time = time.perf_counter() - # TODO(laigd): refactor the part to compute/report the accuracy. Currently - # it only works for image models. - top_1_accuracy_sum = 0.0 - top_5_accuracy_sum = 0.0 - total_eval_count = self.num_batches * self.batch_size - for step in xrange(self.num_batches): - if (summary_writer and self.params.save_summaries_steps > 0 and - (step + 1) % self.params.save_summaries_steps == 0): - results, summary_str = sess.run([fetches, summary_op]) - summary_writer.add_summary(summary_str) - else: - results = sess.run(fetches) - # Make global_step available in results for postprocessing. - results['global_step'] = global_step - results = self.model.postprocess(results) - top_1_accuracy_sum += results['top_1_accuracy'] - top_5_accuracy_sum += results['top_5_accuracy'] - if (step + 1) % self.params.display_every == 0: - duration = time.perf_counter() - start_time - examples_per_sec = ( - self.batch_size * self.params.display_every / duration) - log_fn('%i\t%.1f examples/sec' % (step + 1, examples_per_sec)) - start_time = time.perf_counter() - if image_producer is not None: - image_producer.notify_image_consumption() - loop_end_time = time.perf_counter() - accuracy_at_1 = top_1_accuracy_sum / self.num_batches - accuracy_at_5 = top_5_accuracy_sum / self.num_batches - summary = tf.Summary() - summary.value.add(tag='eval/Accuracy@1', simple_value=accuracy_at_1) - summary.value.add(tag='eval/Accuracy@5', simple_value=accuracy_at_5) - for result_key, result_value in results.items(): - if result_key.startswith(constants.SIMPLE_VALUE_RESULT_PREFIX): - prefix_len = len(constants.SIMPLE_VALUE_RESULT_PREFIX) - summary.value.add(tag='eval/' + result_key[prefix_len:], - simple_value=result_value) - if summary_writer: - summary_writer.add_summary(summary, global_step) - log_fn('Accuracy @ 1 = %.4f Accuracy @ 5 = %.4f [%d examples]' % - (accuracy_at_1, accuracy_at_5, total_eval_count)) - elapsed_time = loop_end_time - loop_start_time - images_per_sec = (self.num_batches * self.batch_size / elapsed_time) - if self.mode != constants.BenchmarkMode.TRAIN_AND_EVAL: - # Note that we compute the top 1 accuracy and top 5 accuracy for each - # batch, which will have a slight performance impact. - log_fn('-' * 64) - log_fn('total images/sec: %.2f' % images_per_sec) - log_fn('-' * 64) - if self.benchmark_logger: - eval_result = { - 'eval_top_1_accuracy', accuracy_at_1, - 'eval_top_5_accuracy', accuracy_at_5, - 'eval_average_examples_per_sec', images_per_sec, - tf.GraphKeys.GLOBAL_STEP, global_step, - } - self.benchmark_logger.log_evaluation_result(eval_result) - mlperf.logger.log_eval_epoch( - mlperf.tags.EVAL_STOP, global_step, self.batch_size) - mlperf.logger.log(key=mlperf.tags.EVAL_SIZE, - value=self.num_batches * self.batch_size) - if self.params.model != 'ssd300': # ssd300 logs eval accuracy elsewhere. - mlperf.logger.log_eval_accuracy( - accuracy_at_1, global_step, self.train_batch_size, - examples_per_epoch=self.dataset.num_examples_per_epoch('train')) - if self.params.stop_at_top_1_accuracy: - mlperf.logger.log(key=mlperf.tags.EVAL_TARGET, - value=self.params.stop_at_top_1_accuracy) - return accuracy_at_1, accuracy_at_5 - - def _benchmark_train(self): - """Run cnn in benchmark mode. Skip the backward pass if forward_only is on. - - Returns: - Dictionary containing training statistics (num_workers, num_steps, - average_wall_time, images_per_sec). - """ - graph = tf.Graph() - with graph.as_default(): - build_result = self._build_graph() - if self.mode == constants.BenchmarkMode.TRAIN_AND_EVAL: - with self.variable_mgr.reuse_variables(): - with tf.name_scope('Evaluation') as ns: - eval_build_results = self._build_eval_graph(ns) - else: - eval_build_results = None - (graph, result_to_benchmark) = self._preprocess_graph(graph, build_result) - with graph.as_default(): - return self._benchmark_graph(result_to_benchmark, eval_build_results) - - GPU_CACHED_INPUT_VARIABLE_NAME = 'gpu_cached_inputs' - - def _unfreezable_local_variables(self, graph): - """Get the local variables that we don't want to freeze.""" - return graph.get_collection( - tf.GraphKeys.LOCAL_VARIABLES, - # We don't freeze the gpu_cached_images local variable so it won't get - # constant folded with ops which process the input. - scope='.*' + BenchmarkCNN.GPU_CACHED_INPUT_VARIABLE_NAME) - - def _build_graph(self): - """Build the graph. - - Returns: - A namedtuple containing the ops/tensors that required by - _benchmark_graph(). - """ - if self.single_session: - (input_producer_op, enqueue_ops, fetches) = ( - self._build_model_single_session()) - else: - (input_producer_op, enqueue_ops, fetches) = self._build_model() - fetches_list = nest.flatten(list(fetches.values())) - main_fetch_group = tf.group(*fetches_list, name='main_fetch_group') - execution_barrier = None - if (not self.single_session and self.job_name and - not self.params.cross_replica_sync): - execution_barrier = self.add_sync_queues_and_barrier( - 'execution_barrier_', []) - - global_step = tf.train.get_global_step() - with tf.device(self.global_step_device), tf.name_scope('inc_global_step'): - with tf.control_dependencies([main_fetch_group]): - fetches['inc_global_step'] = global_step.assign_add(1) - - if ((not self.single_session) and (not self.distributed_collective) and - self.job_name and self.params.cross_replica_sync): - # Block all replicas until all replicas are ready for next step. - fetches['sync_queues'] = self.add_sync_queues_and_barrier( - 'sync_queues_step_end_', [main_fetch_group]) - - # Skips the init ops for freezable local variables in forward_only mode so - # we can remove all the assign ops when converting variables to constants. - with tf.name_scope('local_variable_initialization'): - if self.forward_only_and_freeze: - local_var_init_op = tf.variables_initializer( - self._unfreezable_local_variables(tf.get_default_graph())) - else: - local_var_init_op = tf.local_variables_initializer() - table_init_ops = tf.tables_initializer() - - variable_manager_init_ops = [local_var_init_op] - if table_init_ops: - variable_manager_init_ops.extend([table_init_ops]) - if not self.forward_only_and_freeze: - with tf.control_dependencies([local_var_init_op]): - variable_manager_init_ops.extend(self.variable_mgr.get_post_init_ops()) - if ((not self.single_session) and (not self.distributed_collective) and - self.job_name and self.params.cross_replica_sync): - # Ensure all workers execute variable_manager_init_ops before they start - # executing the model. - variable_manager_init_ops.append( - self.add_sync_queues_and_barrier('init_ops_end_', - variable_manager_init_ops)) - local_var_init_op_group = tf.group(*variable_manager_init_ops, - name='local_var_init_op_group') - summary_op = tf.summary.merge_all() - - return GraphInfo( - input_producer_op=input_producer_op, - enqueue_ops=enqueue_ops, - fetches=fetches, - execution_barrier=execution_barrier, - global_step=global_step, - local_var_init_op_group=local_var_init_op_group, - summary_op=summary_op) - - def _benchmark_graph(self, graph_info, eval_graph_info): - """Benchmark the training graph. - - Args: - graph_info: the namedtuple returned by _build_graph() which - contains all necessary information to benchmark the graph, including - named tensors/ops list, fetches, etc. - eval_graph_info: Similar to graph_info but for the eval graph if - --eval_during_training_* is used. Otherwise, None. - Returns: - Dictionary containing training statistics (num_workers, num_steps, - average_wall_time, images_per_sec). - """ - log_fn('Initializing graph') - if self.params.variable_update == 'horovod': - import horovod.tensorflow as hvd # pylint: disable=g-import-not-at-top - # First worker will be 'chief' - it will write summaries and - # save checkpoints. - is_chief = hvd.rank() == 0 - else: - is_chief = (not self.job_name or self.task_index == 0) - - summary_writer = None - if (is_chief and self.params.summary_verbosity and self.params.train_dir and - self.params.save_summaries_steps > 0): - summary_writer = tf.summary.FileWriter(self.params.train_dir, - tf.get_default_graph()) - - # We want to start the benchmark timer right after a image_producer barrier - # and avoids undesired waiting times on barriers. - if ((self.num_warmup_batches + len(graph_info.enqueue_ops) - 1) % - self.batch_group_size) != 0: - self.num_warmup_batches = int( - math.ceil( - (self.num_warmup_batches + len(graph_info.enqueue_ops) - 1.0) / - (self.batch_group_size)) * self.batch_group_size - - len(graph_info.enqueue_ops) + 1) - log_fn('Round up warm up steps to %d to match batch_group_size' % - self.num_warmup_batches) - assert ((self.num_warmup_batches + len(graph_info.enqueue_ops) - 1) % - self.batch_group_size) == 0 - # We run the summaries in the same thread as the training operations by - # passing in None for summary_op to avoid a summary_thread being started. - # Running summaries and training operations in parallel could run out of - # GPU memory. - if is_chief and not self.forward_only_and_freeze: - saver = tf.train.Saver( - self.variable_mgr.savable_variables(), - save_relative_paths=True, - max_to_keep=self.params.max_ckpts_to_keep) - else: - saver = None - ready_for_local_init_op = None - if self.job_name and not (self.single_session or - self.distributed_collective): - # In distributed mode, we don't want to run local_var_init_op_group until - # the global variables are initialized, because local_var_init_op_group - # may use global variables (such as in distributed replicated mode). We - # don't set this in non-distributed mode, because in non-distributed mode, - # local_var_init_op_group may itself initialize global variables (such as - # in replicated mode). - ready_for_local_init_op = tf.report_uninitialized_variables( - tf.global_variables()) - if self.params.variable_update == 'horovod': - import horovod.tensorflow as hvd # pylint: disable=g-import-not-at-top - bcast_global_variables_op = hvd.broadcast_global_variables(0) - else: - bcast_global_variables_op = None - - if self.params.variable_update == 'collective_all_reduce': - # It doesn't matter what this collective_graph_key value is, - # so long as it's > 0 and the same at every worker. - init_run_options = tf.RunOptions() - init_run_options.experimental.collective_graph_key = 6 - else: - init_run_options = tf.RunOptions() - local_var_init_ops = [graph_info.local_var_init_op_group] - if eval_graph_info: - # `eval_graph_info.local_var_init_op_group` also includes some of the - # training initializer ops, since it's difficult to filter them out. - # Rerunning the training initializer ops is OK, but we add a control - # dependency since running two sets of training initializer ops at the - # same time can cause race conditions. - with tf.control_dependencies(local_var_init_ops): - local_var_init_ops.append(eval_graph_info.local_var_init_op_group) - sv = tf.train.Supervisor( - # For the purpose of Supervisor, all Horovod workers are 'chiefs', - # since we want session to be initialized symmetrically on all the - # workers. - is_chief=is_chief or (self.params.variable_update == 'horovod' - or self.distributed_collective), - # Log dir should be unset on non-chief workers to prevent Horovod - # workers from corrupting each other's checkpoints. - logdir=self.params.train_dir if is_chief else None, - ready_for_local_init_op=ready_for_local_init_op, - local_init_op=local_var_init_ops, - saver=saver, - global_step=graph_info.global_step, - summary_op=None, - save_model_secs=self.params.save_model_secs, - summary_writer=summary_writer, - local_init_run_options=init_run_options) - - profiler = tf.profiler.Profiler() if self.params.tfprof_file else None - if self.graph_file is not None: - path, filename = os.path.split(self.graph_file) - as_text = filename.endswith('txt') - log_fn('Writing GraphDef as %s to %s' % ( # pyformat break - 'text' if as_text else 'binary', self.graph_file)) - tf.train.write_graph(tf.get_default_graph().as_graph_def(add_shapes=True), - path, filename, as_text) - - start_standard_services = ( - self.params.train_dir or - self.dataset.queue_runner_required()) - target = self.cluster_manager.get_target() if self.cluster_manager else '' - with sv.managed_session( - master=target, - config=create_config_proto(self.params), - start_standard_services=start_standard_services) as sess: - # Anything that can potentially raise an OutOfRangeError with 'sess' MUST - # be under this try block. The managed_session() context manager silently - # ignores OutOfRangeError, so we must catch them and wrap them with - # a different exception type so that they can be propagated up to the - # caller. - try: - stats = self.benchmark_with_session( - sess, sv, graph_info, eval_graph_info, bcast_global_variables_op, - is_chief, summary_writer, profiler) - except tf.errors.OutOfRangeError: - raise RuntimeError( - 'Received OutOfRangeError. Wrapping in Runtime error to avoid ' - 'Supervisor from suppressing the error. Original OutOfRangeError ' - 'with traceback:\n' + traceback.format_exc()) - - sv.stop() - if profiler: - generate_tfprof_profile(profiler, self.params.tfprof_file) - return stats - - def benchmark_with_session(self, sess, supervisor, graph_info, - eval_graph_info, bcast_global_variables_op, - is_chief, summary_writer, profiler): - """Benchmarks the graph with the given session. - - Args: - sess: The session to benchmark the graph with - supervisor: The Supervisor that created the session. - graph_info: the namedtuple returned by _build_graph() which - contains all necessary information to benchmark the graph, including - named tensors/ops list, fetches, etc. - eval_graph_info: Similar to graph_info but for the eval graph if - --eval_during_training_every_n_steps is used. Otherwise, None. - bcast_global_variables_op: If Horovod is used, the op to broadcast the - global variables to all the processes. None if Horovod is not used. - is_chief: True if this is the chief process. - summary_writer: The SummaryWriter used to write summaries, or None if - summaries are not used. - profiler: The tf.profiler.Profiler, or None if tfprof is not used. - - Returns: - Dictionary containing training statistics (num_workers, num_steps, - average_wall_time, images_per_sec). - """ - if self.params.backbone_model_path is not None: - self.model.load_backbone_model(sess, self.params.backbone_model_path) - if bcast_global_variables_op: - sess.run(bcast_global_variables_op) - image_producer = None - if graph_info.input_producer_op is not None: - image_producer = cnn_util.ImageProducer( - sess, graph_info.input_producer_op, self.batch_group_size, - self.params.use_python32_barrier) - image_producer.start() - if graph_info.enqueue_ops: - for i in xrange(len(graph_info.enqueue_ops)): - sess.run(graph_info.enqueue_ops[:(i + 1)]) - if image_producer is not None: - image_producer.notify_image_consumption() - self.init_global_step, = sess.run([graph_info.global_step]) - if self.job_name and not self.params.cross_replica_sync: - # TODO(zhengxq): Do we need to use a global step watcher at all? - global_step_watcher = GlobalStepWatcher( - sess, graph_info.global_step, - self.num_workers * self.num_warmup_batches + - self.init_global_step, - self.num_workers * (self.num_warmup_batches + self.num_batches) - 1) - global_step_watcher.start() - else: - global_step_watcher = None - eval_image_producer = None - if eval_graph_info: - # We pass local_var_init_op_group=None because the Supervisor already - # initialized local variables above. We need to have the Supervisor - # initialize the local variables, because otherwise it throws an error - # complaining that not all variables were initialized. - eval_image_producer = self._initialize_eval_graph( - eval_graph_info.enqueue_ops, eval_graph_info.input_producer_op, - local_var_init_op_group=None, sess=sess) - step_train_times = [] - log_fn('Running warm up') - local_step = -1 * self.num_warmup_batches - if self.single_session: - # In single session mode, each step, the global_step is incremented by - # 1. In non-single session mode, each step, the global_step is - # incremented once per worker. This means we need to divide - # init_global_step by num_workers only in non-single session mode. - end_local_step = self.num_batches - self.init_global_step - else: - end_local_step = self.num_batches - (self.init_global_step // - self.num_workers) - if not global_step_watcher: - # In cross-replica sync mode, all workers must run the same number of - # local steps, or else the workers running the extra step will block. - done_fn = lambda: local_step >= end_local_step - else: - done_fn = global_step_watcher.done - if self.params.debugger is not None: - if self.params.debugger == 'cli': - log_fn('The CLI TensorFlow debugger will be used.') - sess = tf_debug.LocalCLIDebugWrapperSession(sess) - else: - log_fn('The TensorBoard debugger plugin will be used.') - sess = tf_debug.TensorBoardDebugWrapperSession(sess, - self.params.debugger) - mlperf.logger.log(key=mlperf.tags.TRAIN_LOOP) - skip_final_eval = False - accuracy_at_1 = None - accuracy_at_5 = None - last_eval_step = local_step - loop_start_time = time.perf_counter() - last_average_loss = None - while not done_fn(): - if local_step == 0: - log_fn('Done warm up') - if graph_info.execution_barrier: - log_fn('Waiting for other replicas to finish warm up') - sess.run([graph_info.execution_barrier]) - - # TODO(laigd): rename 'Img' to maybe 'Input'. - header_str = ('Step\tImg/sec\t' + - self.params.loss_type_to_report.replace('/', ' ')) - if self.params.print_training_accuracy or self.params.forward_only: - # TODO(laigd): use the actual accuracy op names of the model. - header_str += '\ttop_1_accuracy\ttop_5_accuracy' - log_fn(header_str) - assert len(step_train_times) == self.num_warmup_batches - # reset times to ignore warm up batch - step_train_times = [] - loop_start_time = time.perf_counter() - if (summary_writer and - (local_step + 1) % self.params.save_summaries_steps == 0): - fetch_summary = graph_info.summary_op - else: - fetch_summary = None - collective_graph_key = 7 if ( - self.params.variable_update == 'collective_all_reduce') else 0 - (summary_str, last_average_loss) = benchmark_one_step( - sess, graph_info.fetches, local_step, - self.batch_size * (self.num_workers - if self.single_session else 1), step_train_times, - self.trace_filename, self.params.partitioned_graph_file_prefix, - profiler, image_producer, self.params, fetch_summary, - benchmark_logger=self.benchmark_logger, - collective_graph_key=collective_graph_key, - should_output_files=(self.params.variable_update != 'horovod' or - is_chief)) - if summary_str is not None and is_chief: - supervisor.summary_computed(sess, summary_str) - local_step += 1 - if (self.params.save_model_steps and - local_step % self.params.save_model_steps == 0 and - local_step > 0 and - is_chief): - supervisor.saver.save(sess, supervisor.save_path, - supervisor.global_step) - if (eval_graph_info and local_step > 0 and not done_fn() and - self._should_eval_during_training(local_step)): - python_global_step = sess.run(graph_info.global_step) - num_steps_since_last_eval = local_step - last_eval_step - # The INPUT_SIZE tag value might not match the - # PREPROC_NUM_TRAIN_EXAMPLES tag value, because the number of examples - # run, which is INPUT_SIZE, is rounded up to the nearest multiple of - # self.batch_size. - mlperf.logger.log( - key=mlperf.tags.INPUT_SIZE, - value=num_steps_since_last_eval * self.batch_size) - log_fn('Running evaluation at global_step {}'.format( - python_global_step)) - accuracy_at_1, accuracy_at_5 = self._eval_once( - sess, summary_writer, eval_graph_info.fetches, - eval_graph_info.summary_op, eval_image_producer, - python_global_step) - last_eval_step = local_step - if (self.params.stop_at_top_1_accuracy and - accuracy_at_1 >= self.params.stop_at_top_1_accuracy): - log_fn('Stopping, as eval accuracy at least %s was reached' % - self.params.stop_at_top_1_accuracy) - skip_final_eval = True - break - else: - log_fn('Resuming training') - if eval_graph_info and self.model.reached_target(): - log_fn('Stopping, as the model indicates its custom goal was reached') - skip_final_eval = True - break - loop_end_time = time.perf_counter() - # Waits for the global step to be done, regardless of done_fn. - if global_step_watcher: - while not global_step_watcher.done(): - time.sleep(.25) - if not global_step_watcher: - elapsed_time = loop_end_time - loop_start_time - average_wall_time = elapsed_time / local_step if local_step > 0 else 0 - images_per_sec = (self.num_workers * local_step * self.batch_size / - elapsed_time) - num_steps = local_step * self.num_workers - else: - # NOTE: Each worker independently increases the global step. So, - # num_steps will be the sum of the local_steps from each worker. - num_steps = global_step_watcher.num_steps() - elapsed_time = global_step_watcher.elapsed_time() - average_wall_time = (elapsed_time * self.num_workers / num_steps - if num_steps > 0 else 0) - images_per_sec = num_steps * self.batch_size / elapsed_time - - # We skip printing images/sec if --eval_during_training_* is specified, - # because we are both processing training and evaluation images, so a - # singular "images/sec" value is meaningless. - if self.mode != constants.BenchmarkMode.TRAIN_AND_EVAL: - log_fn('-' * 64) - # TODO(laigd): rename 'images' to maybe 'inputs'. - log_fn('total images/sec: %.2f' % images_per_sec) - log_fn('-' * 64) - else: - log_fn('Done with training') - num_steps_since_last_eval = local_step - last_eval_step - mlperf.logger.log( - key=mlperf.tags.INPUT_SIZE, - value=num_steps_since_last_eval * self.batch_size) - python_global_step = sess.run(graph_info.global_step) - if eval_graph_info and not skip_final_eval: - log_fn('Running final evaluation at global_step {}'.format( - python_global_step)) - accuracy_at_1, accuracy_at_5 = self._eval_once( - sess, summary_writer, eval_graph_info.fetches, - eval_graph_info.summary_op, eval_image_producer, python_global_step) - num_epochs_ran = (python_global_step * self.batch_size / - self.dataset.num_examples_per_epoch('train')) - mlperf.logger.log_train_epochs(num_epochs_ran) - if image_producer is not None: - image_producer.done() - if eval_image_producer is not None: - eval_image_producer.done() - if is_chief: - if self.benchmark_logger: - self.benchmark_logger.log_metric( - 'average_examples_per_sec', images_per_sec, global_step=num_steps) - - # Save the model checkpoint. - if self.params.train_dir is not None and is_chief: - checkpoint_path = os.path.join(self.params.train_dir, 'model.ckpt') - if not gfile.Exists(self.params.train_dir): - gfile.MakeDirs(self.params.train_dir) - supervisor.saver.save(sess, checkpoint_path, graph_info.global_step) - if graph_info.execution_barrier: - # Wait for other workers to reach the end, so this worker doesn't - # go away underneath them. - sess.run([graph_info.execution_barrier]) - stats = { - 'num_workers': self.num_workers, - 'num_steps': num_steps, - 'average_wall_time': average_wall_time, - 'images_per_sec': images_per_sec - } - if last_average_loss is not None: - stats['last_average_loss'] = last_average_loss - if accuracy_at_1 is not None: - stats['top_1_accuracy'] = accuracy_at_1 - if accuracy_at_5 is not None: - stats['top_5_accuracy'] = accuracy_at_5 - - success = bool(self.model.reached_target() or - (accuracy_at_1 and self.params.stop_at_top_1_accuracy and - accuracy_at_1 >= self.params.stop_at_top_1_accuracy)) - mlperf.logger.log(key=mlperf.tags.RUN_STOP, value={'success': success}) - mlperf.logger.log(key=mlperf.tags.RUN_FINAL) - return stats - - def _should_eval_during_training(self, step): - """Return True iff should run eval during training at current step.""" - - assert self.mode == constants.BenchmarkMode.TRAIN_AND_EVAL - - if self.params.eval_during_training_every_n_steps: - return step % self.params.eval_during_training_every_n_steps == 0 - - # All other --eval_during_training_* flags are converted to step numbers - # at which the model should run evaluation during training. - return step in self.eval_during_training_at_specified_steps - - def _preprocess_graph(self, graph, graph_info): - """Preprocess the graph before executing. - - Depending on the params, it runs various preprocessing on the graph, - including freezing, TensorRT conversion, etc. - - Args: - graph: the graph to preprocess. - graph_info: the namedtuple returned by _build_graph() which - contains all necessary information to benchmark the graph, including - named tensors/ops list, fetches, etc. - - Returns: - The updated graph and graph_info with the ops/tensors/fetches updated - according to the imported graph. - """ - assert isinstance(graph_info.fetches, dict) - assert isinstance(graph_info.global_step, tf.Variable) - if not self.forward_only_and_freeze: - return (graph, graph_info) - - # Get the names of the ops that need to keep during conversion. - flattened_op_names = list( - set([ - v.name.split(':')[0] - for v in nest.flatten(graph_info) - if v is not None - ])) - # Get variables that we don't want to freeze. - # Only keep unfreezable variables in forward_only_and_freeze mode. - # TODO(laigd): consider making global_step a constant. - variables_to_keep = {graph_info.global_step: tf.GraphKeys.GLOBAL_VARIABLES} - variables_to_keep.update({ - local_variable: tf.GraphKeys.LOCAL_VARIABLES - for local_variable in self._unfreezable_local_variables(graph) - }) - - variable_initializers = [ - variable.initializer.name for variable in variables_to_keep] - output_node_names = ( - flattened_op_names + - # Add variable initializer and read ops to the output list, so - # convert_variables_to_constants() will keep them. - variable_initializers + - [variable.value().op.name for variable in variables_to_keep]) - graphdef = graph.as_graph_def(add_shapes=True) - - # Freeze the graph. - with graph.as_default(): - with tf.Session(config=create_config_proto(self.params)) as sess: - sess.run(tf.global_variables_initializer()) - sess.run(tf.local_variables_initializer()) - graphdef = graph_util.convert_variables_to_constants( - sess, - graphdef, - output_node_names, - variable_names_blacklist=[ - variable.op.name for variable in variables_to_keep - ]) - - # Run TensorRT conversion. - if self.params.trt_mode: - # Import here instead of at top, because this will crash if TensorRT is - # not installed - from tensorflow.python.compiler.tensorrt import trt_convert # pylint: disable=g-import-not-at-top - # Avoid TF-TRT bridge from touching all variable initializer ops and their - # dependencies, since they can directly be fetched by sess.run()s that - # initialize the variables. - # pylint: disable=protected-access - name_to_input_name, _, _ = graph_util_impl._extract_graph_summary( - graphdef) - initializer_subgraph_ops = graph_util_impl._bfs_for_reachable_nodes( - variable_initializers, name_to_input_name) - # pylint: enable=protected-access - - graphdef = trt_convert.create_inference_graph( - graphdef, - outputs=output_node_names + list(initializer_subgraph_ops), - max_batch_size=self.model.get_batch_size(), - max_workspace_size_bytes=self.params.trt_max_workspace_size_bytes, - precision_mode=self.params.trt_mode) - - # Creates a new graph as the default and import the converted graph back. - updated_graph = tf.Graph() - - def _get_tensors_or_ops(inputs): - """Gets the updated tensors or ops from 'updated_graph'.""" - - def _get_fn(element): - if element is None: - return None - if ':' in element.name: - return updated_graph.get_tensor_by_name(element.name) - return updated_graph.get_operation_by_name(element.name) - - if isinstance(inputs, (list, dict, tuple)): - return nest.map_structure(_get_fn, inputs) - else: - return _get_fn(inputs) - - with updated_graph.as_default(): - importer.import_graph_def(graph_def=graphdef, name='') - - # Update the variables - for variable in variables_to_keep: - updated_variable = tf.Variable.from_proto(variable.to_proto()) - tf.add_to_collection(variables_to_keep[variable], updated_variable) - if variable is graph_info.global_step: - updated_global_step = updated_variable - - updated_graph_info = GraphInfo( - input_producer_op=_get_tensors_or_ops(graph_info.input_producer_op), - enqueue_ops=_get_tensors_or_ops(graph_info.enqueue_ops), - execution_barrier=_get_tensors_or_ops(graph_info.execution_barrier), - local_var_init_op_group=_get_tensors_or_ops( - graph_info.local_var_init_op_group), - fetches=_get_tensors_or_ops(graph_info.fetches), - global_step=updated_global_step, - summary_op=None) - return (updated_graph, updated_graph_info) - - def _build_input_processing(self, shift_ratio=0): - """"Build the image (pre)processing portion of the model graph. - - Args: - shift_ratio: shift_ratio for data_flow_ops.RecordInput. - - Returns: - An InputProcessingInfo containing all the input sources to the model. - """ - input_processing_info = InputProcessingInfo( - input_producer_op=None, - input_producer_stages=None, - multi_device_iterator_input=None) - - mlperf.logger.log(key=mlperf.tags.INPUT_ORDER) - if not self._doing_eval: - mlperf.logger.log(key=mlperf.tags.INPUT_BATCH_SIZE, value=self.batch_size) - - # If using synthetic gpu inputs, do nothing on the cpu side. - if self.dataset.use_synthetic_gpu_inputs(): - assert not self.datasets_use_prefetch - return input_processing_info - - if self._doing_eval: - input_preprocessor = self.eval_input_preprocessor - mlperf.logger.log(key=mlperf.tags.PREPROC_NUM_EVAL_EXAMPLES, - value=self.dataset.num_examples_per_epoch('validation')) - else: - input_preprocessor = self.input_preprocessor - mlperf.logger.log(key=mlperf.tags.PREPROC_NUM_TRAIN_EXAMPLES, - value=self.dataset.num_examples_per_epoch('train')) - - # Use prefetching mechanism provided by dataset input pipeline. - if self.datasets_use_prefetch: - multi_device_iterator = ( - input_preprocessor.build_multi_device_iterator( - self.batch_size, len(self.devices), self.cpu_device, self.params, - self.raw_devices, self.dataset, self._doing_eval)) - return input_processing_info._replace( - multi_device_iterator_input=multi_device_iterator.get_next()) - - # Not using dataset prefetching. Use a staging area to mimic the prefetching - # behavior instead. - with tf.device(self.cpu_device): - if self._doing_eval: - subset = 'validation' - else: - subset = 'train' - input_list = input_preprocessor.minibatch( - self.dataset, - subset=subset, - params=self.params, - shift_ratio=shift_ratio) - - input_producer_op = [] - input_producer_stages = [] - for device_num in range(len(self.devices)): - staging_area = data_flow_ops.StagingArea( - [parts[0].dtype for parts in input_list], - shapes=[parts[0].get_shape() for parts in input_list], - shared_name='input_producer_staging_area_%d_eval_%s' % - (device_num, self._doing_eval)) - input_producer_stages.append(staging_area) - for group_index in xrange(self.batch_group_size): - batch_index = group_index + device_num * self.batch_group_size - put_op = staging_area.put( - [parts[batch_index] for parts in input_list]) - input_producer_op.append(put_op) - assert input_producer_op - - return input_processing_info._replace( - input_producer_op=input_producer_op, - input_producer_stages=input_producer_stages) - - def _maybe_initialize_fp16(self): - """Initialize fp16 settings.""" - if self.params.use_fp16 and not self._doing_eval: - init_loss_scale_val = float(self.params.fp16_loss_scale or - self.model.get_fp16_loss_scale()) - self.loss_scale = None - self.loss_scale_normal_steps = None - if self.enable_auto_loss_scale or init_loss_scale_val != 1: - self.loss_scale = tf.get_variable( - name='loss_scale', - initializer=init_loss_scale_val, - dtype=tf.float32, - trainable=False) - if self.enable_auto_loss_scale: - self.loss_scale_normal_steps = tf.get_variable( - name='loss_scale_normal_steps', initializer=0, trainable=False) - - def _build_model(self): - """Build the TensorFlow graph.""" - if self.datasets_use_prefetch: - assert not self.params.staged_vars - assert not self.variable_mgr.supports_staged_vars() - - # Adjust seed so different workers start read different input files. - if self.params.variable_update == 'horovod': - import horovod.tensorflow as hvd # pylint: disable=g-import-not-at-top - seed_adjustment = hvd.rank() - else: - seed_adjustment = 0 - mlperf.logger.log(key=mlperf.tags.RUN_SET_RANDOM_SEED, - value=self.params.tf_random_seed + seed_adjustment) - tf.set_random_seed(self.params.tf_random_seed + seed_adjustment) - mlperf.logger.log(key=mlperf.tags.RUN_SET_RANDOM_SEED, - value=4321 + seed_adjustment) - np.random.seed(4321 + seed_adjustment) - phase_train = not (self._doing_eval or self.params.forward_only) - - if self._doing_eval: - mode_string = 'evaluation' - else: - mode_string = 'training' - - log_fn('Generating {} model'.format(mode_string)) - losses = [] - device_grads = [] - all_logits = [] - all_accuracy_ops = {} - gpu_compute_stage_ops = [] - gpu_grad_stage_ops = [] - - with tf.device(self.global_step_device): - global_step = tf.train.get_or_create_global_step() - self._maybe_initialize_fp16() - - # Build the processing and model for the worker. - input_producer_op = None - with tf.name_scope('input_processing'): - input_processing_info = self._build_input_processing(shift_ratio=0) - if input_processing_info.input_producer_op is not None: - input_producer_op = tf.group(*input_processing_info.input_producer_op) - update_ops = None - staging_delta_ops = [] - - for device_num in range(len(self.devices)): - with tf.name_scope('tower_%i' % device_num) as name_scope, ( - self.variable_mgr.create_outer_variable_scope(device_num)): - results = self.add_forward_pass_and_gradients( - phase_train, device_num, device_num, input_processing_info, - gpu_compute_stage_ops, gpu_grad_stage_ops) - - if self.params.backbone_model_path: - self.model.add_backbone_saver() - - if phase_train: - losses.append(results['loss']) - device_grads.append(results['gradvars']) - else: - all_logits.append(results['logits']) - if not phase_train or self.params.print_training_accuracy: - for name, op in results.items(): - if name.startswith('accuracy:'): - key = name[9:] - if key not in all_accuracy_ops: - all_accuracy_ops[key] = [] - all_accuracy_ops[key].append(op) - - if device_num == 0: - # Retain the Batch Normalization updates operations only from the - # first tower. These operations update the moving mean and moving - # variance variables, which are updated (but not used) during - # training, and used during evaluation. The moving mean and variance - # approximate the true mean and variance across all images in the - # dataset. Therefore, in replicated mode, these moving averages would - # be almost identical for each tower, and so we only update and save - # the moving averages for one tower. In parameter server mode, all - # towers share a copy of the variables so we also only need to update - # and save the moving averages once. - update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, name_scope) - if self.datasets_use_prefetch: - assert not self.variable_mgr.staging_delta_ops - else: - staging_delta_ops = list(self.variable_mgr.staging_delta_ops) - - enqueue_ops = [] - if not self.datasets_use_prefetch: - if self.variable_mgr.supports_staged_vars(): - for staging_ops in self.variable_mgr.staging_vars_on_devices: - gpu_compute_stage_ops.extend( - [put_op for _, (put_op, _) in six.iteritems(staging_ops)]) - enqueue_ops.append(tf.group(*gpu_compute_stage_ops, - name='gpu_compute_stage_ops_group')) - if gpu_grad_stage_ops: - staging_delta_ops += gpu_grad_stage_ops - if staging_delta_ops: - enqueue_ops.append(tf.group(*(staging_delta_ops))) - - if (self.mode == constants.BenchmarkMode.TRAIN_AND_EVAL and - self.params.variable_update == 'replicated'): - # We need to get all the update ops instead of only those for the first - # tower. This is because during evaluation, each tower will read from its - # own tower's moving averages instead of the first tower's moving - # averages. - # TODO(reedwm): Have each tower read from the first tower's moving - # averages for a slight performance gain. - update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) - mlperf.logger.log(key=mlperf.tags.INPUT_BN_SPAN, - value=self.batch_size // len(self.raw_devices)) - - fetches = self._build_fetches(global_step, all_logits, losses, device_grads, - enqueue_ops, update_ops, all_accuracy_ops, - phase_train) - return (input_producer_op, enqueue_ops, fetches) - - def _build_fetches(self, global_step, all_logits, losses, device_grads, - enqueue_ops, update_ops, all_accuracy_ops, phase_train): - """Complete construction of model graph, populating the fetches map.""" - fetches = {} - if enqueue_ops: - fetches['enqueue_ops'] = enqueue_ops - for name, ops in all_accuracy_ops.items(): - # For fetches that starts with 'tensor:', keep dimension and skip reducing - # them to scalars. - if name.startswith(constants.UNREDUCED_ACCURACY_OP_PREFIX): - key = name[len(constants.UNREDUCED_ACCURACY_OP_PREFIX):] - fetches[key] = tf.concat(ops, 0) - else: - fetches[name] = ( - tf.reduce_sum(ops) / - (self.batch_size * - (self.num_workers if self.single_session else 1))) - if self.task_index == 0 and self.params.summary_verbosity >= 1: - tf.summary.scalar(name, fetches[name]) - - if not phase_train: - if self.params.forward_only: - fetches['all_logits'] = tf.concat(all_logits, 0) - return fetches - apply_gradient_devices, gradient_state = ( - self.variable_mgr.preprocess_device_grads(device_grads)) - - # TODO(reedwm): Greatly simplify the learning rate code. - if (self.params.variable_update == 'horovod' or - self.params.variable_update == 'collective_all_reduce'): - # Each worker independently increments global_step. - examples_per_step = self.batch_size * self.num_workers - else: - # global_step is shared by all workers, and so every iteration - # global_step is incremented by num_workers. - examples_per_step = self.batch_size - if self.params.compute_lr_on_cpu: - with tf.device(self.cpu_device): - learning_rate = get_learning_rate(self.params, global_step, - self.dataset.num_examples_per_epoch(), - self.model, examples_per_step) - - training_ops = [] - for d, device in enumerate(apply_gradient_devices): - with tf.device(device): - with tf.name_scope('average_loss'): - average_loss = tf.reduce_mean(losses) - with tf.name_scope('get_gradients_to_apply'): - avg_grads = self.variable_mgr.get_gradients_to_apply(d, - gradient_state) - - if not self.params.compute_lr_on_cpu: - # We compute the learning rate once for each device in - # `apply_gradient_devices`. - learning_rate = get_learning_rate( - self.params, global_step, self.dataset.num_examples_per_epoch(), - self.model, examples_per_step) - gradient_clip = self.params.gradient_clip - if gradient_clip is not None: - with tf.name_scope('clip_gradients'): - clipped_grads = [(tf.clip_by_value(grad, -gradient_clip, - +gradient_clip), var) - for grad, var in avg_grads] - else: - clipped_grads = avg_grads - - learning_rate = tf.identity(learning_rate, name='learning_rate_tensor') - opt = get_optimizer(self.params, learning_rate) - loss_scale_params = variable_mgr_util.AutoLossScaleParams( - enable_auto_loss_scale=self.enable_auto_loss_scale, - loss_scale=self.loss_scale, - loss_scale_normal_steps=self.loss_scale_normal_steps, - inc_loss_scale_every_n=self.params.fp16_inc_loss_scale_every_n, - is_chief=not self.job_name or self.task_index == 0) - - with tf.name_scope('append_apply_gradient_ops'): - self.variable_mgr.append_apply_gradients_ops( - gradient_state, opt, clipped_grads, training_ops, - loss_scale_params) - train_op = tf.group(*(training_ops + update_ops), name='train_ops_group') - - with tf.device(self.cpu_device): - if self.task_index == 0 and self.params.summary_verbosity >= 1: - tf.summary.scalar('learning_rate', learning_rate) - tf.summary.scalar(self.params.loss_type_to_report, average_loss) - if self.loss_scale is not None: - tf.summary.scalar('loss_scale', self.loss_scale) - if self.loss_scale_normal_steps: - tf.summary.scalar('loss_scale_normal_steps', - self.loss_scale_normal_steps) - - if self.params.summary_verbosity >= 2: - self.gradient_histogram_summary(avg_grads) - - if self.params.summary_verbosity >= 3: - for grad, var in avg_grads: - if grad is not None: - tf.summary.histogram(var.op.name + '/gradients', grad) - for var in tf.trainable_variables(): - tf.summary.histogram(var.op.name, var) - - fetches['train_op'] = train_op - fetches['average_loss'] = average_loss - return fetches - - def gradient_histogram_summary(self, avg_grads): - """Create histogram of log values of all non-zero gradients.""" - with tf.name_scope('log_gradients_summary'): - all_grads = [] - for grad, _ in avg_grads: - all_grads.append(tf.reshape(grad, [-1])) - grads = tf.abs(tf.concat(all_grads, 0)) - # exclude grads with zero values. - indices_for_non_zero_grads = tf.where(tf.not_equal(grads, 0)) - log_grads = tf.reshape( - tf.log(tf.gather(grads, indices_for_non_zero_grads)), [-1]) - tf.summary.histogram('log_gradients', log_grads) - - def _build_model_single_session(self): - """Build the TensorFlow graph for multiple replicas in a single_session. - - Returns: - input_producer_op: - enqueue_ops: - fetches: - - Raises: - ValueError: optimizer not recognized. - - Single session runs multiple model replicas as part of one large - distributed graph, whose global execution is always step-synchronized. - """ - # verify assumptions - assert self.params.task_index == 0 - assert not self._doing_eval - assert not self.params.forward_only - assert not self.params.staged_vars - - tf.set_random_seed(self.params.tf_random_seed) - np.random.seed(4321) - phase_train = True - - log_fn('Generating training model') - losses = [] - device_grads = [] - all_logits = [] - all_accuracy_ops = {} - gpu_compute_stage_ops = [] - gpu_grad_stage_ops = [] - - with tf.device(self.global_step_device): - global_step = tf.train.get_or_create_global_step() - - update_ops = [] - global_input_producer_op = [] - - is_local = not self.job_name - if is_local: - assert self.num_workers == 1 - for task_num in range(self.num_workers): - # Reset the devices that self.variable_mgr knows about to those - # belonging to the next worker (task). - self.reset_devices_for_task(task_num, is_local) - # Build the per-worker image processing - with tf.name_scope('input_processing'): - input_processing_info = self._build_input_processing( - shift_ratio=(task_num / self.num_workers)) - if input_processing_info.input_producer_op is not None: - global_input_producer_op.extend(input_processing_info.input_producer_op) - # Build the per-worker model replica. - for rel_device_num in range(len(self.devices)): - abs_device_num = task_num * len(self.devices) + rel_device_num - with self.variable_mgr.create_outer_variable_scope( - abs_device_num), tf.name_scope( - 'task_%i_tower_%i' % (task_num, rel_device_num)) as name_scope: - task_results = self.add_forward_pass_and_gradients( - phase_train, rel_device_num, abs_device_num, - input_processing_info, gpu_compute_stage_ops, gpu_grad_stage_ops) - - if self.params.backbone_model_path: - self.model.add_backbone_saver() - - if phase_train: - losses.append(task_results['loss']) - device_grads.append(task_results['gradvars']) - else: - all_logits.append(task_results['logits']) - if not phase_train or self.params.print_training_accuracy: - for name, op in task_results.items(): - if name.startswith('accuracy:'): - key = name[9:] - if key not in all_accuracy_ops: - all_accuracy_ops[key] = [] - all_accuracy_ops[key].append(op) - - if rel_device_num == 0: - # Retain the Batch Normalization updates operations only - # from the first tower. These operations update the moving - # mean and moving variance variables, which are updated - # (but not used) during training, and used during - # evaluation. The moving mean and variance approximate the - # true mean and variance across all images in the - # dataset. Therefore, in replicated mode, these moving - # averages would be almost identical for each tower, and - # so we only update and save the moving averages for one - # tower. In parameter server mode, all towers share a copy - # of the variables so we also only need to update and save - # the moving averages once. - update_ops.extend( - tf.get_collection(tf.GraphKeys.UPDATE_OPS, name_scope)) - assert not self.variable_mgr.staging_delta_ops - - enqueue_ops = [] - if gpu_compute_stage_ops: - enqueue_ops.append(tf.group(*gpu_compute_stage_ops, - name='gpu_compute_stage_ops')) - assert not self.variable_mgr.supports_staged_vars() - assert not gpu_grad_stage_ops - - fetches = self._build_fetches(global_step, all_logits, losses, device_grads, - enqueue_ops, update_ops, all_accuracy_ops, - phase_train) - if global_input_producer_op: - global_input_producer_op = tf.group(*global_input_producer_op) - else: - global_input_producer_op = None - return (global_input_producer_op, enqueue_ops, fetches) - - def add_forward_pass_and_gradients(self, - phase_train, - rel_device_num, - abs_device_num, - input_processing_info, - gpu_compute_stage_ops, - gpu_grad_stage_ops): - """Add ops for forward-pass and gradient computations.""" - nclass = self.dataset.num_classes - if self.datasets_use_prefetch: - assert input_processing_info.multi_device_iterator_input, ( - 'multi_device_iterator_input cannot be None if ' - 'datasets_use_prefetch=True') - input_list = ( - input_processing_info.multi_device_iterator_input[rel_device_num]) - else: - if not self.dataset.use_synthetic_gpu_inputs(): - input_producer_stage = input_processing_info.input_producer_stages[ - rel_device_num] - with tf.device(self.cpu_device): - host_input_list = input_producer_stage.get() - with tf.device(self.raw_devices[rel_device_num]): - gpu_compute_stage = data_flow_ops.StagingArea( - [inp.dtype for inp in host_input_list], - shapes=[inp.get_shape() for inp in host_input_list]) - # The CPU-to-GPU copy is triggered here. - gpu_compute_stage_op = gpu_compute_stage.put(host_input_list) - input_list = gpu_compute_stage.get() - gpu_compute_stage_ops.append(gpu_compute_stage_op) - else: - with tf.device(self.raw_devices[rel_device_num]): - # Minor hack to avoid H2D copy when using synthetic data - input_list = self.model.get_synthetic_inputs( - BenchmarkCNN.GPU_CACHED_INPUT_VARIABLE_NAME, nclass) - - # Labels reshaping happens all on gpu:0. Reshaping synthetic labels on - # multiple devices slows down XLA computation for an unknown reason. - # TODO(b/116875203): Find/address root cause of XLA slow down. - labels_device_placement_hack = ( - self.dataset.use_synthetic_gpu_inputs() and self.params.xla_compile) - - def device_aware_reshape(tensor, shape): - device = self.devices[rel_device_num] - # Labels are int32, place reshapes on gpu:0 (no device placement) when the - # hack is enabled. - if labels_device_placement_hack and tensor.dtype == tf.int32: - device = '' - with tf.device(device): - return tf.reshape(tensor, shape=shape) - - subset = 'validation' if self._doing_eval else 'train' - input_shapes = self.model.get_input_shapes(subset) - input_list = [ - device_aware_reshape(input_list[i], shape=input_shapes[i]) - for i in range(len(input_list)) - ] - - def forward_pass_and_gradients(): - """Builds forward pass and gradient computation network. - - When phase_train=True and print_training_accuracy=False: - return [loss] + grads - - When phase_train=True and print_training_accuracy=True: - return [logits, loss] + grads - - When phase_train=False, - return [logits] - - Its output can always be unpacked by - - ``` - outputs = forward_pass_and_gradients() - logits, loss, grads = unpack_forward_pass_and_gradients_output(outputs) - ``` - - Returns: - outputs: A list of tensors depending on different modes. - """ - - build_network_result = self.model.build_network( - input_list, phase_train, nclass) - logits = build_network_result.logits - - if not phase_train: - return [logits] - - base_loss = self.model.loss_function(input_list, build_network_result) - params = self.variable_mgr.trainable_variables_on_device( - rel_device_num, abs_device_num) - l2_loss = None - total_loss = base_loss - with tf.name_scope('l2_loss'): - fp32_params = params - if self.model.data_type == tf.float16 and self.params.fp16_vars: - # fp16 reductions are very slow on GPUs, so cast to fp32 before - # calling tf.nn.l2_loss and tf.add_n. - # TODO(b/36217816): Once the bug is fixed, investigate if we should do - # this reduction in fp16. - fp32_params = (tf.cast(p, tf.float32) for p in params) - filtered_params = self.model.filter_l2_loss_vars(fp32_params) - if rel_device_num == len(self.devices) - 1: - # We compute the L2 loss for only one device instead of all of them, - # because the L2 loss for each device is the same. To adjust for this, - # we multiply the L2 loss by the number of devices. We choose the - # last device because for some reason, on a Volta DGX1, the first four - # GPUs take slightly longer to complete a step than the last four. - # TODO(reedwm): Shard the L2 loss computations across GPUs. - if self.params.single_l2_loss_op: - # TODO(reedwm): If faster, create a fused op that does the L2 loss - # on multiple tensors, and use that instead of concatenating - # tensors. - reshaped_params = [tf.reshape(p, (-1,)) for p in filtered_params] - l2_loss = tf.nn.l2_loss(tf.concat(reshaped_params, axis=0)) - else: - l2_loss = tf.add_n([tf.nn.l2_loss(v) for v in filtered_params]) - weight_decay = self.params.weight_decay - mlperf.logger.log(key=mlperf.tags.OPT_WEIGHT_DECAY, value=weight_decay) - if (weight_decay is not None and weight_decay != 0. and - l2_loss is not None): - mlperf.logger.log(key=mlperf.tags.MODEL_L2_REGULARIZATION, - value=weight_decay) - total_loss += len(self.devices) * weight_decay * l2_loss - - aggmeth = tf.AggregationMethod.DEFAULT - scaled_loss = (total_loss if self.loss_scale is None - else total_loss * self.loss_scale) - grads = tf.gradients(scaled_loss, params, aggregation_method=aggmeth) - if self.params.sparse_to_dense_grads: - # Passing a sparse gradient to convert_to_tensor turns it into a dense - # gradient. A sparse gradient is an instance of tf.IndexedSlices. - # convert_to_tensor does not modify dense tensors. - grads = [tf.convert_to_tensor(g) for g in grads] - if self.loss_scale is not None: - # TODO(reedwm): If automatic loss scaling is not used, we could avoid - # these multiplications by directly modifying the learning rate instead. - # If this is done, care must be taken to ensure that this scaling method - # is correct, as some optimizers square gradients and do other - # operations which might not be compatible with modifying both the - # gradients and the learning rate. - - grads = [ - grad * tf.cast(1. / self.loss_scale, grad.dtype) for grad in grads - ] - - if self.params.variable_update == 'horovod': - import horovod.tensorflow as hvd # pylint: disable=g-import-not-at-top - if self.params.horovod_device: - horovod_device = '/%s:0' % self.params.horovod_device - else: - horovod_device = '' - # All-reduce gradients using Horovod. - grads = [hvd.allreduce(grad, average=False, device_dense=horovod_device) - for grad in grads] - - if self.params.staged_vars: - grad_dtypes = [grad.dtype for grad in grads] - grad_shapes = [grad.shape for grad in grads] - grad_stage = data_flow_ops.StagingArea(grad_dtypes, grad_shapes) - grad_stage_op = grad_stage.put(grads) - # In general, this decouples the computation of the gradients and - # the updates of the weights. - # During the pipeline warm up, this runs enough training to produce - # the first set of gradients. - gpu_grad_stage_ops.append(grad_stage_op) - grads = grad_stage.get() - - if self.params.loss_type_to_report == 'total_loss': - loss = total_loss - else: - loss = base_loss - - if self.params.print_training_accuracy: - return [logits, loss] + grads - else: - return [loss] + grads - - def unpack_forward_pass_and_gradients_output(forward_pass_and_grad_outputs): - """Unpacks outputs from forward_pass_and_gradients. - - Args: - forward_pass_and_grad_outputs: Output from forward_pass_and_gradients. - - Returns: - logits: Unscaled probability distribution from forward pass. - If unavailable, None is returned. - loss: Loss function result from logits. - If unavailable, None is returned. - grads: Gradients for all trainable variables. - If unavailable, None is returned. - """ - logits = None - # logits is only fetched in non-train mode or when - # print_training_accuracy is set. - if not phase_train or self.params.print_training_accuracy: - logits = forward_pass_and_grad_outputs.pop(0) - - loss = ( - forward_pass_and_grad_outputs[0] - if forward_pass_and_grad_outputs else None) - grads = ( - forward_pass_and_grad_outputs[1:] - if forward_pass_and_grad_outputs else None) - - return logits, loss, grads - - def make_results(logits, loss, grads): - """Generate results based on logits, loss and grads.""" - results = {} # The return value - - if logits is not None: - results['logits'] = logits - accuracy_ops = self.model.accuracy_function(input_list, logits) - for name, op in accuracy_ops.items(): - results['accuracy:' + name] = op - - if loss is not None: - results['loss'] = loss - - if grads is not None: - param_refs = self.variable_mgr.trainable_variables_on_device( - rel_device_num, abs_device_num, writable=True) - results['gradvars'] = list(zip(grads, param_refs)) - - return results - - with tf.device(self.devices[rel_device_num]): - outputs = maybe_compile(forward_pass_and_gradients, self.params) - logits, loss, grads = unpack_forward_pass_and_gradients_output(outputs) - return make_results(logits, loss, grads) - - def get_input_preprocessor(self): - """Returns the image preprocessor to used, based on the model. - - Returns: - The image preprocessor, or None if synthetic data should be used. - """ - shift_ratio = 0 - if self.job_name: - # shift_ratio prevents multiple workers from processing the same batch - # during a step - shift_ratio = self.task_index / self.num_workers - - processor_class = self.dataset.get_input_preprocessor( - self.params.input_preprocessor) - assert processor_class - subset = 'validation' if self._doing_eval else 'train' - return processor_class( - self.batch_size * self.batch_group_size, - self.model.get_input_shapes(subset), - len(self.devices) * self.batch_group_size, - dtype=self.model.data_type, - train=(not self._doing_eval), - # TODO(laigd): refactor away image model specific parameters. - distortions=self.params.distortions, - resize_method=self.resize_method, - shift_ratio=shift_ratio, - summary_verbosity=self.params.summary_verbosity, - distort_color_in_yiq=self.params.distort_color_in_yiq, - fuse_decode_and_crop=self.params.fuse_decode_and_crop, - match_mlperf=self.params.ml_perf) - - def add_sync_queues_and_barrier(self, name_prefix, enqueue_after_list): - """Adds ops to enqueue on all worker queues. - - Args: - name_prefix: prefixed for the shared_name of ops. - enqueue_after_list: control dependency from ops. - - Returns: - An op that should be used as control dependency before starting next step. - """ - self.sync_queue_counter += 1 - with tf.device(self.sync_queue_devices[( - self.sync_queue_counter % len(self.sync_queue_devices))]): - sync_queues = [ - tf.FIFOQueue(self.num_workers, [tf.bool], shapes=[[]], - shared_name='%s%s' % (name_prefix, i)) - for i in range(self.num_workers)] - queue_ops = [] - # For each other worker, add an entry in a queue, signaling that it can - # finish this step. - token = tf.constant(False) - with tf.control_dependencies(enqueue_after_list): - for i, q in enumerate(sync_queues): - if i == self.task_index: - queue_ops.append(tf.no_op()) - else: - queue_ops.append(q.enqueue(token)) - - # Drain tokens off queue for this worker, one for each other worker. - queue_ops.append( - sync_queues[self.task_index].dequeue_many(len(sync_queues) - 1)) - - return tf.group(*queue_ops) - - -def _is_mkl_flag_absent(mkl_flag): - return not (absl_flags.FLAGS.is_parsed() and mkl_flag in absl_flags.FLAGS - and absl_flags.FLAGS[mkl_flag].present) - - -def _print_os_env_ignored_warning(mkl_flag, flag_default_val, os_env_var): - tf.logging.warn( - ('OS ENV variable %s=%s is ignored and script default: ' - '%s is used. Use --%s to override.') % - (os_env_var, os.environ[os_env_var], flag_default_val, mkl_flag)) - - -def set_default_param_values_and_env_vars(params): - """Sets up the default param values and environment variables .""" - if params.batchnorm_persistent: - os.environ['TF_USE_CUDNN_BATCHNORM_SPATIAL_PERSISTENT'] = '1' - else: - os.environ.pop('TF_USE_CUDNN_BATCHNORM_SPATIAL_PERSISTENT', None) - if params.winograd_nonfused: - os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1' - else: - os.environ.pop('TF_ENABLE_WINOGRAD_NONFUSED', None) - if params.autotune_threshold: - os.environ['TF_AUTOTUNE_THRESHOLD'] = str(params.autotune_threshold) - os.environ['TF_SYNC_ON_FINISH'] = str(int(params.sync_on_finish)) - argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - # Sets environment variables for MKL - # If OS ENV vars are overridden by script defaults, a warning msg is printed. - if params.mkl: - mkl_flags = ['kmp_blocktime', 'kmp_settings', 'kmp_affinity', - 'num_intra_threads'] - for mkl_flag in mkl_flags: - os_env_var = mkl_flag.upper() - if mkl_flag == 'num_intra_threads': - os_env_var = 'OMP_NUM_THREADS' - flag_val = str(getattr(params, mkl_flag)) - if _is_mkl_flag_absent(mkl_flag) and os_env_var in os.environ: - _print_os_env_ignored_warning(mkl_flag, flag_val, os_env_var) - os.environ[os_env_var] = flag_val - if mkl_flag == 'num_intra_threads' and not params.num_intra_threads: - os.environ.pop(os_env_var, None) - - # Sets GPU thread settings - if params.device.lower() == 'gpu': - params = params._replace(gpu_thread_mode=params.gpu_thread_mode.lower()) - if params.gpu_thread_mode not in ['global', 'gpu_shared', 'gpu_private']: - raise ValueError('Invalid gpu_thread_mode: %s' % params.gpu_thread_mode) - os.environ['TF_GPU_THREAD_MODE'] = params.gpu_thread_mode - - if params.per_gpu_thread_count and params.gpu_thread_mode == 'global': - raise ValueError( - 'Invalid per_gpu_thread_count with gpu_thread_mode=global: %s' % - params.per_gpu_thread_count) - # Default to two threads. One for the device compute and the other for - # memory copies. - per_gpu_thread_count = params.per_gpu_thread_count or 2 - total_gpu_thread_count = per_gpu_thread_count * params.num_gpus - - if params.gpu_thread_mode == 'gpu_private': - os.environ['TF_GPU_THREAD_COUNT'] = str(per_gpu_thread_count) - elif params.gpu_thread_mode == 'gpu_shared': - os.environ['TF_GPU_THREAD_COUNT'] = str(total_gpu_thread_count) - - cpu_count = multiprocessing.cpu_count() - if not params.num_inter_threads and params.gpu_thread_mode in [ - 'gpu_private', 'gpu_shared' - ]: - main_thread_count = max(cpu_count - total_gpu_thread_count, 1) - params = params._replace(num_inter_threads=main_thread_count) - - if (params.datasets_use_prefetch and - params.datasets_num_private_threads is None): - # From the total cpu thread count, subtract the total_gpu_thread_count, - # and then 2 threads per GPU device for event monitoring and sending / - # receiving tensors - num_monitoring_threads = 2 * params.num_gpus - num_private_threads = max( - cpu_count - total_gpu_thread_count - num_monitoring_threads, 1) - params = params._replace(datasets_num_private_threads=num_private_threads) - return params - - -def setup(params): - """Sets up the environment that BenchmarkCNN should run in. - - Args: - params: Params tuple, typically created by make_params or - make_params_from_flags. - - Returns: - A potentially modified params. - Raises: - ValueError: invalid parames combinations. - """ - # Set up environment variables before doing any other global initialization to - # make sure it uses the appropriate environment variables. - params = set_default_param_values_and_env_vars(params) - - # horovod needs to be initialized before create_config_proto() call since - # it will be used in config generation if enabled. - if params.variable_update == 'horovod': - import horovod.tensorflow as hvd # pylint: disable=g-import-not-at-top - hvd.init() - - platforms_util.initialize(params, create_config_proto(params)) - - if not params.job_name: - # Create a dummy session to initialize TF global variables using the input - # params. Otherwise, ListDevices function may create global devices using - # the default config instead of using the user provided config. - # - # TODO(hinsu): Find a way to achieve the same for distributed benchmark. It - # is not legal to create distributed session after local session. It is also - # not possible to create distributed session here as that results in - # multiple creation of ClusterManager and Server. - with tf.Session(config=create_config_proto(params)) as sess: - del sess - - return params - - -def maybe_compile(computation, params): - if params and params.xla_compile: - return tf.xla.experimental.compile(computation) - else: - return computation() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/benchmark_cnn_distributed_test.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/benchmark_cnn_distributed_test.py deleted file mode 100644 index 43dac487f90e1014f9429b12a89fa93ac5ef19be..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/benchmark_cnn_distributed_test.py +++ /dev/null @@ -1,493 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests running benchmark_cnn in distributed mode. - -This is done by spawning one process per task. Each process runs -benchmark_cnn_distributed_test_runner.py. - -The output for each process is written to disk and can be viewed to debug tests. -See get_test_output_dir() in platforms/default/util.py for more info. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from collections import namedtuple -import os -import subprocess -import time -import unittest - -from absl import flags as absl_flags -import portpicker -import six -import tensorflow.compat.v1 as tf -import flags -import test_util -from platforms import util as platforms_util - -FLAGS = absl_flags.FLAGS - - -def _convert_params_to_flags_list(params): - """Converts Params to a list of flags. Skips default-valued parameters. - - E.g., converts - benchmark_cnn.make_params(batch_size=32, model='resnet50') - to - ['--batch_size=32', '--model=resnet50'] - - Args: - params: Params for BenchmarkCNN. - Returns: - A list of flags. - """ - return [ - '--%s=%s' % (k, str(v)) for k, v in six.iteritems(params._asdict()) - if v != flags.param_specs[k].default_value - ] - - -# When outputting a process's output in the log, maximum number of characters -# to output. The log system does not allow us to output more than this in a -# single log message, but this limit is also useful to avoid the logs from -# becoming too large (the full process output is written to disk). -MAX_OUTPUT_CHARS = 15000 - - -# A process. name is a string identifying the process in logs. stdout and -# stderr are file objects of the process's stdout and stderr, respectively. -_ProcessInfo = namedtuple('_ProcessInfo', ['name', 'popen', 'stdout', 'stderr']) - - -def _create_task_process(job_name, task_index, args, env, output_dir): - """Creates a process for a single task for benchmark_cnn. - - Args: - job_name: 'worker' or 'ps' or ''. Empty string used for non-distributed - mode. - task_index: The index of the task within the cluster. - args: A list of arguments to pass to the task. This function additionally - sets --task_index and --job_name - env: The environment to use for the task. - output_dir: Where to place the output files, storing the task's stdout and - stderr. - Returns: - A _ProcessInfo namedtuple of the running process. The stdout and stderr - fields of this tuple must be closed by the caller once the process ends. - """ - args = args[:] - args += ['--task_index=%s' % task_index, '--job_name=%s' % job_name] - name_prefix = job_name or 'local' - process_name = '%s_%s' % (name_prefix, task_index) - tf.logging.info('Spawning %s process: %s' % (process_name, ' '.join(args))) - stdout_filename = os.path.join(output_dir, '%s_stdout.txt' % process_name) - stderr_filename = os.path.join(output_dir, '%s_stderr.txt' % process_name) - stdout_file = open(stdout_filename, 'w+') - stderr_file = open(stderr_filename, 'w+') - popen = subprocess.Popen( - args, stdout=stdout_file, stderr=stderr_file, env=env) - return _ProcessInfo(process_name, popen, stdout_file, stderr_file) - - -def _wait_for_processes(wait_processes, kill_processes): - """Waits until all `wait_processes` finish, then kills `kill_processes`. - - Fails an assert if a process in `wait_processes` finishes unsuccessfully. - The processes in `kill_processes` are assumed to never finish so they are - killed. - - Args: - wait_processes: A list of _ProcessInfo tuples. This function will wait - for each to finish. - kill_processes: A list of _ProcessInfo tuples. Each will be killed once - every process in `wait_processes` is finished. - Returns: - A list of strings, each which is a string of the stdout of a wait process. - """ - wait_process_stdouts = [None] * len(wait_processes) - finished_wait_processes = set() - while len(finished_wait_processes) < len(wait_processes): - for i, wait_process in enumerate(wait_processes): - if i in finished_wait_processes: - continue - ret_code = wait_process.popen.poll() - if ret_code is None: - continue - tf.logging.info('{} finished'.format(wait_process.name)) - wait_process.stdout.seek(0) - wait_process_stdouts[i] = wait_process.stdout.read() - tf.logging.info('stdout for {} (last {} chars): {}\n'.format( - wait_process.name, MAX_OUTPUT_CHARS, - wait_process_stdouts[i][-MAX_OUTPUT_CHARS:])) - wait_process.stderr.seek(0) - tf.logging.info('stderr for {} (last {} chars): {}\n'.format( - wait_process.name, MAX_OUTPUT_CHARS, - wait_process.stderr.read()[-MAX_OUTPUT_CHARS:])) - assert ret_code == 0, 'Process failed with return code %d' % ret_code - finished_wait_processes.add(i) - for kill_process in kill_processes: - ret_code = kill_process.popen.poll() - # kill processes should not end until we kill them. - assert ret_code is None, 'Process returned early with code %d' % ret_code - time.sleep(0.25) - tf.logging.info('All wait processes finished') - for i, kill_process in enumerate(kill_processes): - # Kill each kill process. - kill_process.popen.kill() - kill_process.popen.wait() - kill_process.stdout.seek(0) - tf.logging.info('stdout for {} (last {} chars): {}\n'.format( - kill_process.name, MAX_OUTPUT_CHARS, - kill_process.stdout.read()[-MAX_OUTPUT_CHARS:])) - kill_process.stderr.seek(0) - tf.logging.info('stderr for {} (last {} chars): {}\n'.format( - kill_process.name, MAX_OUTPUT_CHARS, - kill_process.stderr.read()[-MAX_OUTPUT_CHARS:])) - return wait_process_stdouts - - -def _spawn_benchmark_processes(output_dir_path, num_workers, num_ps, - num_controllers, params): - """Run training or evaluation in spawned processes. - - Runs locally if num_workers == 1, num_ps == 0, and num_controllers == 0, - otherwise runs in distributed mode. In either case, one process is spawned - per worker and ps. Waits for training/evaluation to finish before returning. - - Args: - output_dir_path: Relative path where stdout and stderr files will be - placed. - num_workers: Number of workers to spawn. - num_ps: Number of ps processes to spawn. - num_controllers: Number of controller processes to spawn (must be 0 or 1). - params: Params for BenchmarkCNN in each subprocess. - Returns: - A list output_list of outputs from all processes that output the - images/sec and accuracy. This process is the controller host in - distributed_all_reduce, and the workers otherwise. output_list[i] is a - list of lines from the ith worker's stdout. - """ - run_distributed = num_workers != 1 or num_ps != 0 or num_controllers != 0 - if params.variable_update == 'distributed_all_reduce': - assert num_controllers == 1 or not run_distributed - assert num_ps == 0 - else: - assert num_controllers == 0 - output_base_dir = platforms_util.get_test_output_dir() - output_dir = os.path.join(output_base_dir, output_dir_path) - os.makedirs(output_dir) - tf.logging.info('Outputs of processes will be outputted to: %s' % output_dir) - - args = platforms_util.get_command_to_run_python_module( - 'benchmark_cnn_distributed_test_runner') - args += _convert_params_to_flags_list(params) - if run_distributed: - worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)] - ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)] - controller_ports = [portpicker.pick_unused_port() - for _ in range(num_controllers)] - # The numerator is 0.7 instead of 1 to leave some memory for the Cuda - # runtime, etc. - gpu_memory_frac = 0.7 / num_workers - args += [ - '--gpu_memory_frac_for_testing=%f' % gpu_memory_frac, - '--worker_hosts=' + ','.join('localhost:%d' % p for p in worker_ports) - ] - if num_ps > 0: - ps_hosts_str = ','.join('localhost:%d' % p for p in ps_ports) - args.append('--ps_hosts=' + ps_hosts_str) - else: - controller_host_str = ','.join('localhost:%d' % p - for p in controller_ports) - args.append('--controller_host=' + controller_host_str) - env = os.environ.copy() - # Allow stdout to be viewed before the process ends. - env['PYTHONUNBUFFERED'] = '1' - - worker_processes = [] - ps_processes = [] - controller_processes = [] - try: - for i in range(num_workers): - job_name = 'worker' if run_distributed else '' - process = _create_task_process(job_name, i, args, env, output_dir) - worker_processes.append(process) - # Don't let ps or controller processes use the gpu. - env['CUDA_VISIBLE_DEVICES'] = '' - - for i in range(num_ps): - process = _create_task_process('ps', i, args, env, output_dir) - ps_processes.append(process) - for i in range(num_controllers): - process = _create_task_process('controller', i, args, env, output_dir) - controller_processes.append(process) - # If all distributed all reduce mode is being used, the controller process - # finishes and the worker processes block forever. Otherwise, the worker - # processes finish and the ps processes block forever. We set - # wait_processes and kill_processes accordingly. - if controller_processes: - wait_processes = controller_processes - kill_processes = worker_processes - else: - wait_processes = worker_processes - kill_processes = ps_processes - outputs = _wait_for_processes(wait_processes, kill_processes) - finally: - for process in worker_processes + ps_processes + controller_processes: - try: - process.popen.kill() - except OSError: - pass # It's OK (and expected) if the process already exited. - process.stdout.close() - process.stderr.close() - return [output.splitlines() for output in outputs] - - -# When this test class is run, a method will fail about 0.3% of the time with a -# gRPC error. It is not clear why this occurs. -# TODO(reedwm): Fix this test class. -class TfCnnBenchmarksDistributedTest(tf.test.TestCase): - """Tests running benchmark_cnn in distributed mode.""" - - # We cannot check for a GPU via tf.test.is_gpu_available() before the tests in - # this class because it allocates all the GPU memory which would cause the - # spawned processes to run out of GPU memory. - - def _test_distributed(self, - test_name, - num_workers, - num_ps, - params, - num_controllers=0, - check_output_values=False, - skip=None): - # TODO(reedwm): check_output_values should default to True and be enabled - # on every test. See the TODO in benchmark_cnn_test.py. - def run_fn(run_type, inner_params): - output_dir_path = os.path.join(test_name, run_type) - if run_type == 'Evaluation': - # Distributed evaluation is not supported, so we use a single process. - # We still must spawn another process, because if we evaluate in the - # current process, it would allocate the GPU memory causing future test - # methods to fail. - if inner_params.variable_update == 'distributed_replicated': - inner_params = inner_params._replace(variable_update='replicated') - return _spawn_benchmark_processes( - output_dir_path, num_workers=1, num_ps=0, num_controllers=0, - params=inner_params) - else: - return _spawn_benchmark_processes(output_dir_path, num_workers, num_ps, - num_controllers, inner_params) - - return test_util.train_and_eval(self, run_fn, params, - check_output_values=check_output_values, - skip=skip) - - def testParameterServer(self): - test_name = 'testParameterServer' - params = test_util.get_params(test_name) - self._test_distributed(test_name, 2, 2, params) - - def testParameterServerStaged(self): - test_name = 'testParameterServerStaged' - params = test_util.get_params(test_name)._replace(staged_vars=True) - self._test_distributed(test_name, 2, 2, params) - - def testReplicated(self): - test_name = 'testReplicated' - params = test_util.get_params(test_name)._replace( - variable_update='distributed_replicated') - self._test_distributed(test_name, 2, 2, params) - - def testAllReducePsgpu(self): - test_name = 'testAllReducePsgpu' - flags_dict = test_util.get_params(test_name)._replace( - variable_update='distributed_all_reduce', - all_reduce_spec='psgpu#4') - self._test_distributed(test_name, 2, 0, flags_dict, num_controllers=1) - - def testAllReducePscpuXring(self): - test_name = 'testAllReducePscpuXring' - flags_dict = test_util.get_params(test_name)._replace( - variable_update='distributed_all_reduce', - all_reduce_spec='pscpu:2k:xring') - self._test_distributed(test_name, 2, 0, flags_dict, num_controllers=1) - - def testForwardOnly(self): - test_name = 'testForwardOnly' - params = test_util.get_params(test_name)._replace(forward_only=True) - # Evaluation is not supported with --forward_only, so we set skip='eval'. - self._test_distributed(test_name, 2, 2, params, skip='eval') - - def testSingleWorkerAndPs(self): - test_name = 'testSingleWorkerAndPs' - params = test_util.get_params(test_name) - self._test_distributed(test_name, 1, 1, params) - - def testThreeWorkersAndPses(self): - test_name = 'testThreeWorkersAndPses' - params = test_util.get_params(test_name) - self._test_distributed(test_name, 3, 3, params) - - def testOneWorkerThreePses(self): - test_name = 'testOneWorkerThreePses' - params = test_util.get_params(test_name) - self._test_distributed(test_name, 1, 3, params) - - def testThreeWorkersOnePs(self): - test_name = 'testThreeWorkersOnePs' - params = test_util.get_params(test_name) - self._test_distributed(test_name, 3, 1, params) - - def testNoPrintTrainingAccuracy(self): - test_name = 'testNoPrintTrainingAccuracy' - params = test_util.get_params(test_name)._replace( - print_training_accuracy=False) - self._test_distributed(test_name, 2, 2, params) - - def testRmspropParameterServer(self): - test_name = 'testRmspropParameterServer' - params = test_util.get_params(test_name)._replace(optimizer='rmsprop') - self._test_distributed(test_name, 2, 2, params) - - def testMomentumReplicated(self): - test_name = 'testMomentumReplicated' - params = test_util.get_params(test_name)._replace( - optimizer='momentum', variable_update='distributed_replicated') - self._test_distributed(test_name, 2, 2, params) - - def testNoCrossReplicaSyncParameterServerStaged(self): - test_name = 'testNoCrossReplicaSyncParameterServerStaged' - params = test_util.get_params(test_name)._replace( - staged_vars=True, cross_replica_sync=False) - self._test_distributed(test_name, 2, 2, params) - - def testSingleGpu(self): - test_name = 'testSingleGpu' - params = test_util.get_params(test_name)._replace(num_gpus=1) - self._test_distributed(test_name, 2, 2, params) - - def testBatchGroupSize(self): - test_name = 'testBatchGroupSize' - params = test_util.get_params(test_name)._replace( - batch_group_size=4, num_batches=100, num_warmup_batches=5) - self._test_distributed(test_name, 2, 2, params) - - def testFp16WithFp32Vars(self): - test_name = 'testFp16WithFp32Vars' - params = test_util.get_params(test_name)._replace( - use_fp16=True, fp16_vars=False) - self._test_distributed(test_name, 2, 2, params) - - def testFp16WithFp16Vars(self): - test_name = 'testFp16WithFp16Vars' - params = test_util.get_params(test_name)._replace( - use_fp16=True, fp16_vars=True, fp16_loss_scale=1.) - self._test_distributed(test_name, 2, 2, params) - - def testFp16Replicated(self): - test_name = 'testFp16Replicated' - params = test_util.get_params(test_name)._replace( - use_fp16=True, variable_update='distributed_replicated') - self._test_distributed(test_name, 2, 2, params) - - @unittest.skip('b/147310862: Fails for unknown reason') - def testReplicatedRealData(self): - test_name = 'testReplicatedRealData' - imagenet_dir = os.path.join(platforms_util.get_test_data_dir(), - 'fake_tf_record_data') - params = test_util.get_params(test_name)._replace( - variable_update='distributed_replicated', - data_dir=imagenet_dir, - data_name='imagenet') - self._test_distributed(test_name, 2, 2, params) - - -class DistributedVariableUpdateTest(tf.test.TestCase): - """Tests that variables are updated correctly in distributed mode.""" - - def _test_variable_update(self, - test_name, - num_workers, - num_ps, - params, - num_controllers=0): - """Tests variables are updated correctly when the given params are used.""" - output_dir_path = os.path.join(test_name, 'variable_update') - logs = _spawn_benchmark_processes(output_dir_path, num_workers, num_ps, - num_controllers, params) - actual_losses = [] - for worker_logs in logs: - outputs = test_util.get_training_outputs_from_logs( - worker_logs, params.print_training_accuracy) - actual_losses.append([x.loss for x in outputs]) - - inputs = test_util.get_fake_var_update_inputs() - expected_losses = test_util.TestCNNModel().manually_compute_losses( - inputs, num_workers, params) - if params.variable_update == 'distributed_all_reduce': - # In distributed all reduce, each step, the controller outputs the average - # of the loss from each worker. So we modify expected losses accordingly. - # E.g, we change [[1, 2], [4, 5]] to [[2.5, 3.5]] - expected_losses = [[sum(losses) / num_workers - for losses in zip(*expected_losses)]] - rtol = 3e-2 if params.use_fp16 else 1e-5 - for worker_actual_losses, worker_expected_losses in zip(actual_losses, - expected_losses): - self.assertAllClose(worker_actual_losses[:len(worker_expected_losses)], - worker_expected_losses, rtol=rtol, atol=0.) - - def _test_variable_updates(self, test_name, params): - """Tests variables are updated correctly with various variable updates.""" - - # Unfortunately, distributed parameter server is non-deterministic with - # multiple workers, because one worker may write to a variable before - # another worker reads it. This probably does not harm training, but it - # does mean we cannot easily test that case. So, we use one worker. - self._test_variable_update( - test_name + '_ps', num_workers=1, num_ps=2, num_controllers=0, - params=params._replace(variable_update='parameter_server')) - - self._test_variable_update( - test_name + '_rep', num_workers=2, num_ps=1, num_controllers=0, - params=params._replace(variable_update='distributed_replicated')) - - self._test_variable_update( - test_name + '_allreduce', num_workers=2, num_ps=0, num_controllers=1, - params=params._replace(variable_update='distributed_all_reduce', - all_reduce_spec='psgpu#%d' % params.num_gpus)) - - def testVarUpdateDefault(self): - params = test_util.get_var_update_params() - self._test_variable_updates('testVarUpdateDefault', params) - - def testVarUpdateCpuAsLocalParamDevice(self): - params = test_util.get_var_update_params()._replace( - local_parameter_device='cpu') - self._test_variable_updates('testVarUpdateCpuAsLocalParamDevice', params) - - def testVarUpdateFp16(self): - params = test_util.get_var_update_params()._replace(use_fp16=True) - self._test_variable_updates('testVarUpdateFp16', params) - - def testVarUpdateResourceVars(self): - params = test_util.get_var_update_params()._replace(use_resource_vars=True) - self._test_variable_updates('testVarUpdateResourceVars', params) - - -if __name__ == '__main__': - tf.disable_v2_behavior() - tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/benchmark_cnn_distributed_test_runner.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/benchmark_cnn_distributed_test_runner.py deleted file mode 100644 index 9291a801e4606c2b1982e5e1e0df833227a45e8f..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/benchmark_cnn_distributed_test_runner.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Used to run benchmark_cnn for distributed tests. - -In distributed tests, we spawn processes to run tf_cnn_benchmark tasks. We could -directly spawn tf_cnn_benchmark processes, but we want some added functionality, -such as being able to inject custom images during training. So instead, this -file is spawned as a Python process, which supports the added functionality. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl import flags as absl_flags -import numpy as np -import tensorflow.compat.v1 as tf -import benchmark_cnn -import flags -import preprocessing -import test_util - - -absl_flags.DEFINE_string('fake_input', 'none', - """What fake input to inject into benchmark_cnn. This - is ignored if --model=test_model. - Options are: - none: Do not inject any fake input. - zeros_and_ones: Half the images will be all 0s with - a label of 0. Half the images will be all 1s with a - label of 1.""") - -flags.define_flags() -FLAGS = flags.FLAGS - - -def get_test_image_preprocessor(batch_size, params): - """Returns the preprocessing.TestImagePreprocessor that should be injected. - - Returns None if no preprocessor should be injected. - - Args: - batch_size: The batch size across all GPUs. - params: BenchmarkCNN's parameters. - Returns: - Returns the preprocessing.TestImagePreprocessor that should be injected. - Raises: - ValueError: Flag --fake_input is an invalid value. - """ - if FLAGS.fake_input == 'none': - return None - elif FLAGS.fake_input == 'zeros_and_ones': - half_batch_size = batch_size // 2 - images = np.zeros((batch_size, 227, 227, 3), dtype=np.float32) - images[half_batch_size:, :, :, :] = 1 - labels = np.array([0] * half_batch_size + [1] * half_batch_size, - dtype=np.int32) - preprocessor = preprocessing.TestImagePreprocessor( - batch_size, [227, 227, 3], params.num_gpus, - benchmark_cnn.get_data_type(params)) - preprocessor.set_fake_data(images, labels) - preprocessor.expected_subset = 'validation' if params.eval else 'train' - return preprocessor - else: - raise ValueError('Invalid --fake_input: %s' % FLAGS.fake_input) - - -def run_with_real_model(params): - """Runs tf_cnn_benchmarks with a real model.""" - bench = benchmark_cnn.BenchmarkCNN(params) - bench.print_info() - preprocessor = get_test_image_preprocessor(bench.batch_size, params) - if preprocessor is not None: - # The test image preprocessor requires queue runners. Since this file is - # used for testing, it is OK to access protected members. - # pylint: disable=protected-access - bench.dataset._queue_runner_required = True - # pylint: enable=protected-access - bench.input_preprocessor = preprocessor - bench.run() - - -def run_with_test_model(params): - """Runs tf_cnn_benchmarks with a test model.""" - model = test_util.TestCNNModel() - inputs = test_util.get_fake_var_update_inputs() - with test_util.monkey_patch(benchmark_cnn, - LOSS_AND_ACCURACY_DIGITS_TO_SHOW=15): - bench = benchmark_cnn.BenchmarkCNN(params, dataset=test_util.TestDataSet(), - model=model) - # The test model does not use labels when computing loss, so the label - # values do not matter as long as it's the right shape. - labels = np.array([1] * inputs.shape[0]) - bench.input_preprocessor.set_fake_data(inputs, labels) - bench.run() - - -def main(_): - params = benchmark_cnn.make_params_from_flags() - params = benchmark_cnn.setup(params) - if params.model == 'test_model': - run_with_test_model(params) - else: - run_with_real_model(params) - - -if __name__ == '__main__': - tf.disable_v2_behavior() - tf.app.run() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/benchmark_cnn_test.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/benchmark_cnn_test.py deleted file mode 100644 index 9e849739c4687e2f53803fdb8d40d9a7e97ccb80..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/benchmark_cnn_test.py +++ /dev/null @@ -1,1493 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tests for benchmark_cnn.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import glob -import os -import re -import unittest - -import mock -import numpy as np -import tensorflow.compat.v1 as tf -from google.protobuf import text_format -from tensorflow.core.framework import step_stats_pb2 -from tensorflow.core.profiler import tfprof_log_pb2 -from tensorflow.python.platform import test -import benchmark_cnn -import datasets -import flags -import preprocessing -import test_util -import variable_mgr_util -from platforms import util as platforms_util - - -def _check_has_gpu(): - if not test.is_gpu_available(cuda_only=True): - raise ValueError( - """You have asked to run part or all of this on GPU, but it appears - that no GPU is available. If your machine has GPUs it is possible you - do not have a version of TensorFlow with GPU support. To build with GPU - support, add --config=cuda to the build flags.\n """) - - -class TfCnnBenchmarksModelTest(tf.test.TestCase): - """Tests which are run with multiple models.""" - - def setUp(self): - super(TfCnnBenchmarksModelTest, self).setUp() - benchmark_cnn.setup(benchmark_cnn.make_params()) - - def get_model_name(self): - return None - - # Return true to run tests that don't need to be run on every model. - # This should be done for one or two cheap models. - def extended_tests(self): - return False - - # Return false to suppress actually running the model; this is useful - # for tests that are large. - def model_execution_test(self): - return False - - # Return false to suppress actually saving and loading the model. - def model_save_load_test(self): - return False - - def testSaveLoadModel(self): - _check_has_gpu() - if not self.get_model_name() or not self.model_save_load_test(): - return - - params = benchmark_cnn.make_params( - model=self.get_model_name(), - num_batches=1, - num_intra_threads=0, - num_inter_threads=0, - distortions=False, - batch_size=2, - variable_update='replicated', - num_warmup_batches=0, - num_gpus=2, - train_dir=test_util.get_temp_dir('testSaveLoadModel_' + - self.get_model_name())) - - # Run one batch and save the model. - # Note that this uses a non-test session. - bench = benchmark_cnn.BenchmarkCNN(params) - bench.run() - self.assertEqual(bench.init_global_step, 0) - # Clear the default graph. - tf.reset_default_graph() - # Test if checkpoint had been saved. - ckpt = tf.train.get_checkpoint_state(params.train_dir) - match = re.match(os.path.join(params.train_dir, r'model.ckpt-(\d+).index'), - ckpt.model_checkpoint_path + '.index') - self.assertTrue(match) - self.assertGreaterEqual(int(match.group(1)), params.num_batches) - params = params._replace(num_batches=2) - # Reload the model - bench = benchmark_cnn.BenchmarkCNN(params) - bench.run() - # Check if global step has been restored. - self.assertNotEqual(bench.init_global_step, 0) - ckpt = tf.train.get_checkpoint_state(params.train_dir) - match = re.match(os.path.join(params.train_dir, r'model.ckpt-(\d+).index'), - ckpt.model_checkpoint_path + '.index') - self.assertTrue(match) - self.assertGreaterEqual(int(match.group(1)), params.num_batches) - # Check that the batch norm moving averages are restored from checkpoints - with tf.Graph().as_default(): - bench = benchmark_cnn.BenchmarkCNN(params) - bench._build_model() - saver = tf.train.Saver(bench.variable_mgr.savable_variables()) - with tf.Session(config=benchmark_cnn.create_config_proto(params)) as sess: - benchmark_cnn.load_checkpoint(saver, sess, params.train_dir) - sess.run(bench.variable_mgr.get_post_init_ops()) - bn_moving_vars = [ - v for v in tf.global_variables() - if '/batchnorm' in v.name and '/moving' in v.name - ] - self.assertGreater(len(bn_moving_vars), 0) - for moving_var in bn_moving_vars: - moving_var_value = sess.run(moving_var) - # Check that the moving means and moving variances have been restored - # by asserting they are not their default values of 0 and 1, - # respectively - if '/moving_mean' in moving_var.name: - self.assertFalse(np.array_equal(moving_var_value, - np.zeros(moving_var_value.shape, - moving_var_value.dtype))) - else: - self.assertIn('/moving_variance', moving_var.name) - self.assertFalse(np.array_equal(moving_var_value, - np.ones(moving_var_value.shape, - moving_var_value.dtype))) - - def testModel(self): - _check_has_gpu() - if not self.get_model_name() or not self.model_execution_test(): - return - - params = benchmark_cnn.make_params( - model=self.get_model_name(), - num_batches=1, - num_intra_threads=1, - num_inter_threads=12, - batch_size=2, - distortions=False) - - # Run this one; note that this uses a non-test session. - bench = benchmark_cnn.BenchmarkCNN(params) - bench.run() - - def testSendRecvVariables(self): - self._testVariables('parameter_server') - if self.extended_tests(): - self._testVariables('parameter_server', local_parameter_device='CPU') - self._testVariables('parameter_server', optimizer='sgd') - - def testReplicatedVariables(self): - self._testVariables('replicated') - if self.extended_tests(): - self._testVariables('replicated', all_reduce_spec=None) - self._testVariables('replicated', use_fp16=True, fp16_vars=False) - self._testVariables( - 'replicated', - all_reduce_spec=None, - use_fp16=True, - fp16_vars=False, - fp16_enable_auto_loss_scale=True, - fp16_inc_loss_scale_every_n=4) - - def testIndependentVariables(self): - self._testVariables('independent') - self._testVariables( - 'independent', - all_reduce_spec=None, - use_fp16=True, - fp16_vars=False, - fp16_enable_auto_loss_scale=True, - fp16_inc_loss_scale_every_n=4) - - def testSummaryVerbosity(self): - self._testVariables('parameter_server', summary_verbosity=1) - if self.extended_tests(): - self._testVariables('parameter_server', summary_verbosity=2) - self._testVariables('parameter_server', summary_verbosity=3) - - def testStagedVariables(self): - self._testVariables('parameter_server', staged_vars=True) - if self.extended_tests(): - self._testVariables('parameter_server', staged_vars=True, - local_parameter_device='CPU') - self._testVariables('parameter_server', staged_vars=True, use_fp16=True, - fp16_vars=True) - - def _assert_correct_var_type(self, var, params): - if 'gpu_cached_inputs' not in var.name: - if params.use_fp16 and params.fp16_vars and 'batchnorm' not in var.name: - expected_type = tf.float16 - else: - expected_type = tf.float32 - self.assertEqual(var.dtype.base_dtype, expected_type) - - def _testVariables(self, - variable_update, - summary_verbosity=0, - local_parameter_device='GPU', - staged_vars=False, - optimizer='momentum', - # TODO(b/80125832): Enable nccl in tests - # all_reduce_spec='nccl', - all_reduce_spec='', - use_fp16=False, - fp16_vars=False, - fp16_enable_auto_loss_scale=False, - fp16_inc_loss_scale_every_n=10): - if not self.get_model_name(): - return - _check_has_gpu() - - params = benchmark_cnn.make_params( - model=self.get_model_name(), - num_batches=1, - num_intra_threads=1, - num_inter_threads=12, - distortions=False, - variable_update=variable_update, - local_parameter_device=local_parameter_device, - num_gpus=2, - summary_verbosity=summary_verbosity, - staged_vars=staged_vars, - optimizer=optimizer, - all_reduce_spec=all_reduce_spec, - compact_gradient_transfer=False if all_reduce_spec == 'nccl' else True, - use_fp16=use_fp16, - fp16_loss_scale=2., - fp16_vars=fp16_vars, - fp16_enable_auto_loss_scale=fp16_enable_auto_loss_scale, - fp16_inc_loss_scale_every_n=fp16_inc_loss_scale_every_n, - ) - - # Test building models using multiple GPUs, but don't - # run them. - with self.test_session(graph=tf.Graph()): - bench = benchmark_cnn.BenchmarkCNN(params) - bench._build_model() - - # Rough validation of variable type and placement, depending on mode. - all_vars = tf.global_variables() + tf.local_variables() - if params.variable_update == 'parameter_server': - for v in all_vars: - tf.logging.debug('var: %s' % v.name) - match = re.match(r'tower_(\d+)/v/gpu_cached_inputs:0', v.name) - if match: - self.assertEqual(v.device, '/device:GPU:%s' % match.group(1)) - elif v.name.startswith('v/'): - self.assertEqual(v.device, '/device:%s:0' % local_parameter_device) - self._assert_correct_var_type(v, params) - elif v.name in ('input_processing/images:0', - 'input_processing/labels:0', 'init_learning_rate:0', - 'global_step:0', 'loss_scale:0', - 'loss_scale_normal_steps:0'): - self.assertEqual(v.device, '/device:CPU:0') - else: - raise ValueError('Unexpected variable %s' % v.name) - else: - v0_count = 0 - v1_count = 0 - for v in all_vars: - if v.name.startswith('tower_0/v0/'): - self.assertEqual(v.name, 'tower_0/v0/gpu_cached_inputs:0') - self.assertEqual(v.device, '/device:GPU:0') - elif v.name.startswith('tower_1/v1/'): - self.assertEqual(v.name, 'tower_1/v1/gpu_cached_inputs:0') - self.assertEqual(v.device, '/device:GPU:1') - elif v.name.startswith('v0/'): - v0_count += 1 - self.assertEqual(v.device, '/device:GPU:0') - self._assert_correct_var_type(v, params) - elif v.name.startswith('v1/'): - v1_count += 1 - self.assertEqual(v.device, '/device:GPU:1') - self._assert_correct_var_type(v, params) - elif v.name in ('input_processing/images:0', - 'input_processing/labels:0', 'init_learning_rate:0', - 'global_step:0', 'loss_scale:0', - 'loss_scale_normal_steps:0'): - self.assertEqual(v.device, '/device:CPU:0') - else: - raise ValueError('Unexpected variable %s' % v.name) - self.assertEqual(v0_count, v1_count) - - # Validate summary ops in the model depending on verbosity level - summary_ops = tf.get_collection(tf.GraphKeys.SUMMARIES) - num_summary_ops = len(summary_ops) - self.assertEqual(num_summary_ops > 0, summary_verbosity > 0) - if summary_verbosity > 0: - has_affine_histogram = False - has_gradient_histogram = False - has_log_gradients_histogram = False - for op in summary_ops: - if '/gradients' in op.name: - has_gradient_histogram = True - elif '/affine' in op.name: - has_affine_histogram = True - elif 'log_gradients' in op.name: - has_log_gradients_histogram = True - self.assertEqual(summary_verbosity >= 3, has_affine_histogram) - self.assertEqual(summary_verbosity >= 3, has_gradient_histogram) - self.assertEqual(summary_verbosity >= 2, has_log_gradients_histogram) - if summary_verbosity == 1: - self.assertLess(num_summary_ops, 10) - - -class TrivialModelTest(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'trivial' - - -class TestVgg1Model(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'vgg11' - - -class TestVgg19Model(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'vgg19' - - -class TestLenet5Model(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'lenet' - - -class TestGooglenetModel(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'googlenet' - - -class TestOverfeatModel(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'overfeat' - - -class TestAlexnetModel(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'alexnet' - - def extended_tests(self): - return True - - -class TestTrivialModel(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'trivial' - - -class TestInceptionv3Model(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'inception3' - - def extended_tests(self): - return True - - -class TestInceptionv4Model(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'inception4' - - -class TestResnet50Model(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'resnet50' - - def model_save_load_test(self): - return True - - -class TestResnet101Model(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'resnet101' - - -class TestResnet152Model(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'resnet152' - - -class TestResnet50V2Model(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'resnet50_v2' - - -class TestResnet101V2Model(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'resnet101_v2' - - -class TestResnet152V2Model(TfCnnBenchmarksModelTest): - - def get_model_name(self): - return 'resnet152_v2' - - -class TfCnnBenchmarksTest(tf.test.TestCase): - """Tests that benchmark_cnn runs correctly.""" - - def setUp(self): - super(TfCnnBenchmarksTest, self).setUp() - _check_has_gpu() - benchmark_cnn.setup(benchmark_cnn.make_params()) - - def _run_benchmark_cnn(self, params): - logs = [] - benchmark_cnn.log_fn = test_util.print_and_add_to_list(logs) - benchmark_cnn.BenchmarkCNN(params).run() - return logs - - def _run_benchmark_cnn_with_fake_images(self, params, images, labels): - logs = [] - benchmark_cnn.log_fn = test_util.print_and_add_to_list(logs) - bench = benchmark_cnn.BenchmarkCNN(params) - bench.input_preprocessor = preprocessing.TestImagePreprocessor( - params.batch_size * params.num_gpus, - [[params.batch_size, 227, 227, 3], [params.batch_size]], - params.num_gpus, - bench.model.data_type) - bench.dataset._queue_runner_required = True - bench.input_preprocessor.set_fake_data(images, labels) - bench.input_preprocessor.expected_subset = ('validation' - if params.eval else 'train') - bench.run() - return logs - - def _run_benchmark_cnn_with_black_and_white_images(self, params): - """Runs BenchmarkCNN with black and white images. - - A BenchmarkCNN is created and run with black and white images as input. Half - the images are black (i.e., filled with 0s) and half are white (i.e., filled - with 255s). - - Args: - params: Params for BenchmarkCNN. - - Returns: - A list of lines from the output of BenchmarkCNN. - """ - # TODO(reedwm): Instead of generating images here, use black and white - # tfrecords by calling test_util.create_black_and_white_images(). - effective_batch_size = params.batch_size * params.num_gpus - half_batch_size = effective_batch_size // 2 - images = np.zeros((effective_batch_size, 227, 227, 3), dtype=np.float32) - images[half_batch_size:, :, :, :] = 255 - labels = np.array([0] * half_batch_size + [1] * half_batch_size, - dtype=np.int32) - return self._run_benchmark_cnn_with_fake_images(params, images, labels) - - def _train_and_eval_local(self, - params, - check_output_values=False, - max_final_loss=10., - skip=None, - use_test_preprocessor=True): - # TODO(reedwm): check_output_values should default to True and be enabled - # on every test. Currently, if check_output_values=True and the calls to - # tf.set_random_seed(...) and np.seed(...) are passed certain seed values in - # benchmark_cnn.py, then most tests will fail. This indicates the tests - # are brittle and could fail with small changes when - # check_output_values=True, so check_output_values defaults to False for - # now. - - def run_fn(run_type, inner_params): - del run_type - if use_test_preprocessor: - return [ - self._run_benchmark_cnn_with_black_and_white_images(inner_params) - ] - else: - return [self._run_benchmark_cnn(inner_params)] - - return test_util.train_and_eval(self, run_fn, params, - check_output_values=check_output_values, - max_final_loss=max_final_loss, - skip=skip) - - def testAlexnet(self): - params = test_util.get_params('testAlexnet')._replace( - num_batches=30, init_learning_rate=0.01, model='alexnet') - self._train_and_eval_local(params) - - def testNoPrintAccuracy(self): - params = test_util.get_params('testNoPrintAccuracy')._replace( - print_training_accuracy=False) - self._train_and_eval_local(params) - - def testLowAccuracy(self): - params = test_util.get_params('testLowAccuracy')._replace( - print_training_accuracy=True, batch_size=5, num_batches=10) - # We force low accuracy by having each batch containing 10 identical images, - # each with a different label. This guarantees a top-1 accuracy of exactly - # 0.1 and a top-5 accuracy of exactly 0.5. - images = np.zeros((10, 227, 227, 3), dtype=np.float32) - labels = np.arange(10, dtype=np.int32) - logs = self._run_benchmark_cnn_with_fake_images(params, images, labels) - training_outputs = test_util.get_training_outputs_from_logs( - logs, params.print_training_accuracy) - last_output = training_outputs[-1] - # TODO(reedwm): These should be assertEqual but for some reason, - # occasionally the accuracies are lower (Running this test 500 times, these - # asserts failed twice). Investigate this problem. - self.assertLessEqual(last_output.top_1_accuracy, 0.1) - self.assertLessEqual(last_output.top_5_accuracy, 0.5) - - def testParameterServer(self): - params = test_util.get_params('testParameterServer') - self._train_and_eval_local(params) - - def testParameterServerStaged(self): - params = test_util.get_params('testParameterServerStaged')._replace( - staged_vars=True) - self._train_and_eval_local(params) - - def testReplicated(self): - params = test_util.get_params('testReplicated')._replace( - variable_update='replicated') - self._train_and_eval_local(params) - - def testIndependent(self): - params = test_util.get_params('testIndependent')._replace( - variable_update='independent') - self._train_and_eval_local(params) - - def testForwardOnly(self): - params = test_util.get_params('testForwardOnly')._replace(forward_only=True) - # Evaluation is not supported with --forward_only, so we set skip='eval'. - self._train_and_eval_local(params, skip='eval') - - def testForwardOnlyAndFreeze(self): - params = test_util.get_params('testForwardOnlyAndFreeze')._replace( - forward_only=True, freeze_when_forward_only=True, train_dir=None) - # Training is not supported with --freeze_when_forward_only. - self._train_and_eval_local(params, skip='eval_and_train_from_checkpoint') - - def testNoDistortions(self): - params = test_util.get_params('testNoDistortions')._replace( - distortions=False) - self._train_and_eval_local(params) - - def testCpuAsLocalParamDevice(self): - params = test_util.get_params('testCpuAsLocalParamDevice')._replace( - local_parameter_device='cpu') - self._train_and_eval_local(params) - - def testNHWC(self): - params = test_util.get_params('testNHWC')._replace(data_format='NHWC') - self._train_and_eval_local(params) - - def testCpuAsDevice(self): - params = test_util.get_params('testCpuAsDevice')._replace( - device='cpu', data_format='NHWC') # NHWC required when --device=cpu - self._train_and_eval_local(params) - - def testMomentumParameterServer(self): - params = test_util.get_params('testMomentumParameterServer')._replace( - optimizer='momentum', momentum=0.8) - self._train_and_eval_local(params) - - def testRmspropReplicated(self): - params = test_util.get_params('testRmspropReplicated')._replace( - variable_update='replicated', - optimizer='rmsprop', - rmsprop_decay=0.8, - rmsprop_momentum=0.6, - rmsprop_epsilon=0.7, - init_learning_rate=0.01) - self._train_and_eval_local(params) - - def testBatchGroupSize(self): - params = test_util.get_params('testBatchGroupSize')._replace( - batch_group_size=4, num_batches=100, num_warmup_batches=5) - self._train_and_eval_local(params) - - def testGradientClip(self): - params = test_util.get_params('testGradientClip')._replace( - gradient_clip=100.0) - self._train_and_eval_local(params) - - def testWeightDecay(self): - params = test_util.get_params('testWeightDecay')._replace( - weight_decay=0.0001) - self._train_and_eval_local(params) - - def testNoLayers(self): - params = test_util.get_params('testNoLayers')._replace(use_tf_layers=False) - self._train_and_eval_local(params) - - def testSaveModelSteps(self): - params = test_util.get_params('testSaveModelSteps')._replace( - save_model_steps=2, num_warmup_batches=0, num_batches=10, - max_ckpts_to_keep=3) - self._train_and_eval_local(params) - for i in range(1, 20 + 1): - # We train for 20 steps, since self._train_and_eval_local() does two - # training runs of 10 steps each. We save a checkpoint every 2 steps and - # keep the last 3 checkpoints, so at the end, we should have checkpoints - # for steps 16, 18, and 20. - matches = glob.glob(os.path.join(params.train_dir, - 'model.ckpt-{}.*'.format(i))) - if i in (16, 18, 20): - self.assertTrue(matches) - else: - self.assertFalse(matches) - - def testFp16WithFp32Vars(self): - params = test_util.get_params('testFp16WithFp32Vars')._replace( - use_fp16=True, fp16_vars=False, fp16_loss_scale=1.) - self._train_and_eval_local(params) - - def testFp16WithFp16Vars(self): - params = test_util.get_params('testFp16WithFp16Vars')._replace( - use_fp16=True, fp16_vars=True) - self._train_and_eval_local(params) - - def testXlaCompile(self): - params = test_util.get_params('testXlaCompile')._replace(xla_compile=True) - self._train_and_eval_local(params) - - @unittest.skip('Fails for unknown reason') - def testXlaCompileWithFp16(self): - params = test_util.get_params('testXlaCompileWithFp16')._replace( - use_fp16=True, xla_compile=True) - self._train_and_eval_local(params) - - def testGradientRepacking(self): - params = test_util.get_params('testGradientRepacking1')._replace( - gradient_repacking=2) - self._train_and_eval_local(params, skip='eval_and_train_from_checkpoint') - params = test_util.get_params('testGradientRepacking2')._replace( - gradient_repacking=2, use_fp16=True) - self._train_and_eval_local(params, skip='eval_and_train_from_checkpoint') - - def testTraceFileChromeTraceFormat(self): - trace_file = os.path.join(self.get_temp_dir(), - 'testTraceFileChromeTraceFormat_tracefile') - params = test_util.get_params('testTraceFileChromeTraceFormat')._replace( - trace_file=trace_file, use_chrome_trace_format=True) - self._train_and_eval_local(params) - self.assertGreater(os.stat(trace_file).st_size, 0) - - def testTraceFileStepStatsProto(self): - trace_file = os.path.join(self.get_temp_dir(), - 'testTraceFileStepStatsProto_tracefile') - params = test_util.get_params('testTraceFileStepStatsProto')._replace( - trace_file=trace_file, use_chrome_trace_format=False) - self._train_and_eval_local(params) - self.assertGreater(os.stat(trace_file).st_size, 0) - with open(trace_file) as f: - step_stats = step_stats_pb2.StepStats() - # The following statement should not raise an exception. - contents = f.read() - text_format.Merge(contents, step_stats) - - def testTfprofFile(self): - tfprof_file = os.path.join(self.get_temp_dir(), 'testTfprofFile_tfproffile') - params = test_util.get_params('testTfprofFile')._replace( - tfprof_file=tfprof_file) - self._train_and_eval_local(params, skip='eval_and_train_from_checkpoint') - self.assertGreater(os.stat(tfprof_file).st_size, 0) - with open(tfprof_file, 'rb') as f: - profile_proto = tfprof_log_pb2.ProfileProto() - # The following statement should not raise an exception. - profile_proto.ParseFromString(f.read()) - - @unittest.skip('Fails for unknown reason') - def testMoveTrainDir(self): - params = test_util.get_params('testMoveTrainDir') - self._train_and_eval_local(params) - new_train_dir = params.train_dir + '_moved' - os.rename(params.train_dir, new_train_dir) - params = params._replace(train_dir=new_train_dir, eval=True) - self._run_benchmark_cnn_with_black_and_white_images(params) - - @mock.patch('tensorflow.compat.v1.train.Saver') - @mock.patch('benchmark_cnn._get_checkpoint_to_load') - def testLoadCheckpoint(self, mock_checkpoint_to_load, mock_saver): - """Tests load checkpoint with full path to checkpoint.""" - expected_checkpoint = '/path/to/checkpoints/model.ckpt-1243' - mock_checkpoint_to_load.return_value = expected_checkpoint - - global_batch = benchmark_cnn.load_checkpoint(mock_saver, - None, - expected_checkpoint) - self.assertEqual(global_batch, 1243) - - def testGetCheckpointToLoadFullPath(self): - """Tests passing full path.""" - ckpt_path = '/foo/bar/model.ckpt-189' - full_path = benchmark_cnn._get_checkpoint_to_load(ckpt_path) - self.assertEqual(full_path, ckpt_path) - - def testGetCheckpointToLoadException(self): - """Tests exception for directory without a checkpoint.""" - ckpt_path = '/foo/bar/checkpoints' - self.assertRaises(benchmark_cnn.CheckpointNotFoundException, - benchmark_cnn._get_checkpoint_to_load, ckpt_path) - - @mock.patch('tensorflow.compat.v1.train.get_checkpoint_state') - def testGetCheckpointToLoad(self, mock_checkpoint_state): - """Tests passing path to checkpoint folder.""" - expected_checkpoint = '/path/to/checkpoints/model.ckpt-1243' - mock_checkpoint_state.return_value = mock.Mock( - model_checkpoint_path=expected_checkpoint) - ckpt_path = '/path/to/checkpoints/' - full_path = benchmark_cnn._get_checkpoint_to_load(ckpt_path) - self.assertEqual(full_path, expected_checkpoint) - - def testImagenetPreprocessor(self): - imagenet_dir = os.path.join(platforms_util.get_test_data_dir(), - 'fake_tf_record_data') - params = test_util.get_params('testImagenetPreprocessor')._replace( - data_dir=imagenet_dir, data_name='imagenet') - self._train_and_eval_local(params, use_test_preprocessor=False) - - def testImagenetPreprocessorNoDistortions(self): - imagenet_dir = os.path.join(platforms_util.get_test_data_dir(), - 'fake_tf_record_data') - params = test_util.get_params( - 'testImagenetPreprocessorNoDistortions')._replace( - data_dir=imagenet_dir, data_name='imagenet', distortions=False) - self._train_and_eval_local(params, use_test_preprocessor=False) - - def testImagenetPreprocessorVerboseSummary(self): - imagenet_dir = os.path.join(platforms_util.get_test_data_dir(), - 'fake_tf_record_data') - params = test_util.get_params( - 'testImagenetPreprocessorVerboseSummary')._replace( - data_dir=imagenet_dir, data_name='imagenet', distortions=False, - summary_verbosity=2) - self._train_and_eval_local(params, use_test_preprocessor=False) - - def testCifar10SyntheticData(self): - params = test_util.get_params('testCifar10SyntheticData')._replace( - data_name='cifar10') - self._train_and_eval_local(params) - - def testShiftRatio(self): - test_util.monkey_patch_base_cluster_manager() - params = benchmark_cnn.make_params( - data_name='imagenet', - data_dir=os.path.join(platforms_util.get_test_data_dir(), - 'fake_tf_record_data'), - job_name='worker', - worker_hosts='w1,w2,w3,w4', - ps_hosts='p1', - task_index=0) - self.assertEqual( - benchmark_cnn.BenchmarkCNN(params).input_preprocessor.shift_ratio, 0.0) - params = params._replace(task_index=3) - self.assertEqual( - benchmark_cnn.BenchmarkCNN(params).input_preprocessor.shift_ratio, 0.75) - - def testDistributedReplicatedSavableVars(self): - test_util.monkey_patch_base_cluster_manager() - params = benchmark_cnn.make_params( - variable_update='distributed_replicated', - model='inception4', - data_name='imagenet', - data_dir=os.path.join(platforms_util.get_test_data_dir(), - 'fake_tf_record_data'), - job_name='worker', - worker_hosts='w1,w2,w3,w4', - ps_hosts='p1', - datasets_use_prefetch=False) - - bench = benchmark_cnn.BenchmarkCNN(params) - with tf.Graph().as_default(): - bench._build_model() - savable_vars = bench.variable_mgr.savable_variables() - # Assert all global variables are in savable_vars - for v in tf.global_variables(): - if not v.name.startswith( - variable_mgr_util.PS_SHADOW_VAR_PREFIX + '/v0'): - self.assertEqual(v.name, 'global_step:0') - name = bench.variable_mgr._strip_port(v.name) - if name.startswith(variable_mgr_util.PS_SHADOW_VAR_PREFIX): - name = name[len(variable_mgr_util.PS_SHADOW_VAR_PREFIX + '/'):] - self.assertIn(name, savable_vars) - self.assertIn(savable_vars[name], tf.global_variables()) - # Assert all local variables on the first tower are in savable_vars - for v in tf.local_variables(): - if v.name.startswith('v0/'): - name = bench.variable_mgr._strip_port(v.name) - self.assertIn(name, savable_vars) - - def _test_preprocessing_eval(self, image_height, image_width, output_height, - output_width): - image = tf.fill((image_height, image_width, 3), - tf.constant(128, dtype=tf.uint8)) - params = benchmark_cnn.make_params() - new_image = preprocessing.eval_image(image, output_height, output_width, 0, - 'bilinear', params.summary_verbosity) - with self.test_session() as sess: - new_image_value = sess.run(new_image) - self.assertAllEqual(new_image_value, - np.full((output_height, output_width, 3), 128, - dtype=np.uint8)) - - def testPreprocessingEval(self): - self._test_preprocessing_eval(10, 10, 4, 4) - self._test_preprocessing_eval(4, 4, 10, 10) - self._test_preprocessing_eval(1, 100, 100, 1) - self._test_preprocessing_eval(100, 1, 1, 100) - self._test_preprocessing_eval(1, 100, 1, 100) - - def _test_preprocessing_traing(self, image_buf, image_color, - output_height, output_width, bbox, - batch_position, resize_method, distortions, - summary_verbosity, fuse_decode_and_crop): - new_image = preprocessing.train_image( - image_buf, - output_height, - output_width, - bbox, - batch_position, - resize_method, - distortions, - summary_verbosity=summary_verbosity, - fuse_decode_and_crop=fuse_decode_and_crop) - self.assertEqual(new_image.shape, [output_height, output_width, 3]) - with self.test_session(use_gpu=True) as sess: - new_image_value = sess.run(new_image) - self.assertAllClose( - new_image_value, - np.full( - [output_height, output_width, 3], - image_color, - dtype=np.float32), - atol=50., - rtol=0.) - - def testPreprocessingTrain(self): - test_data_dir = os.path.join(platforms_util.get_test_data_dir(), 'images') - black_file = os.path.join(test_data_dir, 'black_image.jpg') - with open(black_file, 'rb') as f: - black_jpg_buffer = f.read() - white_file = os.path.join(test_data_dir, 'white_image.jpg') - with open(white_file, 'rb') as f: - white_jpg_buffer = f.read() - bbox = tf.zeros((1, 0, 4), dtype=tf.float32) - batch_position = 0 - # Each size config is (output_height, output_width, resize_method) - size_configs = [(100, 100, 'round_robin'), (150, 10, 'bilinear'), - (10, 150, 'nearest')] - # Each image config is (image_buf, image_color) - image_configs = [(white_jpg_buffer, 255), (black_jpg_buffer, 0)] - for (image_buf, image_color) in image_configs: - for output_height, output_width, resize_method in size_configs: - for distortions in [True, False]: - for summary_verbosity in [0, 2]: - for fuse_decode_and_crop in [True, False]: - self._test_preprocessing_traing( - image_buf, image_color, output_height, output_width, bbox, - batch_position, resize_method, distortions, summary_verbosity, - fuse_decode_and_crop) - - def _test_learning_rate(self, params, global_step_to_expected_learning_rate): - self.longMessage = True # pylint: disable=invalid-name - bench = benchmark_cnn.BenchmarkCNN(params) - with tf.Graph().as_default() as graph: - bench._build_model() - global_step = graph.get_tensor_by_name('global_step:0') - learning_rate = graph.get_tensor_by_name('learning_rate_tensor:0') - with self.test_session(graph=graph, use_gpu=True) as sess: - items = global_step_to_expected_learning_rate.items() - for global_step_val, expected_learning_rate in items: - self.assertAlmostEqual(sess.run(learning_rate, - {global_step: global_step_val}), - expected_learning_rate, - msg='at global_step:{}'. - format(global_step_val)) - - def testLearningRateModelSpecificResNet(self): - params = benchmark_cnn.make_params(model='resnet50', - batch_size=256, - variable_update='parameter_server', - num_gpus=1) - self._test_learning_rate(params, { - 0: 0, - 150136: 0.128, - 150137: 0.0128, - 300273: 0.0128, - 300274: 0.00128, - 10000000: 0.0000128 - }) - - def testLearningRateUserProvidedInitLr(self): - params = benchmark_cnn.make_params(model='resnet50', - batch_size=256, - variable_update='replicated', - init_learning_rate=1.) - self._test_learning_rate(params, { - 0: 1., - 10000000: 1. - }) - - def testLearningRateUserProvidedInitLrAndWarmup(self): - params = benchmark_cnn.make_params(model='resnet50', - batch_size=256, - variable_update='replicated', - init_learning_rate=1., - num_learning_rate_warmup_epochs=5) - self._test_learning_rate(params, { - 0: 0., - 12511: 0.5, - 25022: 1., - 10000000: 1. - }) - - def testLearningRateUserProvidedDecayInfo(self): - params = benchmark_cnn.make_params(model='resnet50', - init_learning_rate=1., - learning_rate_decay_factor=0.5, - num_epochs_per_decay=2, - minimum_learning_rate=0.3750, - batch_size=32) - self._test_learning_rate(params, { - 0: 1., - 80071: 1., - 80072: 0.5, - 160143: 0.5, - 160144: 0.375, - 10000000: 0.375 - }) - - def testLearningRateUserProvidedZeroDecay(self): - params = benchmark_cnn.make_params(model='resnet50', - num_learning_rate_warmup_epochs=0, - learning_rate_decay_factor=0.5, - num_epochs_per_decay=0, - minimum_learning_rate=0.3750, - batch_size=32) - with self.assertRaises(ValueError): - with tf.Graph().as_default(): - # This will fail because params.learning_rate_decay_factor cannot be - # nonzero if params.num_epochs_per_decay is zero. - benchmark_cnn.BenchmarkCNN(params)._build_model() - - def testLearningRateUserProvidedSchedule(self): - params = benchmark_cnn.make_params( - model='trivial', - batch_size=32, - piecewise_learning_rate_schedule='1;3;.1;5;.01') - self._test_learning_rate(params, { - 0: 1., - 120108: 1., - 120109: 0.1, - 200181: 0.1, - 200182: 0.01, - 100000000: 0.01 - }) - - def testNumBatchesAndEpochs(self): - params = benchmark_cnn.make_params() - batches, epochs = benchmark_cnn.get_num_batches_and_epochs(params, 10, 100) - self.assertEqual(batches, benchmark_cnn._DEFAULT_NUM_BATCHES) - self.assertAlmostEqual(epochs, - float(benchmark_cnn._DEFAULT_NUM_BATCHES) / 10) - - params = benchmark_cnn.make_params(num_batches=21) - batches, epochs = benchmark_cnn.get_num_batches_and_epochs(params, 25, 50) - self.assertEqual(batches, 21) - self.assertAlmostEqual(epochs, 10.5) - - params = benchmark_cnn.make_params(num_epochs=3) - batches, epochs = benchmark_cnn.get_num_batches_and_epochs(params, 2, 3) - self.assertEqual(batches, 5) - self.assertAlmostEqual(epochs, 10./3.) - - params = benchmark_cnn.make_params(num_epochs=4) - batches, epochs = benchmark_cnn.get_num_batches_and_epochs(params, 2, 3) - self.assertEqual(batches, 6) - self.assertAlmostEqual(epochs, 4) - - with self.assertRaises(ValueError): - params = benchmark_cnn.make_params(num_batches=100, num_epochs=100) - benchmark_cnn.get_num_batches_and_epochs(params, 1, 1) - - def _testEvalDuringTraining(self, params, expected_num_eval_batches_found): - # The idea of this test is that all train images are black and all eval - # images are white. We pass the images through the TestModel, and ensure - # the outputs are as expected. - - batch_size = params.batch_size - eval_batch_size = params.eval_batch_size or params.batch_size - - class TestModel(test_util.TestCNNModel): - - def __init__(self): - super(TestModel, self).__init__() - self.depth = 3 - - def add_inference(self, cnn): - if cnn.phase_train: - # This will allow us to test that 100 is only added during training - # and not during eval. - cnn.top_layer += 100 - assert cnn.top_layer.shape[0] == batch_size - else: - assert cnn.top_layer.shape[0] == eval_batch_size - - # Reduce the image to a single number. The number should be (-1 + 100) - # during training and 1 during testing. - cnn.top_layer = tf.reshape(cnn.top_layer, (cnn.top_layer.shape[0], -1)) - cnn.top_layer = tf.reduce_mean(cnn.top_layer, axis=1) - cnn.top_layer = tf.reshape(cnn.top_layer, - (cnn.top_layer.shape[0], 1, 1, 1)) - cnn.top_size = 1 - trainable_vars = tf.trainable_variables() - - # The super method will compute image*A*B, where A=1 and B=2. - super(TestModel, self).add_inference(cnn) - - if not cnn.phase_train: - # Assert no new variables were added, since they should be reused from - # training. - assert len(trainable_vars) == len(tf.trainable_variables()) - - model = TestModel() - dataset = datasets.ImagenetDataset(params.data_dir) - logs = [] - bench_cnn = benchmark_cnn.BenchmarkCNN(params, model=model, dataset=dataset) - with test_util.monkey_patch(benchmark_cnn, - log_fn=test_util.print_and_add_to_list(logs)): - bench_cnn.run() - training_outputs = test_util.get_training_outputs_from_logs( - logs, print_training_accuracy=False) - self.assertEqual(len(training_outputs), params.num_batches) - expected_training_output = (-1 + 100) * 1 * 2 - for training_output in training_outputs: - self.assertEqual(training_output.loss, expected_training_output) - eval_outputs = test_util.get_evaluation_outputs_from_logs(logs) - self.assertTrue(eval_outputs) - expected_eval_output = 1 * 1 * 2 - for eval_output in eval_outputs: - self.assertEqual(eval_output.top_1_accuracy, expected_eval_output) - self.assertEqual(eval_output.top_5_accuracy, expected_eval_output) - - num_eval_batches_found = 0 - eval_batch_regex = re.compile(r'^\d+\t[0-9.]+ examples/sec$') - for log in logs: - if eval_batch_regex.match(log): - num_eval_batches_found += 1 - self.assertEqual(num_eval_batches_found, expected_num_eval_batches_found) - - def testEvalDuringTraining(self): - data_dir = test_util.create_black_and_white_images() - base_params = test_util.get_params('testEvalDuringTraining') - train_dir = base_params.train_dir - base_params = base_params._replace( - train_dir=None, print_training_accuracy=False, num_warmup_batches=0, - num_batches=7, num_eval_batches=2, display_every=1, - init_learning_rate=0, weight_decay=0, - distortions=False, data_dir=data_dir) - expected_num_eval_batches_found = ( - base_params.num_eval_batches * (base_params.num_batches // 2 + 1)) - - # Test --eval_during_training_every_n_steps - self._testEvalDuringTraining( - base_params._replace(eval_during_training_every_n_steps=2, - variable_update='parameter_server'), - expected_num_eval_batches_found) - self._testEvalDuringTraining( - base_params._replace(eval_during_training_every_n_steps=2, - variable_update='replicated'), - expected_num_eval_batches_found) - self._testEvalDuringTraining( - base_params._replace(eval_during_training_every_n_steps=2, - variable_update='replicated', - summary_verbosity=2, - save_summaries_steps=2, - datasets_use_prefetch=False), - expected_num_eval_batches_found) - self._testEvalDuringTraining( - base_params._replace(eval_during_training_every_n_steps=2, - variable_update='replicated', - use_fp16=True, train_dir=train_dir, - eval_batch_size=base_params.batch_size + 2), - expected_num_eval_batches_found) - - # Test --eval_during_training_every_n_epochs - every_n_epochs = (2 * base_params.batch_size * base_params.num_gpus / - datasets.IMAGENET_NUM_TRAIN_IMAGES) - self._testEvalDuringTraining( - base_params._replace(eval_during_training_every_n_epochs=every_n_epochs, - variable_update='replicated'), - expected_num_eval_batches_found) - - # Test --eval_during_training_at_specified_steps - list_steps = [2, 3, 5, 7, 1000] - num_eval_steps = 1 + sum(1 for step in list_steps - if step < base_params.num_batches) - expected_num_eval_batches_found = ( - base_params.num_eval_batches * num_eval_steps) - - self._testEvalDuringTraining( - base_params._replace(eval_during_training_at_specified_steps=list_steps, - variable_update='replicated'), - expected_num_eval_batches_found) - - # Test --eval_during_training_at_specified_epochs - list_epochs = [(step * base_params.batch_size * base_params.num_gpus / - datasets.IMAGENET_NUM_TRAIN_IMAGES) - for step in list_steps] - self._testEvalDuringTraining( - base_params._replace( - eval_during_training_at_specified_epochs=list_epochs, - variable_update='replicated'), - expected_num_eval_batches_found) - - # Test --eval_during_training_every_n_steps runs with synthetic data. - params = base_params._replace( - variable_update='replicated', data_dir=None, - eval_during_training_every_n_steps=2, num_batches=2) - benchmark_cnn.BenchmarkCNN(params).run() - - def testEvalDuringTrainingNumEpochs(self): - params = benchmark_cnn.make_params( - batch_size=1, eval_batch_size=2, eval_during_training_every_n_steps=1, - num_batches=30, num_eval_epochs=100 / datasets.IMAGENET_NUM_VAL_IMAGES) - bench_cnn = benchmark_cnn.BenchmarkCNN(params) - self.assertEqual(bench_cnn.num_batches, 30) - self.assertAlmostEqual(bench_cnn.num_epochs, - 30 / datasets.IMAGENET_NUM_TRAIN_IMAGES) - self.assertAlmostEqual(bench_cnn.num_eval_batches, 50) - self.assertAlmostEqual(bench_cnn.num_eval_epochs, - 100 / datasets.IMAGENET_NUM_VAL_IMAGES) - - def testEarlyStopping(self): - params = benchmark_cnn.make_params( - batch_size=2, - display_every=1, - num_batches=100, - eval_during_training_every_n_steps=2, - stop_at_top_1_accuracy=0.4, - ) - with mock.patch.object(benchmark_cnn.BenchmarkCNN, '_eval_once', - side_effect=[(0.1, 0.1), (0.5, 0.5), (0.2, 0.2)] - ) as mock_eval_once: - logs = [] - bench_cnn = benchmark_cnn.BenchmarkCNN(params) - with test_util.monkey_patch(benchmark_cnn, - log_fn=test_util.print_and_add_to_list(logs)): - bench_cnn.run() - training_outputs = test_util.get_training_outputs_from_logs( - logs, print_training_accuracy=False) - # We should stop after the second evaluation, and we evaluate every 2 - # steps. So there should be 2 * 2 = 4 training outputs. - self.assertEqual(len(training_outputs), 4) - self.assertEqual(mock_eval_once.call_count, 2) - - def testOutOfRangeErrorsAreNotIgnored(self): - error_msg = 'Fake OutOfRangeError error message' - with mock.patch.object(benchmark_cnn.BenchmarkCNN, 'benchmark_with_session', - side_effect=tf.errors.OutOfRangeError(None, None, - error_msg)): - with self.assertRaisesRegex(RuntimeError, error_msg): - benchmark_cnn.BenchmarkCNN(benchmark_cnn.make_params()).run() - - def testInvalidFlags(self): - params = benchmark_cnn.make_params(device='cpu', data_format='NCHW') - with self.assertRaises(ValueError): - benchmark_cnn.BenchmarkCNN(params) - - params = benchmark_cnn.make_params(use_fp16=True, fp16_vars=True, - variable_update='replicated', - all_reduce_spec='nccl') - with self.assertRaises(ValueError): - benchmark_cnn.BenchmarkCNN(params) - - # Automatic loss scaling is only supported for 'replicated', 'ps', - # and 'independent' variable_updates. - invalid_variable_updates = [ - 'distributed_replicated', 'distributed_all_reduce' - ] - for variable_update in invalid_variable_updates: - params = benchmark_cnn.make_params( - use_fp16=True, - fp16_vars=True, - fp16_enable_auto_loss_scale=True, - variable_update=variable_update) - with self.assertRaises(ValueError): - benchmark_cnn.BenchmarkCNN(params) - - # Automatic loss scaling is not supported for 'nccl'. - params = benchmark_cnn.make_params( - use_fp16=True, - fp16_vars=True, - fp16_enable_auto_loss_scale=True, - all_reduce_spec='nccl') - with self.assertRaises(ValueError): - benchmark_cnn.BenchmarkCNN(params) - - # Automatic loss scaling is not supported for 'staged_vars'. - params = benchmark_cnn.make_params( - use_fp16=True, - fp16_vars=True, - fp16_enable_auto_loss_scale=True, - staged_vars=True) - with self.assertRaises(ValueError): - benchmark_cnn.BenchmarkCNN(params) - - def testMakeParams(self): - default_params = benchmark_cnn.make_params() - self.assertEqual(default_params.model, - flags.param_specs['model'].default_value) - params = benchmark_cnn.make_params(model='foo') - self.assertEqual(params.model, 'foo') - with self.assertRaises(ValueError): - benchmark_cnn.make_params(job_name='foo') - with self.assertRaises(ValueError): - benchmark_cnn.make_params(gpu_memory_frac_for_testing=-1.) - - -class VariableUpdateTest(tf.test.TestCase): - """Tests that variables are updated correctly. - - These tests use a very simple deterministic model. For example, some tests use - the model - - loss = image * A * B - - where image is a 1x1 images (with a single scalar value), and A and B are - scalar variables. Tests will run tf_cnn_benchmarks with such a model, on a - sequence of scalar images, and assert that the losses are the correct value. - Since the losses depend on the variables, this indirectly tests variables are - updated correctly. - """ - - def setUp(self): - super(VariableUpdateTest, self).setUp() - _check_has_gpu() - benchmark_cnn.setup(benchmark_cnn.make_params()) - - def _get_benchmark_cnn_losses(self, inputs, params): - """Returns the losses of BenchmarkCNN on the given inputs and params.""" - logs = [] - model = test_util.TestCNNModel() - with test_util.monkey_patch(benchmark_cnn, - log_fn=test_util.print_and_add_to_list(logs), - LOSS_AND_ACCURACY_DIGITS_TO_SHOW=15): - bench = benchmark_cnn.BenchmarkCNN( - params, dataset=test_util.TestDataSet(), model=model) - # The test model does not use labels when computing loss, so the label - # values do not matter as long as it's the right shape. - labels = np.array([1] * inputs.shape[0]) - bench.input_preprocessor.set_fake_data(inputs, labels) - if bench.eval_input_preprocessor: - bench.eval_input_preprocessor.set_fake_data(inputs, labels) - bench.run() - - outputs = test_util.get_training_outputs_from_logs( - logs, params.print_training_accuracy) - return [x.loss for x in outputs] - - def _test_variable_update(self, params): - """Tests variables are updated correctly when the given params are used. - - A BenchmarkCNN is created with a TestCNNModel, and is run with some scalar - images. The losses are then compared with the losses obtained with - TestCNNModel().manually_compute_losses() - - Args: - params: a Params tuple used to create BenchmarkCNN. - """ - inputs = test_util.get_fake_var_update_inputs() - actual_losses = self._get_benchmark_cnn_losses(inputs, params) - expected_losses, = test_util.TestCNNModel().manually_compute_losses( - inputs, 1, params) - rtol = 3e-2 if params.use_fp16 else 1e-5 - self.assertAllClose(actual_losses[:len(expected_losses)], expected_losses, - rtol=rtol, atol=0.) - - def _test_variable_updates(self, params, - var_updates=('parameter_server', 'replicated')): - for var_update in var_updates: - self._test_variable_update(params._replace(variable_update=var_update)) - - def testDefault(self): - params = test_util.get_var_update_params() - self._test_variable_updates(params) - - # For some reason, this test doesn't always pass - - # def testCpuAsDevice(self): - # params = test_util.get_var_update_params()._replace( - # device='cpu', - # data_format='NHWC') # NHWC required when --device=cpu - # self._test_variable_updates(params) - - def testCpuAsLocalParamDevice(self): - params = test_util.get_var_update_params()._replace( - local_parameter_device='cpu') - self._test_variable_updates(params) - - def testFp16(self): - params = test_util.get_var_update_params()._replace(use_fp16=True) - self._test_variable_updates(params) - - def testMomentum(self): - params = test_util.get_var_update_params()._replace(optimizer='momentum') - self._test_variable_updates(params) - - def testRmsprop(self): - params = test_util.get_var_update_params()._replace(optimizer='rmsprop') - self._test_variable_updates(params) - - def testNoLayers(self): - params = test_util.get_var_update_params()._replace(use_tf_layers=False) - self._test_variable_updates(params) - - def testVariousAllReduceSpecs(self): - # We do not test xring, because it requires all Variables to have at least - # two elements. - params = test_util.get_var_update_params()._replace(all_reduce_spec='pscpu') - self._test_variable_updates(params, var_updates=('replicated',)) - params = params._replace(all_reduce_spec='psgpu') - self._test_variable_updates(params, var_updates=('replicated',)) - # TODO(b/80125832): Enable nccl in tests - # params = params._replace(all_reduce_spec='nccl', - # compact_gradient_transfer=False) - # self._test_variable_updates(params, var_updates=('replicated',)) - - def testPrintBaseLoss(self): - params = test_util.get_var_update_params()._replace( - loss_type_to_report='base_loss') - self._test_variable_updates(params) - - def testSingleL2LossOp(self): - params = test_util.get_var_update_params()._replace( - single_l2_loss_op=True) - self._test_variable_updates(params) - - def testResourceVars(self): - params = test_util.get_var_update_params()._replace( - use_resource_vars=True) - self._test_variable_updates(params) - - def testEvalDuringTrainingEveryNSteps(self): - # TODO(reedwm): Test that the eval results are correct. This only tests that - # training results are correct. - params = test_util.get_var_update_params()._replace( - eval_during_training_every_n_steps=1) - self._test_variable_updates(params, var_updates=('replicated',)) - - -class VariableMgrLocalReplicatedTest(tf.test.TestCase): - - def _test_grad_aggregation_with_var_mgr(self, variable_mgr, num_towers, - num_vars, deferred_grads): - tower_devices = ['/gpu:%d' % i for i in range(num_towers)] - tower_grads = [] - expected_sums = [0.] * num_vars - for i, tower_device in enumerate(tower_devices): - with tf.device(tower_device): - grad_vars = [] - for j in range(num_vars): - n = num_towers * i + j - grad_vars.append((tf.constant(n, dtype=tf.float32), - tf.Variable(n, dtype=tf.float32))) - expected_sums[j] += n - tower_grads.append(grad_vars) - - _, agg_device_grads = variable_mgr.preprocess_device_grads( - tower_grads) - expected_device_grads = [] - for i in range(num_towers): - expected_grad_vars = [] - for j in range(num_vars): - expected_grad_and_var = [expected_sums[j], num_towers * i + j] - if isinstance(agg_device_grads[i][j], tuple): - # agg_device_grads[i][j] can be a list or tuple. - expected_grad_and_var = tuple(expected_grad_and_var) - expected_grad_vars.append(expected_grad_and_var) - if isinstance(agg_device_grads[i], tuple): - # agg_device_grads[i] can be a list or tuple. - expected_grad_vars = tuple(expected_grad_vars) - expected_device_grads.append(expected_grad_vars) - config = tf.ConfigProto(allow_soft_placement=True) - with tf.Session(config=config) as sess: - sess.run(tf.initialize_all_variables()) - sess.run(variable_mgr._warmup_ops) - if deferred_grads: - # With deferred grads, the result of a session run is always the summed - # gradients from the previous session run. - sess.run(agg_device_grads) - feed_dict = {g: 0 for grad_vars in tower_grads for g, _ in grad_vars} - agg_device_grads_ = sess.run(agg_device_grads, feed_dict) - else: - agg_device_grads_ = sess.run(agg_device_grads) - self.assertEqual(agg_device_grads_, expected_device_grads) - - def _test_grad_aggregation(self, params, num_vars): - bench = benchmark_cnn.BenchmarkCNN(params) - deferred_grads = (params.variable_consistency == 'relaxed') - self._test_grad_aggregation_with_var_mgr(bench.variable_mgr, bench.num_gpus, - num_vars, deferred_grads) - - def test_grad_aggregation(self): - base_params = benchmark_cnn.make_params(num_gpus=10, - variable_update='replicated', - use_fp16=True) - params = base_params - self._test_grad_aggregation(params, 10) - params = base_params._replace(gradient_repacking=3) - self._test_grad_aggregation(params, 10) - params = base_params._replace(variable_consistency='relaxed') - self._test_grad_aggregation(params, 10) - params = base_params._replace(compact_gradient_transfer=False) - self._test_grad_aggregation(params, 10) - params = base_params._replace(gradient_repacking=3, - variable_consistency='relaxed') - self._test_grad_aggregation(params, 10) - params = base_params._replace(gradient_repacking=3, - compact_gradient_transfer=False) - self._test_grad_aggregation(params, 10) - params = base_params._replace(variable_consistency='relaxed', - compact_gradient_transfer=False) - self._test_grad_aggregation(params, 10) - params = base_params._replace(gradient_repacking=3, - variable_consistency='relaxed', - compact_gradient_transfer=False) - self._test_grad_aggregation(params, 10) - params = base_params._replace(num_gpus=8, hierarchical_copy=True) - self._test_grad_aggregation(params, 10) - # TODO(b/80125832): Enable nccl in tests - # params = base_params._replace(all_reduce_spec='nccl', - # compact_gradient_transfer=False, - # # For some reason, this test freezes when - # # num_gpus=10 - # num_gpus=8) - # self._test_grad_aggregation(params, 10) - params = base_params._replace(all_reduce_spec='pscpu') - self._test_grad_aggregation(params, 10) - - params = base_params._replace(num_gpus=8, - gradient_repacking=3, - variable_consistency='relaxed', - hierarchical_copy=True) - self._test_grad_aggregation(params, 10) - # TODO(b/80125832): Enable nccl in tests - # params = base_params._replace(num_gpus=8, - # gradient_repacking=3, - # variable_consistency='relaxed', - # all_reduce_spec='nccl', - # compact_gradient_transfer=False) - # self._test_grad_aggregation(params, 10) - params = base_params._replace(gradient_repacking=3, - variable_consistency='relaxed', - all_reduce_spec='pscpu') - self._test_grad_aggregation(params, 10) - params = base_params._replace(gradient_repacking=3, - variable_consistency='relaxed', - all_reduce_spec='xring') - self._test_grad_aggregation(params, 10) - - -if __name__ == '__main__': - tf.disable_v2_behavior() - tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/cnn_util.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/cnn_util.py deleted file mode 100644 index 2cde5104fdc1b8a2f4a474e543974f5e42689052..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/cnn_util.py +++ /dev/null @@ -1,253 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Utilities for CNN benchmarks.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import sys -import threading - -import numpy as np -import tensorflow.compat.v1 as tf - - -def tensorflow_version_tuple(): - v = tf.__version__ - major, minor, patch = v.split('.') - return (int(major), int(minor), patch) - - -def tensorflow_version(): - vt = tensorflow_version_tuple() - return vt[0] * 1000 + vt[1] - - -def log_fn(log): - print(log, flush=True) - - -def roll_numpy_batches(array, batch_size, shift_ratio): - """Moves a proportion of batches from start to the end of the array. - - This function moves a proportion of batches, specified by `shift_ratio`, from - the starts of the array to the end. The number of batches moved is rounded - down to the nearest integer. For example, - - ``` - roll_numpy_batches([1, 2, 3, 4, 5, 6], 2, 0.34) == [3, 4, 5, 6, 1, 2] - ``` - - Args: - array: A Numpy array whose first dimension is the batch dimension. - batch_size: The batch size. - shift_ratio: Proportion of batches to move from the start of the array to - the end of the array. - Returns: - A new Numpy array, with a proportion of the batches at the start of `array` - moved to the end. - """ - num_items = array.shape[0] - assert num_items % batch_size == 0 - num_batches = num_items // batch_size - starting_batch = int(num_batches * shift_ratio) - starting_item = starting_batch * batch_size - return np.roll(array, -starting_item, axis=0) - - -# For Python 2.7 compatibility, we do not use threading.Barrier. -class Barrier(object): - """Implements a lightweight Barrier. - - Useful for synchronizing a fixed number of threads at known synchronization - points. Threads block on 'wait()' and simultaneously return once they have - all made that call. - - # Implementation adopted from boost/thread/barrier.hpp - """ - - def __init__(self, parties): - """Create a barrier, initialised to 'parties' threads.""" - self.cond = threading.Condition(threading.Lock()) - self.parties = parties - # Indicates the number of waiting parties. - self.waiting = 0 - # generation is needed to deal with spurious wakeups. If self.cond.wait() - # wakes up for other reasons, generation will force it go back to wait(). - self.generation = 0 - self.broken = False - - def wait(self): - """Wait for the barrier.""" - with self.cond: - # Check if the barrier has been disabled or not. - if self.broken: - return - gen = self.generation - self.waiting += 1 - if self.waiting == self.parties: - self.waiting = 0 - self.generation += 1 - self.cond.notify_all() - # loop because of spurious wakeups - while gen == self.generation: - self.cond.wait() - - # TODO(huangyp): Remove this method once we find a way to know which step - # is the last barrier. - def abort(self): - """Clear existing barrier and disable this barrier.""" - with self.cond: - if self.waiting > 0: - self.generation += 1 - self.cond.notify_all() - self.broken = True - - -class ImageProducer(object): - """An image producer that puts images into a staging area periodically. - - This class is useful for periodically running a set of ops, `put_ops` on a - different thread every `batch_group_size` steps. - - The notify_image_consumption() method is used to increment an internal counter - so that every `batch_group_size` times it is called, `put_ops` is executed. A - barrier is placed so that notify_image_consumption() will block until - the previous call to `put_ops` has been executed. - - The start() method is used to start the thread that runs `put_ops`. - - The done() method waits until the last put_ops is executed and stops the - thread. - - The purpose of this class is to fill an image input pipeline every - `batch_group_size` steps. Suppose `put_ops` supplies `batch_group_size` images - to the input pipeline when run, and that every step, 1 batch of images is - consumed. Then, by calling notify_image_consumption() every step, images are - supplied to the input pipeline at the same amount they are consumed. - - Example usage: - ``` - put_ops = ... # Enqueues `batch_group_size` batches to a StagingArea - get_op = ... # Dequeues 1 batch, and does some operations on it - batch_group_size = 4 - with tf.Session() as sess: - image_producer = cnn_util.ImageProducer(sess, put_op, batch_group_size) - image_producer.start() - for _ in range(100): - sess.run(get_op) - image_producer.notify_image_consumption() - ``` - """ - - def __init__(self, sess, put_ops, batch_group_size, use_python32_barrier): - self.sess = sess - self.num_gets = 0 - self.put_ops = put_ops - self.batch_group_size = batch_group_size - self.done_event = threading.Event() - if (use_python32_barrier and - sys.version_info[0] == 3 and sys.version_info[1] >= 2): - self.put_barrier = threading.Barrier(2) - else: - self.put_barrier = Barrier(2) - - def _should_put(self): - return (self.num_gets + 1) % self.batch_group_size == 0 - - def done(self): - """Stop the image producer.""" - self.done_event.set() - self.put_barrier.abort() - self.thread.join() - - def start(self): - """Start the image producer.""" - self.sess.run([self.put_ops]) - self.thread = threading.Thread(target=self._loop_producer) - # Set daemon to true to allow Ctrl + C to terminate all threads. - self.thread.daemon = True - self.thread.start() - - def notify_image_consumption(self): - """Increment the counter of image_producer by 1. - - This should only be called by the main thread that consumes images and runs - the model computation. One batch of images should be consumed between - calling start() and the first call to this method. Then, one batch of images - should be consumed between any two successive calls to this method. - """ - if self._should_put(): - self.put_barrier.wait() - self.num_gets += 1 - - def _loop_producer(self): - while not self.done_event.isSet(): - self.sess.run([self.put_ops]) - self.put_barrier.wait() - - -class BaseClusterManager(object): - """The manager for the cluster of servers running the benchmark.""" - - def __init__(self, params): - worker_hosts = params.worker_hosts.split(',') - ps_hosts = params.ps_hosts.split(',') if params.ps_hosts else [] - cluster = {'worker': worker_hosts} - if ps_hosts: - cluster['ps'] = ps_hosts - self._cluster_spec = tf.train.ClusterSpec(cluster) - - def get_target(self): - """Returns a target to be passed to tf.Session().""" - raise NotImplementedError('get_target must be implemented by subclass') - - def join_server(self): - raise NotImplementedError('join must be implemented by subclass') - - def get_cluster_spec(self): - return self._cluster_spec - - def num_workers(self): - return len(self._cluster_spec.job_tasks('worker')) - - def num_ps(self): - if 'ps' in self._cluster_spec.jobs: - return len(self._cluster_spec.job_tasks('ps')) - else: - return 0 - - -class GrpcClusterManager(BaseClusterManager): - """A cluster manager for a cluster networked with gRPC.""" - - def __init__(self, params, config_proto): - super(GrpcClusterManager, self).__init__(params) - if params.job_name == 'controller': - self._target = 'grpc://%s' % self._cluster_spec.job_tasks('worker')[0] - else: - self._server = tf.train.Server(self._cluster_spec, - job_name=params.job_name, - task_index=params.task_index, - config=config_proto, - protocol=params.server_protocol) - self._target = self._server.target - - def get_target(self): - return self._target - - def join_server(self): - return self._server.join() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/cnn_util_test.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/cnn_util_test.py deleted file mode 100644 index 7c245afbf8de9d72f8b9287e5a104f1ffd42bde8..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/cnn_util_test.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Tests for tf_cnn_benchmarks.cnn_util.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import threading -import time - -import tensorflow.compat.v1 as tf - -import cnn_util - - -class CnnUtilBarrierTest(tf.test.TestCase): - - def testBarrier(self): - num_tasks = 20 - num_waits = 4 - barrier = cnn_util.Barrier(num_tasks) - threads = [] - sync_matrix = [] - for i in range(num_tasks): - sync_times = [0] * num_waits - thread = threading.Thread( - target=self._run_task, args=(barrier, sync_times)) - thread.start() - threads.append(thread) - sync_matrix.append(sync_times) - for thread in threads: - thread.join() - for wait_index in range(num_waits - 1): - # Max of times at iteration i < min of times at iteration i + 1 - self.assertLessEqual( - max([sync_matrix[i][wait_index] for i in range(num_tasks)]), - min([sync_matrix[i][wait_index + 1] for i in range(num_tasks)])) - - def _run_task(self, barrier, sync_times): - for wait_index in range(len(sync_times)): - sync_times[wait_index] = time.time() - barrier.wait() - - def testBarrierAbort(self): - num_tasks = 2 - num_waits = 1 - sync_times = [0] * num_waits - barrier = cnn_util.Barrier(num_tasks) - thread = threading.Thread( - target=self._run_task, args=(barrier, sync_times)) - thread.start() - barrier.abort() - # thread won't be blocked by done barrier. - thread.join() - - -class ImageProducerTest(tf.test.TestCase): - - def _slow_tensorflow_op(self): - """Returns a TensorFlow op that takes approximately 0.1s to complete.""" - def slow_func(v): - time.sleep(0.1) - return v - return tf.py_func(slow_func, [tf.constant(0.)], tf.float32).op - - def _test_image_producer(self, batch_group_size, put_slower_than_get): - # We use the variable x to simulate a staging area of images. x represents - # the number of batches in the staging area. - x = tf.Variable(0, dtype=tf.int32) - if put_slower_than_get: - put_dep = self._slow_tensorflow_op() - get_dep = tf.no_op() - else: - put_dep = tf.no_op() - get_dep = self._slow_tensorflow_op() - with tf.control_dependencies([put_dep]): - put_op = x.assign_add(batch_group_size, use_locking=True) - with tf.control_dependencies([get_dep]): - get_op = x.assign_sub(1, use_locking=True) - with self.test_session() as sess: - sess.run(tf.variables_initializer([x])) - image_producer = cnn_util.ImageProducer(sess, put_op, batch_group_size, - use_python32_barrier=False) - image_producer.start() - for _ in range(5 * batch_group_size): - sess.run(get_op) - # We assert x is nonnegative, to ensure image_producer never causes - # an unstage op to block. We assert x is at most 2 * batch_group_size, - # to ensure it doesn't use too much memory by storing too many batches - # in the staging area. - self.assertGreaterEqual(sess.run(x), 0) - self.assertLessEqual(sess.run(x), 2 * batch_group_size) - image_producer.notify_image_consumption() - self.assertGreaterEqual(sess.run(x), 0) - self.assertLessEqual(sess.run(x), 2 * batch_group_size) - - image_producer.done() - time.sleep(0.1) - self.assertGreaterEqual(sess.run(x), 0) - self.assertLessEqual(sess.run(x), 2 * batch_group_size) - - def test_image_producer(self): - self._test_image_producer(1, False) - self._test_image_producer(1, True) - self._test_image_producer(2, False) - self._test_image_producer(2, True) - self._test_image_producer(3, False) - self._test_image_producer(3, True) - self._test_image_producer(8, False) - self._test_image_producer(8, True) - - -if __name__ == '__main__': - tf.disable_v2_behavior() - tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/coco_metric.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/coco_metric.py deleted file mode 100644 index d8ba67da47c79da96ec3d96feae91169cac7509c..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/coco_metric.py +++ /dev/null @@ -1,198 +0,0 @@ -# Copyright 2018 Google. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""COCO-style evaluation metrics. - -Forked from reference model implementation. - -COCO API: github.com/cocodataset/cocoapi/ -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import atexit -import tempfile - -from absl import flags - -import numpy as np -from pycocotools.coco import COCO -from pycocotools.cocoeval import COCOeval -import six - -import tensorflow.compat.v1 as tf - -import mlperf -import ssd_constants - -FLAGS = flags.FLAGS - - -# https://github.com/cocodataset/cocoapi/issues/49 -if six.PY3: - import pycocotools.coco - pycocotools.coco.unicode = str - - -def async_eval_runner(queue_predictions, queue_results, val_json_file): - """Load intermediate eval results and get COCO metrics.""" - while True: - message = queue_predictions.get() - if message == 'STOP': # poison pill - break - step, predictions = message - results = compute_map(predictions, val_json_file) - queue_results.put((step, results)) - - -def compute_map(predictions, val_json_file): - """Use model predictions to compute mAP. - - Args: - predictions: a list of tuples returned by decoded_predictions function, - each containing the following elements: - image source_id, box coordinates in XYWH order, probability score, label - val_json_file: path to COCO annotation file - Returns: - A dictionary that maps all COCO metrics (keys) to their values - """ - - if val_json_file.startswith("gs://"): - _, local_val_json = tempfile.mkstemp(suffix=".json") - tf.gfile.Remove(local_val_json) - - tf.gfile.Copy(val_json_file, local_val_json) - atexit.register(tf.gfile.Remove, local_val_json) - else: - local_val_json = val_json_file - - cocoGt = COCO(local_val_json) - cocoDt = cocoGt.loadRes(np.array(predictions)) - E = COCOeval(cocoGt, cocoDt, iouType='bbox') - E.evaluate() - E.accumulate() - E.summarize() - print("Current AP: {:.5f}".format(E.stats[0])) - metric_names = ['AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'ARmax1', - 'ARmax10', 'ARmax100', 'ARs', 'ARm', 'ARl'] - - # Prefix with "COCO" to group in TensorBoard. - return {"COCO/" + key: value for key, value in zip(metric_names, E.stats)} - - -def calc_iou(target, candidates): - target_tiled = np.tile(target[np.newaxis, :], (candidates.shape[0], 1)) - # Left Top & Right Bottom - lt = np.maximum(target_tiled[:,:2], candidates[:,:2]) - - rb = np.minimum(target_tiled[:,2:], candidates[:,2:]) - - delta = np.maximum(rb - lt, 0) - - intersect = delta[:,0] * delta[:,1] - - delta1 = target_tiled[:,2:] - candidates[:,:2] - area1 = delta1[:,0] * delta1[:,1] - delta2 = target_tiled[:,2:] - candidates[:,:2] - area2 = delta2[:,0] * delta2[:,1] - - iou = intersect/(area1 + area2 - intersect) - return iou - - -# TODO(haoyuzhang): Rewrite this NumPy based implementation to TensorFlow based -# implementation under ssd_model.py accuracy_function. -def decode_predictions(labels_and_predictions): - """Decode predictions and remove unused boxes and labels.""" - predictions = [] - for example in labels_and_predictions: - source_id = int(example[ssd_constants.SOURCE_ID]) - pred_box = example[ssd_constants.PRED_BOXES] - pred_scores = example[ssd_constants.PRED_SCORES] - - locs, labels, probs = decode_single( - pred_box, pred_scores, ssd_constants.OVERLAP_CRITERIA, - ssd_constants.MAX_NUM_EVAL_BOXES, ssd_constants.MAX_NUM_EVAL_BOXES) - - raw_height, raw_width, _ = example[ssd_constants.RAW_SHAPE] - for loc, label, prob in zip(locs, labels, probs): - # Ordering convention differs, hence [1], [0] rather than [0], [1] - x, y = loc[1] * raw_width, loc[0] * raw_height - w, h = (loc[3] - loc[1]) * raw_width, (loc[2] - loc[0]) * raw_height - predictions.append( - [source_id, x, y, w, h, prob, ssd_constants.CLASS_INV_MAP[label]]) - mlperf.logger.log(key=mlperf.tags.NMS_THRESHOLD, - value=ssd_constants.OVERLAP_CRITERIA) - mlperf.logger.log(key=mlperf.tags.NMS_MAX_DETECTIONS, - value=ssd_constants.MAX_NUM_EVAL_BOXES) - return predictions - - -def decode_single(bboxes_in, scores_in, criteria, max_output, max_num=200): - # Reference to https://github.com/amdegroot/ssd.pytorch - - bboxes_out = [] - scores_out = [] - labels_out = [] - - for i, score in enumerate(np.split(scores_in, scores_in.shape[1], 1)): - score = np.squeeze(score, 1) - - # skip background - if i == 0: - continue - - mask = score > ssd_constants.MIN_SCORE - if not np.any(mask): - continue - - bboxes, score = bboxes_in[mask, :], score[mask] - - score_idx_sorted = np.argsort(score) - score_sorted = score[score_idx_sorted] - - score_idx_sorted = score_idx_sorted[-max_num:] - candidates = [] - - # perform non-maximum suppression - while len(score_idx_sorted): - idx = score_idx_sorted[-1] - bboxes_sorted = bboxes[score_idx_sorted, :] - bboxes_idx = bboxes[idx, :] - iou = calc_iou(bboxes_idx, bboxes_sorted) - - score_idx_sorted = score_idx_sorted[iou < criteria] - candidates.append(idx) - - bboxes_out.append(bboxes[candidates, :]) - scores_out.append(score[candidates]) - labels_out.extend([i]*len(candidates)) - - if len(scores_out) == 0: - tf.logging.info("No objects detected. Returning dummy values.") - return ( - np.zeros(shape=(1, 4), dtype=np.float32), - np.zeros(shape=(1,), dtype=np.int32), - np.ones(shape=(1,), dtype=np.float32) * ssd_constants.DUMMY_SCORE, - ) - - bboxes_out = np.concatenate(bboxes_out, axis=0) - scores_out = np.concatenate(scores_out, axis=0) - labels_out = np.array(labels_out) - - max_ids = np.argsort(scores_out)[-max_output:] - - return bboxes_out[max_ids, :], labels_out[max_ids], scores_out[max_ids] diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/constants.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/constants.py deleted file mode 100644 index dbb32271bb2669e0ba12588d87d39f7c8924b161..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/constants.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Constants used in tf_cnn_benchmarks.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from enum import Enum - -# Results fetched with this prefix will not be reduced. Instead, they will be -# passed as matrices to model's postprocess function. -UNREDUCED_ACCURACY_OP_PREFIX = "tensor:" - -# Eval result values with this name prefix will be included in summary. -SIMPLE_VALUE_RESULT_PREFIX = "simple_value:" - - -class BenchmarkMode(object): - """Benchmark running mode.""" - TRAIN = "training" - EVAL = "evaluation" - TRAIN_AND_EVAL = "training + evaluation" - FORWARD_ONLY = "forward only" - - -class NetworkTopology(str, Enum): - """Network topology describes how multiple GPUs are inter-connected. - """ - # DGX-1 uses hybrid cube mesh topology with the following device peer to peer - # matrix: - # DMA: 0 1 2 3 4 5 6 7 - # 0: Y Y Y Y Y N N N - # 1: Y Y Y Y N Y N N - # 2: Y Y Y Y N N Y N - # 3: Y Y Y Y N N N Y - # 4: Y N N N Y Y Y Y - # 5: N Y N N Y Y Y Y - # 6: N N Y N Y Y Y Y - # 7: N N N Y Y Y Y Y - DGX1 = "dgx1" - - # V100 in GCP are connected with the following device peer to peer matrix. - # In this topology, bandwidth of the connection depends on if it uses NVLink - # or PCIe link. - # DMA: 0 1 2 3 4 5 6 7 - # 0: Y Y Y Y N Y N N - # 1: Y Y Y Y N N N N - # 2: Y Y Y Y N N N Y - # 3: Y Y Y Y N N N N - # 4: N N N N Y Y Y Y - # 5: Y N N N Y Y Y Y - # 6: N N N N Y Y Y Y - # 7: N N Y N Y Y Y Y - GCP_V100 = "gcp_v100" diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/convnet_builder.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/convnet_builder.py deleted file mode 100644 index 9903de9247e7401b2982bb061fb6f4bdce7be179..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/convnet_builder.py +++ /dev/null @@ -1,498 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""CNN builder.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from collections import defaultdict -import contextlib - -import numpy as np - -import tensorflow.compat.v1 as tf - -# pylint: disable=g-direct-tensorflow-import -import mlperf -from tensorflow.python.layers import convolutional as conv_layers -from tensorflow.python.layers import core as core_layers -from tensorflow.python.layers import normalization as normalization_layers -from tensorflow.python.layers import pooling as pooling_layers -from tensorflow.python.training import moving_averages - - -_data_format_to_channel_axis = {'NCHW': 1, 'NHWC': 3} - - -class ConvNetBuilder(object): - """Builder of cnn net.""" - - def __init__(self, - input_op, - input_nchan, - phase_train, - use_tf_layers, - data_format='NCHW', - dtype=tf.float32, - variable_dtype=tf.float32): - self.top_layer = input_op - self.top_size = input_nchan - self.phase_train = phase_train - self.use_tf_layers = use_tf_layers - self.data_format = data_format - self.dtype = dtype - self.variable_dtype = variable_dtype - self.counts = defaultdict(lambda: 0) - self.use_batch_norm = False - self.batch_norm_config = {} # 'decay': 0.997, 'scale': True} - self.channel_pos = ('channels_last' - if data_format == 'NHWC' else 'channels_first') - self.aux_top_layer = None - self.aux_top_size = 0 - - def get_custom_getter(self): - """Returns a custom getter that this class's methods must be called under. - - All methods of this class must be called under a variable scope that was - passed this custom getter. Example: - - ```python - network = ConvNetBuilder(...) - with tf.variable_scope('cg', custom_getter=network.get_custom_getter()): - network.conv(...) - # Call more methods of network here - ``` - - Currently, this custom getter only does anything if self.use_tf_layers is - True. In that case, it causes variables to be stored as dtype - self.variable_type, then casted to the requested dtype, instead of directly - storing the variable as the requested dtype. - """ - def inner_custom_getter(getter, *args, **kwargs): - """Custom getter that forces variables to have type self.variable_type.""" - if not self.use_tf_layers: - return getter(*args, **kwargs) - requested_dtype = kwargs['dtype'] - if not (requested_dtype == tf.float32 and - self.variable_dtype == tf.float16): - # Only change the variable dtype if doing so does not decrease variable - # precision. - kwargs['dtype'] = self.variable_dtype - var = getter(*args, **kwargs) - # This if statement is needed to guard the cast, because batch norm - # assigns directly to the return value of this custom getter. The cast - # makes the return value not a variable so it cannot be assigned. Batch - # norm variables are always in fp32 so this if statement is never - # triggered for them. - if var.dtype.base_dtype != requested_dtype: - var = tf.cast(var, requested_dtype) - return var - return inner_custom_getter - - @contextlib.contextmanager - def switch_to_aux_top_layer(self): - """Context that construct cnn in the auxiliary arm.""" - if self.aux_top_layer is None: - raise RuntimeError('Empty auxiliary top layer in the network.') - saved_top_layer = self.top_layer - saved_top_size = self.top_size - self.top_layer = self.aux_top_layer - self.top_size = self.aux_top_size - yield - self.aux_top_layer = self.top_layer - self.aux_top_size = self.top_size - self.top_layer = saved_top_layer - self.top_size = saved_top_size - - def get_variable(self, name, shape, dtype, cast_dtype, *args, **kwargs): - # TODO(reedwm): Currently variables and gradients are transferred to other - # devices and machines as type `dtype`, not `cast_dtype`. In particular, - # this means in fp16 mode, variables are transferred as fp32 values, not - # fp16 values, which uses extra bandwidth. - var = tf.get_variable(name, shape, dtype, *args, **kwargs) - return tf.cast(var, cast_dtype) - - def _conv2d_impl(self, input_layer, num_channels_in, filters, kernel_size, - strides, padding, kernel_initializer): - if self.use_tf_layers: - return conv_layers.conv2d(input_layer, filters, kernel_size, strides, - padding, self.channel_pos, - kernel_initializer=kernel_initializer, - use_bias=False) - else: - weights_shape = [kernel_size[0], kernel_size[1], num_channels_in, filters] - # We use the name 'conv2d/kernel' so the variable has the same name as its - # tf.layers equivalent. This way, if a checkpoint is written when - # self.use_tf_layers == True, it can be loaded when - # self.use_tf_layers == False, and vice versa. - weights = self.get_variable('conv2d/kernel', weights_shape, - self.variable_dtype, self.dtype, - initializer=kernel_initializer) - if self.data_format == 'NHWC': - strides = [1] + strides + [1] - else: - strides = [1, 1] + strides - return tf.nn.conv2d(input_layer, weights, strides, padding, - data_format=self.data_format) - - def conv(self, - num_out_channels, - k_height, - k_width, - d_height=1, - d_width=1, - mode='SAME', - input_layer=None, - num_channels_in=None, - use_batch_norm=None, - stddev=None, - activation='relu', - bias=0.0, - kernel_initializer=None): - """Construct a conv2d layer on top of cnn.""" - if input_layer is None: - input_layer = self.top_layer - if num_channels_in is None: - num_channels_in = self.top_size - if stddev is not None and kernel_initializer is None: - kernel_initializer = tf.truncated_normal_initializer(stddev=stddev) - if kernel_initializer is None: - kernel_initializer = tf.variance_scaling_initializer() - name = 'conv' + str(self.counts['conv']) - self.counts['conv'] += 1 - with tf.variable_scope(name): - strides = [1, d_height, d_width, 1] - if self.data_format == 'NCHW': - strides = [strides[0], strides[3], strides[1], strides[2]] - if mode != 'SAME_RESNET': - conv = self._conv2d_impl(input_layer, num_channels_in, num_out_channels, - kernel_size=[k_height, k_width], - strides=[d_height, d_width], padding=mode, - kernel_initializer=kernel_initializer) - else: # Special padding mode for ResNet models - if d_height == 1 and d_width == 1: - conv = self._conv2d_impl(input_layer, num_channels_in, - num_out_channels, - kernel_size=[k_height, k_width], - strides=[d_height, d_width], padding='SAME', - kernel_initializer=kernel_initializer) - else: - rate = 1 # Unused (for 'a trous' convolutions) - kernel_height_effective = k_height + (k_height - 1) * (rate - 1) - pad_h_beg = (kernel_height_effective - 1) // 2 - pad_h_end = kernel_height_effective - 1 - pad_h_beg - kernel_width_effective = k_width + (k_width - 1) * (rate - 1) - pad_w_beg = (kernel_width_effective - 1) // 2 - pad_w_end = kernel_width_effective - 1 - pad_w_beg - padding = [[0, 0], [pad_h_beg, pad_h_end], - [pad_w_beg, pad_w_end], [0, 0]] - if self.data_format == 'NCHW': - padding = [padding[0], padding[3], padding[1], padding[2]] - padded_input_layer = tf.pad(input_layer, padding) - conv = self._conv2d_impl(padded_input_layer, num_channels_in, - num_out_channels, - kernel_size=[k_height, k_width], - strides=[d_height, d_width], padding='VALID', - kernel_initializer=kernel_initializer) - if use_batch_norm is None: - use_batch_norm = self.use_batch_norm - mlperf.logger.log_conv2d(input_tensor=input_layer, output_tensor=conv, - stride_height=d_height, stride_width=d_width, - filters=num_out_channels, - initializer=kernel_initializer, - use_bias=not use_batch_norm and bias is not None) - if not use_batch_norm: - if bias is not None: - biases = self.get_variable('biases', [num_out_channels], - self.variable_dtype, self.dtype, - initializer=tf.constant_initializer(bias)) - biased = tf.reshape( - tf.nn.bias_add(conv, biases, data_format=self.data_format), - conv.get_shape()) - else: - biased = conv - else: - self.top_layer = conv - self.top_size = num_out_channels - biased = self.batch_norm(**self.batch_norm_config) - if activation == 'relu': - mlperf.logger.log(key=mlperf.tags.MODEL_HP_RELU) - conv1 = tf.nn.relu(biased) - elif activation == 'linear' or activation is None: - conv1 = biased - elif activation == 'tanh': - conv1 = tf.nn.tanh(biased) - else: - raise KeyError('Invalid activation type \'%s\'' % activation) - self.top_layer = conv1 - self.top_size = num_out_channels - return conv1 - - def _pool(self, - pool_name, - pool_function, - k_height, - k_width, - d_height, - d_width, - mode, - input_layer, - num_channels_in): - """Construct a pooling layer.""" - if input_layer is None: - input_layer = self.top_layer - else: - self.top_size = num_channels_in - name = pool_name + str(self.counts[pool_name]) - self.counts[pool_name] += 1 - if self.use_tf_layers: - pool = pool_function( - input_layer, [k_height, k_width], [d_height, d_width], - padding=mode, - data_format=self.channel_pos, - name=name) - else: - if self.data_format == 'NHWC': - ksize = [1, k_height, k_width, 1] - strides = [1, d_height, d_width, 1] - else: - ksize = [1, 1, k_height, k_width] - strides = [1, 1, d_height, d_width] - pool = tf.nn.max_pool(input_layer, ksize, strides, padding=mode, - data_format=self.data_format, name=name) - if pool_name == 'mpool': - mlperf.logger.log_max_pool(input_tensor=input_layer, - output_tensor=pool) - self.top_layer = pool - return pool - - def mpool(self, - k_height, - k_width, - d_height=2, - d_width=2, - mode='VALID', - input_layer=None, - num_channels_in=None): - """Construct a max pooling layer.""" - return self._pool('mpool', pooling_layers.max_pooling2d, k_height, k_width, - d_height, d_width, mode, input_layer, num_channels_in) - - def apool(self, - k_height, - k_width, - d_height=2, - d_width=2, - mode='VALID', - input_layer=None, - num_channels_in=None): - """Construct an average pooling layer.""" - return self._pool('apool', pooling_layers.average_pooling2d, k_height, - k_width, d_height, d_width, mode, input_layer, - num_channels_in) - - def reshape(self, shape, input_layer=None): - if input_layer is None: - input_layer = self.top_layer - self.top_layer = tf.reshape(input_layer, shape) - self.top_size = shape[-1] # HACK This may not always work - return self.top_layer - - def affine(self, - num_out_channels, - input_layer=None, - num_channels_in=None, - bias=0.0, - stddev=None, - activation='relu'): - if input_layer is None: - input_layer = self.top_layer - if num_channels_in is None: - num_channels_in = self.top_size - name = 'affine' + str(self.counts['affine']) - self.counts['affine'] += 1 - with tf.variable_scope(name): - init_factor = 2. if activation == 'relu' else 1. - stddev = stddev or np.sqrt(init_factor / num_channels_in) - kernel = self.get_variable( - 'weights', [num_channels_in, num_out_channels], - self.variable_dtype, self.dtype, - initializer=tf.truncated_normal_initializer(stddev=stddev)) - biases = self.get_variable('biases', [num_out_channels], - self.variable_dtype, self.dtype, - initializer=tf.constant_initializer(bias)) - mlperf.logger.log(key=mlperf.tags.MODEL_HP_DENSE, - value=num_out_channels) - logits = tf.nn.xw_plus_b(input_layer, kernel, biases) - if activation == 'relu': - mlperf.logger.log(key=mlperf.tags.MODEL_HP_RELU) - affine1 = tf.nn.relu(logits, name=name) - elif activation == 'linear' or activation is None: - affine1 = logits - else: - raise KeyError('Invalid activation type \'%s\'' % activation) - self.top_layer = affine1 - self.top_size = num_out_channels - return affine1 - - def inception_module(self, name, cols, input_layer=None, in_size=None): - if input_layer is None: - input_layer = self.top_layer - if in_size is None: - in_size = self.top_size - name += str(self.counts[name]) - self.counts[name] += 1 - with tf.variable_scope(name): - col_layers = [] - col_layer_sizes = [] - for c, col in enumerate(cols): - col_layers.append([]) - col_layer_sizes.append([]) - for l, layer in enumerate(col): - ltype, args = layer[0], layer[1:] - kwargs = { - 'input_layer': input_layer, - 'num_channels_in': in_size - } if l == 0 else {} - if ltype == 'conv': - self.conv(*args, **kwargs) - elif ltype == 'mpool': - self.mpool(*args, **kwargs) - elif ltype == 'apool': - self.apool(*args, **kwargs) - elif ltype == 'share': # Share matching layer from previous column - self.top_layer = col_layers[c - 1][l] - self.top_size = col_layer_sizes[c - 1][l] - else: - raise KeyError( - 'Invalid layer type for inception module: \'%s\'' % ltype) - col_layers[c].append(self.top_layer) - col_layer_sizes[c].append(self.top_size) - catdim = 3 if self.data_format == 'NHWC' else 1 - self.top_layer = tf.concat([layers[-1] for layers in col_layers], catdim) - self.top_size = sum([sizes[-1] for sizes in col_layer_sizes]) - return self.top_layer - - def spatial_mean(self, keep_dims=False): - name = 'spatial_mean' + str(self.counts['spatial_mean']) - self.counts['spatial_mean'] += 1 - axes = [1, 2] if self.data_format == 'NHWC' else [2, 3] - self.top_layer = tf.reduce_mean( - self.top_layer, axes, keepdims=keep_dims, name=name) - return self.top_layer - - def dropout(self, keep_prob=0.5, input_layer=None): - if input_layer is None: - input_layer = self.top_layer - else: - self.top_size = None - name = 'dropout' + str(self.counts['dropout']) - with tf.variable_scope(name): - if not self.phase_train: - keep_prob = 1.0 - if self.use_tf_layers: - dropout = core_layers.dropout(input_layer, 1. - keep_prob, - training=self.phase_train) - else: - dropout = tf.nn.dropout(input_layer, keep_prob) - self.top_layer = dropout - return dropout - - def _batch_norm_without_layers(self, input_layer, decay, use_scale, epsilon): - """Batch normalization on `input_layer` without tf.layers.""" - # We make this function as similar as possible to the - # tf.contrib.layers.batch_norm, to minimize the differences between using - # layers and not using layers. - shape = input_layer.shape - num_channels = shape[3] if self.data_format == 'NHWC' else shape[1] - beta = self.get_variable('beta', [num_channels], tf.float32, tf.float32, - initializer=tf.zeros_initializer()) - if use_scale: - gamma = self.get_variable('gamma', [num_channels], tf.float32, - tf.float32, initializer=tf.ones_initializer()) - else: - gamma = tf.constant(1.0, tf.float32, [num_channels]) - # For moving variables, we use tf.get_variable instead of self.get_variable, - # since self.get_variable returns the result of tf.cast which we cannot - # assign to. - moving_mean = tf.get_variable('moving_mean', [num_channels], - tf.float32, - initializer=tf.zeros_initializer(), - trainable=False) - moving_variance = tf.get_variable('moving_variance', [num_channels], - tf.float32, - initializer=tf.ones_initializer(), - trainable=False) - if self.phase_train: - bn, batch_mean, batch_variance = tf.nn.fused_batch_norm( - input_layer, gamma, beta, epsilon=epsilon, - data_format=self.data_format, is_training=True) - mean_update = moving_averages.assign_moving_average( - moving_mean, batch_mean, decay=decay, zero_debias=False) - variance_update = moving_averages.assign_moving_average( - moving_variance, batch_variance, decay=decay, zero_debias=False) - tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, mean_update) - tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, variance_update) - else: - bn, _, _ = tf.nn.fused_batch_norm( - input_layer, gamma, beta, mean=moving_mean, - variance=moving_variance, epsilon=epsilon, - data_format=self.data_format, is_training=False) - return bn - - def batch_norm(self, input_layer=None, decay=0.999, scale=False, - epsilon=0.001): - """Adds a Batch Normalization layer.""" - if input_layer is None: - input_layer = self.top_layer - else: - self.top_size = None - name = 'batchnorm' + str(self.counts['batchnorm']) - self.counts['batchnorm'] += 1 - - center = True - with tf.variable_scope(name) as scope: - if self.use_tf_layers: - layer_obj = normalization_layers.BatchNormalization( - momentum=decay, - scale=scale, - epsilon=epsilon, - fused=True, - axis=_data_format_to_channel_axis[self.data_format], - # We pass this 'scope' argument for compatibility with checkpoints - # created with the contrib version of batch norm. tf_cnn_benchmarks - # used to use the contrib version. - _scope=scope, - center=center, - name=scope.name) - bn = layer_obj.apply(input_layer, training=self.phase_train) - else: - bn = self._batch_norm_without_layers(input_layer, decay, scale, epsilon) - self.top_layer = bn - self.top_size = bn.shape[3] if self.data_format == 'NHWC' else bn.shape[1] - self.top_size = int(self.top_size) - mlperf.logger.log_batch_norm( - input_tensor=input_layer, output_tensor=bn, momentum=decay, - epsilon=epsilon, center=center, scale=scale, training=self.phase_train) - return bn - - def lrn(self, depth_radius, bias, alpha, beta): - """Adds a local response normalization layer.""" - name = 'lrn' + str(self.counts['lrn']) - self.counts['lrn'] += 1 - self.top_layer = tf.nn.lrn( - self.top_layer, depth_radius, bias, alpha, beta, name=name) - return self.top_layer diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/datasets.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/datasets.py deleted file mode 100644 index 875e1123dd9e36175418cb803286fe6a08e93b3f..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/datasets.py +++ /dev/null @@ -1,251 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Benchmark dataset utilities. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from abc import abstractmethod -import os - -import numpy as np -import six -from six.moves import cPickle -from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow.compat.v1 as tf - -from tensorflow.python.platform import gfile -import preprocessing - -IMAGENET_NUM_TRAIN_IMAGES = 1281167 -IMAGENET_NUM_VAL_IMAGES = 50000 - -COCO_NUM_TRAIN_IMAGES = 118287 -COCO_NUM_VAL_IMAGES = 4952 - - -class Dataset(object): - """Abstract class for cnn benchmarks dataset.""" - - def __init__(self, - name, - data_dir=None, - queue_runner_required=False, - num_classes=None): - self.name = name - self.data_dir = data_dir - self._queue_runner_required = queue_runner_required - self._num_classes = num_classes - - def tf_record_pattern(self, subset): - return os.path.join(self.data_dir, '%s-*-of-*' % subset) - - def reader(self): - return tf.TFRecordReader() - - @property - def num_classes(self): - return self._num_classes - - @num_classes.setter - def num_classes(self, val): - self._num_classes = val - - @abstractmethod - def num_examples_per_epoch(self, subset): - pass - - def __str__(self): - return self.name - - def get_input_preprocessor(self, input_preprocessor='default'): - assert not self.use_synthetic_gpu_inputs() - return _SUPPORTED_INPUT_PREPROCESSORS[self.name][input_preprocessor] - - def queue_runner_required(self): - return self._queue_runner_required - - def use_synthetic_gpu_inputs(self): - return not self.data_dir - - -class LibrispeechDataset(Dataset): - """Configuration for LibriSpeech dataset.""" - - def __init__(self, data_dir=None): - super(LibrispeechDataset, self).__init__( - 'librispeech', data_dir, num_classes=29) - - def tf_record_pattern(self, subset): - if subset == 'train': - return os.path.join(self.data_dir, 'train-clean-*.tfrecords') - elif subset == 'validation': - return os.path.join(self.data_dir, 'test-clean.tfrecords') - else: - return '' - - def num_examples_per_epoch(self, subset='train'): - del subset - return 2 # TODO(laigd): currently this is an arbitrary number. - - -class ImageDataset(Dataset): - """Abstract class for image datasets.""" - - def __init__(self, - name, - height, - width, - depth=None, - data_dir=None, - queue_runner_required=False, - num_classes=1001): - super(ImageDataset, self).__init__(name, data_dir, queue_runner_required, - num_classes) - self.height = height - self.width = width - self.depth = depth or 3 - - -class ImagenetDataset(ImageDataset): - """Configuration for Imagenet dataset.""" - - def __init__(self, data_dir=None): - super(ImagenetDataset, self).__init__( - 'imagenet', 300, 300, data_dir=data_dir) - - def num_examples_per_epoch(self, subset='train'): - if subset == 'train': - return IMAGENET_NUM_TRAIN_IMAGES - elif subset == 'validation': - return IMAGENET_NUM_VAL_IMAGES - else: - raise ValueError('Invalid data subset "%s"' % subset) - - -class Cifar10Dataset(ImageDataset): - """Configuration for cifar 10 dataset. - - It will mount all the input images to memory. - """ - - def __init__(self, data_dir=None): - super(Cifar10Dataset, self).__init__( - 'cifar10', - 32, - 32, - data_dir=data_dir, - queue_runner_required=True, - num_classes=11) - - def read_data_files(self, subset='train'): - """Reads from data file and returns images and labels in a numpy array.""" - assert self.data_dir, ('Cannot call `read_data_files` when using synthetic ' - 'data') - if subset == 'train': - filenames = [ - os.path.join(self.data_dir, 'data_batch_%d' % i) - for i in xrange(1, 6) - ] - elif subset == 'validation': - filenames = [os.path.join(self.data_dir, 'test_batch')] - else: - raise ValueError('Invalid data subset "%s"' % subset) - - inputs = [] - for filename in filenames: - with gfile.Open(filename, 'rb') as f: - # python2 does not have the encoding parameter - encoding = {} if six.PY2 else {'encoding': 'bytes'} - inputs.append(cPickle.load(f, **encoding)) - # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the - # input format. - all_images = np.concatenate( - [each_input[b'data'] for each_input in inputs]).astype(np.float32) - all_labels = np.concatenate( - [each_input[b'labels'] for each_input in inputs]) - return all_images, all_labels - - def num_examples_per_epoch(self, subset='train'): - if subset == 'train': - return 50000 - elif subset == 'validation': - return 10000 - else: - raise ValueError('Invalid data subset "%s"' % subset) - - -class COCODataset(ImageDataset): - """COnfiguration for COCO dataset.""" - - def __init__(self, data_dir=None, image_size=300): - super(COCODataset, self).__init__( - 'coco', image_size, image_size, data_dir=data_dir, num_classes=81) - - def num_examples_per_epoch(self, subset='train'): - if subset == 'train': - return COCO_NUM_TRAIN_IMAGES - elif subset == 'validation': - return COCO_NUM_VAL_IMAGES - else: - raise ValueError('Invalid data subset "%s"' % subset) - - -_SUPPORTED_DATASETS = { - 'imagenet': ImagenetDataset, - 'cifar10': Cifar10Dataset, - 'librispeech': LibrispeechDataset, - 'coco': COCODataset, -} - -_SUPPORTED_INPUT_PREPROCESSORS = { - 'imagenet': { - 'default': preprocessing.RecordInputImagePreprocessor, - 'official_models_imagenet': preprocessing.ImagenetPreprocessor, - }, - 'cifar10': { - 'default': preprocessing.Cifar10ImagePreprocessor - }, - 'librispeech': { - 'default': preprocessing.LibrispeechPreprocessor - }, - 'coco': { - 'default': preprocessing.COCOPreprocessor - }, -} - - -def create_dataset(data_dir, data_name): - """Create a Dataset instance based on data_dir and data_name.""" - if not data_dir and not data_name: - # When using synthetic data, use synthetic imagenet images by default. - data_name = 'imagenet' - - # Infere dataset name from data_dir if data_name is not provided. - if data_name is None: - for supported_name in _SUPPORTED_DATASETS: - if supported_name in data_dir: - data_name = supported_name - break - else: # Failed to identify dataset name from data dir. - raise ValueError('Could not identify name of dataset. ' - 'Please specify with --data_name option.') - if data_name not in _SUPPORTED_DATASETS: - raise ValueError('Unknown dataset. Must be one of %s' % ', '.join( - [key for key in sorted(_SUPPORTED_DATASETS.keys())])) - - return _SUPPORTED_DATASETS[data_name](data_dir) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/flags.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/flags.py deleted file mode 100644 index f65898ae2e68c3d0891dd605b877b78cf108e6c0..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/flags.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains functions to define flags and params. - -Calling a DEFINE_* function will add a ParamSpec namedtuple to the param_spec -dict. The DEFINE_* arguments match those in absl. Calling define_flags() creates -a command-line flag for every ParamSpec defined by a DEFINE_* functions. - -The reason we don't use absl flags directly is that we want to be able to use -tf_cnn_benchmarks as a library. When using it as a library, we don't want to -define any flags, but instead pass parameters to the BenchmarkCNN constructor. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from collections import namedtuple - -from absl import flags as absl_flags -import six - - -FLAGS = absl_flags.FLAGS - - -# ParamSpec describes one of benchmark_cnn.BenchmarkCNN's parameters. -ParamSpec = namedtuple('_ParamSpec', - ['flag_type', 'default_value', 'description', - 'kwargs']) - - -# Maps from parameter name to its ParamSpec. -param_specs = {} - - -def DEFINE_string(name, default, help): # pylint: disable=invalid-name,redefined-builtin - param_specs[name] = ParamSpec('string', default, help, {}) - - -def DEFINE_boolean(name, default, help): # pylint: disable=invalid-name,redefined-builtin - param_specs[name] = ParamSpec('boolean', default, help, {}) - - -def DEFINE_integer(name, default, help, lower_bound=None, upper_bound=None): # pylint: disable=invalid-name,redefined-builtin - kwargs = {'lower_bound': lower_bound, 'upper_bound': upper_bound} - param_specs[name] = ParamSpec('integer', default, help, kwargs) - - -def DEFINE_float(name, default, help, lower_bound=None, upper_bound=None): # pylint: disable=invalid-name,redefined-builtin - kwargs = {'lower_bound': lower_bound, 'upper_bound': upper_bound} - param_specs[name] = ParamSpec('float', default, help, kwargs) - - -def DEFINE_enum(name, default, enum_values, help): # pylint: disable=invalid-name,redefined-builtin - kwargs = {'enum_values': enum_values} - param_specs[name] = ParamSpec('enum', default, help, kwargs) - - -def DEFINE_list(name, default, help): # pylint: disable=invalid-name,redefined-builtin - param_specs[name] = ParamSpec('list', default, help, {}) - - -def define_flags(specs=None): - """Define a command line flag for each ParamSpec in flags.param_specs.""" - specs = specs or param_specs - define_flag = { - 'boolean': absl_flags.DEFINE_boolean, - 'float': absl_flags.DEFINE_float, - 'integer': absl_flags.DEFINE_integer, - 'string': absl_flags.DEFINE_string, - 'enum': absl_flags.DEFINE_enum, - 'list': absl_flags.DEFINE_list - } - for name, param_spec in six.iteritems(specs): - if param_spec.flag_type not in define_flag: - raise ValueError('Unknown flag_type %s' % param_spec.flag_type) - else: - define_flag[param_spec.flag_type](name, param_spec.default_value, - help=param_spec.description, - **param_spec.kwargs) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/leading_indicators_test.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/leading_indicators_test.py deleted file mode 100644 index 1bd8715261afc5e19ca4484fe95c81f6c2330d26..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/leading_indicators_test.py +++ /dev/null @@ -1,1003 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Benchmark various leading indicators CNNs. - -The purpose of these tests is to test each model as a high level baseline and -to ensure the various variable_update options have not regressing. Not all -options are tested. The tests focus on the most viable options. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import ctypes -import logging -import os -import sys - -from absl import flags -from absl.testing import absltest # pylint: disable=unused-import -import tensorflow.compat.v1 as tf # pylint: disable=g-bad-import-order -import benchmark_cnn -from platforms import util as platforms_util - -flags.DEFINE_integer('num_batches', None, - 'number of batches to run, excluding warmup') - - -class BenchmarkBase(tf.test.Benchmark): - """Base class for all benchmarks in this file.""" - - def __init__(self, output_dir=None, root_data_dir=None, **kwargs): - """Base class for all benchmarks in this file. - - Args: - output_dir: directory where to output e.g. log files - root_data_dir: directory under which to look for dataset - **kwargs: arbitrary named arguments. This is needed to make the - constructor forward compatible in case PerfZero provides more - named arguments before updating the constructor. - """ - - # Load default values if the benchmark is not run with absl.app.run() - if not flags.FLAGS.is_parsed(): - flags.FLAGS.mark_as_parsed() - - self.fake_data_dir = os.path.join(platforms_util.get_test_data_dir(), - 'fake_tf_record_data') - self.output_dir = output_dir - if root_data_dir is None: - self.data_dir = ('/readahead/200M/placer/prod/home/distbelief/' - 'imagenet-tensorflow/imagenet-2012-tfrecord') - else: - self.data_dir = os.path.join(root_data_dir, 'imagenet') - - def _run_benchmark(self, params): - """Run a CNN benchmark and report its results. - - Args: - params: Params tuple, typically created by benchmark_cnn.make_params or - benchmark_cnn.make_params_from_flags. - """ - logging.info('Running benchmark [%s]', self._get_name()) - params = benchmark_cnn.setup(params) - bench = benchmark_cnn.BenchmarkCNN(params) - bench.print_info() - stats = bench.run() - extras = {} - extras['examples_per_sec'] = stats.get('images_per_sec') - if 'last_average_loss' in stats: - extras['last_average_loss'] = stats['last_average_loss'] - if 'top_1_accuracy' in stats: - extras['top_1_accuracy'] = stats['top_1_accuracy'] - if 'top_5_accuracy' in stats: - extras['top_5_accuracy'] = stats['top_5_accuracy'] - self.report_benchmark( - iters=stats.get('num_steps'), - wall_time=stats.get('average_wall_time'), - extras=extras) - - def _shared_params(self): - """Returns shared parameters for all benchmarks in this file.""" - params = {} - if flags.FLAGS.num_batches is not None: - params['num_batches'] = flags.FLAGS.num_batches - if self.output_dir is not None: - params['benchmark_log_dir'] = self.output_dir - return benchmark_cnn.make_params(**params) - - def _binary_search_batch_size(self, params, init_batch_size): - """Find the max batch_size using binary search.""" - assert init_batch_size > 0 - low_batch_size = 0 - high_batch_size = None - batch_size = init_batch_size - - # No need to run a warmup or many batches; if it doesn't OOM after 10 - # batches, it should work in general. - params = params._replace(num_batches=10, num_warmup_batches=0) - - # Find high_batch_size first. - tf.logging.info( - 'Looking for upper bound to batch size, starting with %d' % batch_size) - while high_batch_size is None: - tf.logging.info('Trying batch_size %d' % batch_size) - params = params._replace(batch_size=batch_size) - bench = benchmark_cnn.BenchmarkCNN(params) - bench.print_info() - try: - bench.run() - low_batch_size = batch_size - batch_size *= 2 - except tf.errors.ResourceExhaustedError: - high_batch_size = batch_size - 1 - - # Binary Search - tf.logging.info( - 'Max batch size is in range (%d, %d]. Starting binary search to find ' - 'exact max batch size.' % (low_batch_size, batch_size)) - while low_batch_size < high_batch_size: - batch_size = (low_batch_size + high_batch_size + 1) // 2 - tf.logging.info('Trying batch_size %d' % batch_size) - params = params._replace(batch_size=batch_size) - bench = benchmark_cnn.BenchmarkCNN(params) - bench.print_info() - try: - bench.run() - low_batch_size = batch_size - except tf.errors.ResourceExhaustedError: - high_batch_size = batch_size - 1 - self.report_benchmark(extras={'max_batch_size': low_batch_size}) - - -class Resnet50BenchmarksInferenceCpu(BenchmarkBase): - """"Benchmarks for ResNet50 inference on CPU.""" - - def _shared_params(self): - """Returns shared parameters for all ResNet50 benchmarks.""" - return BenchmarkBase._shared_params(self)._replace( - num_gpus=1, - model='resnet50', - num_warmup_batches=5, - num_batches=50, - distortions=False, - forward_only=True, - device='cpu', - data_format='NHWC', - num_intra_threads=0) - - def benchmark_synth_forward_batch1(self): - """Tests 1 CPU batch size 1.""" - params = self._shared_params()._replace(batch_size=1) - self._run_benchmark(params) - - def benchmark_synth_forward_batch16(self): - """Tests 1 CPU batch size 16.""" - params = self._shared_params()._replace(batch_size=16) - self._run_benchmark(params) - - -class FrozenResnet50BenchmarksInferenceCpu(Resnet50BenchmarksInferenceCpu): - """"Benchmarks for ResNet50 frozen graph inference on CPU.""" - - def _shared_params(self): - return super(FrozenResnet50BenchmarksInferenceCpu, - self)._shared_params()._replace(freeze_when_forward_only=True) - - -class Resnet50BenchmarksInference(BenchmarkBase): - """"Benchmarks for ResNet50 inference.""" - - def _shared_params(self): - """Returns shared parameters for all ResNet50 benchmarks.""" - return BenchmarkBase._shared_params(self)._replace( - num_gpus=1, model='resnet50', distortions=False, forward_only=True) - - def benchmark_synth_forward_batch128(self): - """Tests 1 GPU batch size 128.""" - params = self._shared_params()._replace(batch_size=128) - self._run_benchmark(params) - - def benchmark_fp16_synth_forward_batch128(self): - """Tests 1 GPU batch size 128 FP16.""" - params = self._shared_params()._replace(batch_size=128, use_fp16=True) - self._run_benchmark(params) - - def benchmark_fp16_synth_forward_batch16(self): - """Tests 1 GPU batch size 16 FP16.""" - params = self._shared_params()._replace(batch_size=16, use_fp16=True) - self._run_benchmark(params) - - def benchmark_xla_synth_forward_batch128(self): - """Tests 1 GPU batch size 128 with XLA.""" - params = self._shared_params()._replace(batch_size=128, xla=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_synth_forward_batch128(self): - """Tests 1 GPU batch size 128 FP16 with XLA.""" - params = self._shared_params()._replace( - batch_size=128, use_fp16=True, xla=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_synth_forward_batch16(self): - """Tests 1 GPU batch size 16 FP16 with XLA.""" - params = self._shared_params()._replace( - batch_size=16, use_fp16=True, xla=True) - self._run_benchmark(params) - - -class FrozenResnet50BenchmarksInference(Resnet50BenchmarksInference): - """"Benchmarks for ResNet50 frozen graph inference.""" - - def _shared_params(self): - return super(FrozenResnet50BenchmarksInference, - self)._shared_params()._replace(freeze_when_forward_only=True) - - def benchmark_trt_synth_forward_batch128(self): - """Tests 1 GPU batch size 128.""" - params = self._shared_params()._replace(batch_size=128, trt_mode='FP32') - self._run_benchmark(params) - - # TODO(laigd): enable fp16 tests for TF-TRT, it's currently not supported yet. - # def benchmark_fp16_trt_synth_forward_batch128(self): - # """Tests 1 GPU batch size 128 FP16.""" - # params = self._shared_params()._replace( - # batch_size=128, use_fp16=True, trt_mode='FP16') - # self._run_benchmark(params) - - # Test with batch size 16 to compare with native TF GPU implementation and - # XLA. - # def benchmark_fp16_trt_synth_forward_batch16(self): - # """Tests 1 GPU batch size 16 FP16.""" - # params = self._shared_params()._replace( - # batch_size=16, use_fp16=True, trt_mode='FP16') - # self._run_benchmark(params) - - -class Resnet50Benchmarks(BenchmarkBase): - """"Benchmark resnet50 configurations.""" - - def _shared_params(self): - """Returns shared parameters for all ResNet50 benchmarks.""" - return BenchmarkBase._shared_params(self)._replace( - model='resnet50', batch_size=128, distortions=False, - optimizer='momentum') - - def _shared_params_fp16(self): - """Returns shared parameters for all ResNet50 FP16 benchmarks.""" - return BenchmarkBase._shared_params(self)._replace( - model='resnet50', - batch_size=256, - distortions=False, - use_fp16=True, - optimizer='momentum', - loss_type_to_report='base_loss', - compute_lr_on_cpu=True, - single_l2_loss_op=True - ) - - def benchmark_synth_1gpu_gpuparams(self): - """Tests 1 gpu with synthetic data.""" - params = self._shared_params()._replace(num_gpus=1) - self._run_benchmark(params) - - def benchmark_fake_1gpu_gpuparams(self): - """Tests 1 gpu with fake data.""" - params = self._shared_params()._replace( - num_gpus=1, data_dir=self.fake_data_dir, data_name='imagenet') - self._run_benchmark(params) - - def benchmark_synth_1gpu_max_batch_size(self): - """Finds largest batch size that can be run with 1 gpu using synth data.""" - params = self._shared_params()._replace( - num_gpus=1, variable_update='parameter_server') - self._binary_search_batch_size(params, init_batch_size=128) - - def benchmark_synth_4gpu_gpureplicated(self): - """Tests 4 gpu with synthetic data with parameters replicated.""" - params = self._shared_params()._replace( - num_gpus=4, - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2) - self._run_benchmark(params) - - def benchmark_synth_8gpu_gpureplicated(self): - """Tests 8 gpu with synthetic data with parameters replicated.""" - params = self._shared_params()._replace( - num_gpus=8, - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2) - self._run_benchmark(params) - - def benchmark_fake_8gpu_gpureplicated(self): - """Tests 8 gpu with fake data with parameters replicated.""" - params = self._shared_params()._replace( - num_gpus=8, - data_dir=self.fake_data_dir, - data_name='imagenet', - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2) - self._run_benchmark(params) - - # FP16 mixed-precision tests. - - def benchmark_fp16_synth_1gpu_gpuparams(self): - """Tests 1 gpu with synthetic data with parameters on the gpu.""" - params = self._shared_params_fp16()._replace( - num_gpus=1, variable_update='parameter_server') - self._run_benchmark(params) - - def benchmark_fp16_synth_1gpu_gpuparams_batch128(self): - """Tests 1 gpu with synthetic data with parameters on the gpu.""" - params = self._shared_params_fp16()._replace( - num_gpus=1, batch_size=128, variable_update='parameter_server') - self._run_benchmark(params) - - def benchmark_fp16_synth_4gpu_gpureplicated(self): - """Tests 4 gpu with synthetic data with nccl and all_reduce.""" - params = self._shared_params_fp16()._replace( - num_gpus=4, - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2) - self._run_benchmark(params) - - def benchmark_fp16_synth_8gpu_gpureplicated(self): - """Tests 8 gpu with synthetic with nccl and all_reduce.""" - params = self._shared_params_fp16()._replace( - num_gpus=8, - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2) - self._run_benchmark(params) - - def benchmark_fp16_fake_1gpu_gpuparams(self): - """Tests 1 gpus with fake data.""" - params = self._shared_params_fp16()._replace( - num_gpus=1, - data_dir=self.fake_data_dir, - data_name='imagenet', - variable_update='parameter_server') - self._run_benchmark(params) - - def benchmark_fp16_fake_8gpu_gpureplicated(self): - """Tests 8 gpus with fake data.""" - params = self._shared_params_fp16()._replace( - num_gpus=8, - data_dir=self.fake_data_dir, - data_name='imagenet', - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2) - self._run_benchmark(params) - - def benchmark_fp16_fakedistort_8gpu_gpureplicated(self): - """Tests 8 gpus with fake distorted data.""" - params = self._shared_params_fp16()._replace( - num_gpus=8, - data_dir=self.fake_data_dir, - data_name='imagenet', - distortions=True, - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2) - self._run_benchmark(params) - - # XLA versions of Resnet50 tests only for single GPU. - def benchmark_xla_synth_1gpu_gpuparams(self): - """Tests 1 gpu with synthetic data with XLA.""" - params = self._shared_params()._replace( - num_gpus=1, variable_update='parameter_server', xla=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_synth_1gpu_gpuparams(self): - """Tests 1 gpu with fp16, synthetic data with XLA.""" - params = self._shared_params_fp16()._replace( - num_gpus=1, variable_update='parameter_server', xla=True) - self._run_benchmark(params) - - # Test does not run as part of continuous testing on guitar. - def benchmark_ng_xla_batch64_synth_1gpu_gpuparams(self): - """Tests 1 gpu with XLA, synth data, and batch 64.""" - params = self._shared_params()._replace( - num_gpus=1, batch_size=64, variable_update='parameter_server', xla=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_batch64_synth_1gpu_gpuparams(self): - """Tests 1 gpu with fp16, XLA, synth data, and batch 64.""" - params = self._shared_params_fp16()._replace( - num_gpus=1, - batch_size=64, - variable_update='parameter_server', - xla=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_batch128_synth_1gpu_gpuparams(self): - """Tests 1 gpu with fp16, XLA, and synth data.""" - params = self._shared_params_fp16()._replace( - num_gpus=1, - batch_size=128, - variable_update='parameter_server', - xla=True) - self._run_benchmark(params) - - def benchmark_xla_synth_1gpu_max_batch_size(self): - """Finds largest batch that can be run with XLA, 1 gpu, and synth data.""" - params = self._shared_params()._replace( - num_gpus=1, variable_update='parameter_server', xla=True) - self._binary_search_batch_size(params, init_batch_size=128) - - def benchmark_xla_real_1gpu_gpuparams(self): - """Tests 1 gpu with real data with XLA.""" - params = self._shared_params()._replace( - num_gpus=1, - data_dir=self.data_dir, - variable_update='parameter_server', - xla=True) - self._run_benchmark(params) - - # Test does not run as part of continuous testing. - def benchmark_xla_fake_1gpu_gpuparams(self): - """Tests 1 gpu with fake data with XLA.""" - params = self._shared_params()._replace( - num_gpus=1, - data_dir=self.fake_data_dir, - data_name='imagenet', - variable_update='parameter_server', - xla=True) - self._run_benchmark(params) - - # Test does not run as part of continuous testing. - def benchmark_xla_fakedistort_1gpu_gpuparams(self): - """Tests 1 gpu with fake distorted data with XLA.""" - params = self._shared_params()._replace( - num_gpus=1, - data_dir=self.fake_data_dir, - data_name='imagenet', - distortions=True, - variable_update='parameter_server', - xla=True) - self._run_benchmark(params) - - -class Resnet50v15Benchmarks(BenchmarkBase): - """"Benchmark various ResNet50V1.5 configurations. - - ResNetV1.5 differs from V1 in stride 2 is used in the first 3x3 convolution of - each block instead of the first 1x1 convolution. - """ - - def _shared_params_fp16(self): - """Returns shared parameters for all ResNet50v1.5 FP16 benchmarks.""" - return BenchmarkBase._shared_params(self)._replace( - model='resnet50_v1.5', - batch_size=256, - distortions=False, - use_fp16=True, - optimizer='momentum', - loss_type_to_report='base_loss', - compute_lr_on_cpu=True, - single_l2_loss_op=True - ) - - def benchmark_fp16_synth_1gpu_gpuparams(self): - """Tests 1 gpu with synthetic data.""" - params = self._shared_params_fp16()._replace(num_gpus=1) - self._run_benchmark(params) - - def benchmark_fp16_batch256_synth_8gpu_gpuparams(self): - """Tests 8 gpus with synthetic data at batch 256.""" - params = self._shared_params_fp16()._replace(num_gpus=8) - self._run_benchmark(params) - - def benchmark_fp16_batch128_synth_1gpu_gpuparams(self): - """Tests 1 gpu with synthetic data at batch 128 (useful for small GPUs).""" - params = self._shared_params_fp16()._replace(num_gpus=1, batch_size=128) - self._run_benchmark(params) - - def benchmark_fp16_fake_1gpu_gpuparams(self): - """Tests 1 gpu with fake data.""" - params = self._shared_params_fp16()._replace( - num_gpus=1, data_dir=self.fake_data_dir, data_name='imagenet') - self._run_benchmark(params) - - def benchmark_fp16_synth_8gpu_gpureplicated(self): - """Tests 8 gpu with synthetic data with parameters replicated.""" - params = self._shared_params_fp16()._replace( - num_gpus=8, - num_batches=200, - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2) - self._run_benchmark(params) - - def benchmark_fp16_fake_8gpu_gpureplicated(self): - """Tests 8 gpu with fake data with parameters replicated.""" - params = self._shared_params_fp16()._replace( - num_gpus=8, - num_batches=200, - data_dir=self.fake_data_dir, - data_name='imagenet', - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2) - self._run_benchmark(params) - - # XLA versions of Resnet50v1.5 tests. - def benchmark_fp16_xla_synth_1gpu_gpuparams(self): - """Tests 1 gpu with fp16, synthetic data with XLA.""" - params = self._shared_params_fp16()._replace(num_gpus=1, xla=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_batch128_synth_1gpu_gpuparams(self): - """Tests 1 gpu with fp16, batch128, synthetic data with XLA.""" - params = self._shared_params_fp16()._replace( - num_gpus=1, batch_size=128, xla=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_compile_synth_1gpu_gpuparams(self): - """Tests 1 gpu with synthetic data.""" - params = self._shared_params_fp16()._replace(num_gpus=1, xla_compile=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_compile_batch128_synth_1gpu_gpuparams(self): - """Tests 1 gpu with synthetic data at batch 128 (useful for small GPUs).""" - params = self._shared_params_fp16()._replace( - num_gpus=1, num_batches=200, batch_size=128, xla_compile=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_batch256_synth_8gpu_gpuparams(self): - """Tests 8 gpu with synthetic data and xla autojit.""" - params = self._shared_params_fp16()._replace( - num_gpus=8, num_batches=200, batch_size=256, xla=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_compile_fake_1gpu_gpuparams(self): - """Tests 1 gpu with fake data.""" - params = self._shared_params_fp16()._replace( - num_gpus=1, - data_dir=self.fake_data_dir, - data_name='imagenet', - xla_compile=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_compile_synth_8gpu_gpureplicated(self): - """Tests 8 gpu with synthetic data with parameters replicated.""" - params = self._shared_params_fp16()._replace( - num_gpus=8, - num_batches=200, - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2, - xla_compile=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_synth_8gpu_gpureplicated(self): - """Tests 8 gpu with synthetic data with parameters replicated.""" - params = self._shared_params_fp16()._replace( - num_gpus=8, - num_batches=200, - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2, - xla=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_compile_fake_8gpu_gpureplicated(self): - """Tests 8 gpu with fake data with parameters replicated.""" - params = self._shared_params_fp16()._replace( - num_gpus=8, - num_batches=200, - data_dir=self.fake_data_dir, - data_name='imagenet', - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2, - xla_compile=True) - self._run_benchmark(params) - - -class Vgg16Benchmarks(BenchmarkBase): - """"Benchmark various vgg16 configurations.""" - - def _shared_params(self): - """Returns shared parameters for all vgg16 benchmarks.""" - return BenchmarkBase._shared_params(self)._replace( - model='vgg16', batch_size=128, distortions=False) - - def benchmark_synth_1gpu_gpuparams(self): - """Tests 1 gpu with synthetic data with parameters on gpu.""" - params = self._shared_params()._replace( - num_gpus=1, variable_update='parameter_server') - self._run_benchmark(params) - - def benchmark_fp16_synth_1gpu_gpuparams(self): - """Tests 1 gpu with synthetic data with parameters on gpu.""" - params = self._shared_params()._replace( - num_gpus=1, use_fp16=True, variable_update='parameter_server') - self._run_benchmark(params) - - def benchmark_synth_8gpu_gpureplicated(self): - """Tests 8 gpu with synthetic data with parameters replicated.""" - params = self._shared_params()._replace( - num_gpus=8, - all_reduce_spec='nccl', - variable_update='replicated', - compact_gradient_transfer=False, - gradient_repacking=2) - self._run_benchmark(params) - - # XLA versions of VGG16 tests only for single GPU. - def benchmark_xla_synth_1gpu_gpuparams(self): - """Tests 1 gpu with synthetic data and XLA.""" - params = self._shared_params()._replace( - num_gpus=1, variable_update='parameter_server', xla=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_synth_1gpu_gpuparams(self): - """Tests 1 gpu with fp16, synthetic data, and XLA.""" - params = self._shared_params()._replace( - num_gpus=1, variable_update='parameter_server', xla=True, use_fp16=True) - self._run_benchmark(params) - - # Test does not run as part of continuous testing. - def benchmark_xla_fake_1gpu_gpuparams(self): - """Tests 1 gpu with fake data and XLA.""" - params = self._shared_params()._replace( - num_gpus=1, - data_dir=self.fake_data_dir, - data_name='imagenet', - variable_update='parameter_server', - xla=True) - self._run_benchmark(params) - - def benchmark_xla_real_1gpu_gpuparams(self): - """Tests 1 gpu with real data and XLA.""" - params = self._shared_params()._replace( - num_gpus=1, - data_dir=self.data_dir, - variable_update='parameter_server', - xla=True) - self._run_benchmark(params) - - -class TrivialBenchmarks(BenchmarkBase): - """"Benchmarks for trivial model. - - The purpose of these tests is to verify the upper bound for the input - pipeline. Fake data creates an upperbound on the input pipeline throughput. - """ - - def _shared_params(self): - """Returns shared parameters for all trivial benchmarks.""" - return BenchmarkBase._shared_params(self)._replace( - model='trivial', - num_gpus=8, - distortions=False, - variable_update='independent', - data_dir=self.fake_data_dir) - - def benchmark_fake_64batch(self): - params = self._shared_params()._replace(batch_size=64, data_name='imagenet') - self._run_benchmark(params) - - def benchmark_fake_128batch(self): - params = self._shared_params()._replace( - batch_size=128, data_name='imagenet') - self._run_benchmark(params) - - def benchmark_fake_256batch(self): - params = self._shared_params()._replace( - batch_size=256, data_name='imagenet') - self._run_benchmark(params) - - def benchmark_fakedistort_128batch(self): - params = self._shared_params()._replace( - batch_size=128, data_name='imagenet', distortions=True) - self._run_benchmark(params) - - -class AlexnetBenchmarks(BenchmarkBase): - """"Benchmarks for alexnet.""" - - def _shared_params(self): - """Returns shared parameters for all alexnet benchmarks.""" - return BenchmarkBase._shared_params(self)._replace( - model='alexnet', batch_size=512, distortions=False) - - def benchmark_synth_1gpu_gpuparams(self): - """Tests 1 gpu with synthetic data with parameters on gpu.""" - params = self._shared_params()._replace( - num_gpus=1, variable_update='parameter_server') - self._run_benchmark(params) - - def benchmark_fp16_synth_1gpu_gpuparams(self): - """Tests 1 gpu with synthetic data with parameters on gpu.""" - params = self._shared_params()._replace( - num_gpus=1, use_fp16=True, variable_update='parameter_server') - self._run_benchmark(params) - - def benchmark_synth_8gpu_gpureplicated(self): - """Tests 8 gpus with synthetic data with parameters replicated.""" - params = self._shared_params()._replace( - num_gpus=8, - variable_update='replicated', - all_reduce_spec='nccl', - compact_gradient_transfer=False, - gradient_repacking=2) - self._run_benchmark(params) - - def benchmark_fake_8gpu_gpureplicated(self): - """Tests 8 gpus with fake data with parameters replicated.""" - params = self._shared_params()._replace( - num_gpus=8, - data_dir=self.fake_data_dir, - data_name='imagenet', - variable_update='replicated', - all_reduce_spec='nccl', - compact_gradient_transfer=False, - gradient_repacking=2) - self._run_benchmark(params) - - # XLA Benchmark tests for AlexNet. - def benchmark_xla_synth_1gpuparams(self): - """Tests 1 gpu with synthetic data and XLA.""" - params = self._shared_params()._replace( - num_gpus=1, variable_update='parameter_server', xla=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_synth_1gpu_gpuparams(self): - """Tests 1 gpu with fp16, synthetic data and XLA.""" - params = self._shared_params()._replace( - num_gpus=1, variable_update='parameter_server', xla=True, use_fp16=True) - self._run_benchmark(params) - - # Test does not run as part of continuous testing. - def benchmark_xla_fake_1gpuparams(self): - """Tests 1 gpu with fake data and XLA.""" - params = self._shared_params()._replace( - num_gpus=1, - data_dir=self.fake_data_dir, - data_name='imagenet', - variable_update='parameter_server', - xla=True) - self._run_benchmark(params) - - def benchmark_xla_real_1gpuparams(self): - """Tests 1 gpu with real data and XLA.""" - params = self._shared_params()._replace( - num_gpus=1, - data_dir=self.data_dir, - variable_update='parameter_server', - xla=True) - self._run_benchmark(params) - - -class InceptionV3Benchmarks(BenchmarkBase): - """"Benchmark for InceptionV3.""" - - def _shared_params(self): - """Returns shared parameters for all InceptionV3 benchmarks.""" - return BenchmarkBase._shared_params(self)._replace( - model='inception3', batch_size=64, distortions=False) - - def benchmark_synth_1gpu_gpuparams(self): - """Tests 1 gpu with synthetic data.""" - params = self._shared_params()._replace( - num_gpus=1, variable_update='parameter_server') - self._run_benchmark(params) - - def benchmark_fp16_synth_1gpu_gpuparams(self): - """Tests 1 gpu with synthetic data.""" - params = self._shared_params()._replace( - num_gpus=1, use_fp16=True, variable_update='parameter_server') - self._run_benchmark(params) - - def benchmark_synth_1gpu_max_batch_size(self): - """Finds largest batch size that can be run with 1 gpu using synth data.""" - params = self._shared_params()._replace( - num_gpus=1, variable_update='parameter_server') - self._binary_search_batch_size(params, init_batch_size=128) - - def benchmark_xla_synth_1gpu_gpuparams(self): - """Tests 1 gpu with synthetic and XLA.""" - params = self._shared_params()._replace( - num_gpus=1, variable_update='parameter_server', xla=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_synth_1gpu_gpuparams(self): - """Tests 1 gpu with fp16, XLA and synthetic data.""" - params = self._shared_params()._replace( - num_gpus=1, variable_update='parameter_server', xla=True, use_fp16=True) - self._run_benchmark(params) - - def benchmark_xla_synth_1gpu_max_batch_size(self): - """Finds largest batch that can be run with XLA, 1 gpu, and synth data.""" - params = self._shared_params()._replace( - num_gpus=1, variable_update='parameter_server', xla=True) - self._binary_search_batch_size(params, init_batch_size=128) - - # Test does not run as part of continuous testing. - def benchmark_xla_fake_1gpu_gpuparams(self): - """Tests 1 gpu with fake data with XLA.""" - params = self._shared_params()._replace( - num_gpus=1, - data_dir=self.fake_data_dir, - data_name='imagenet', - variable_update='parameter_server', - xla=True) - self._run_benchmark(params) - - def benchmark_xla_real_1gpu_gpuparams(self): - """Tests 1 gpu with real data with XLA.""" - params = self._shared_params()._replace( - num_gpus=1, - data_dir=self.data_dir, - variable_update='parameter_server', - xla=True) - self._run_benchmark(params) - - -class NcfBenchmarks(BenchmarkBase): - """Benchmarks for neural collaborative filtering.""" - - def _shared_params(self): - return BenchmarkBase._shared_params(self)._replace( - model='ncf', batch_size=64*1024, num_gpus=1, num_warmup_batches=1) - - def benchmark_synth_1gpu_gpuparams(self): - params = self._shared_params()._replace(variable_update='parameter_server') - self._run_benchmark(params) - - def benchmark_fp16_synth_1gpu_gpuparams(self): - params = self._shared_params()._replace( - variable_update='parameter_server', use_fp16=True) - self._run_benchmark(params) - - def benchmark_xla_synth_1gpu_gpuparams(self): - params = self._shared_params()._replace( - variable_update='parameter_server', xla=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_synth_1gpu_gpuparams(self): - params = self._shared_params()._replace( - variable_update='parameter_server', xla=True, use_fp16=True) - self._run_benchmark(params) - - def benchmark_xla_compile_synth_1gpu_gpuparams(self): - params = self._shared_params()._replace( - variable_update='parameter_server', xla_compile=True) - self._run_benchmark(params) - - def benchmark_fp16_xla_compile_synth_1gpu_gpuparams(self): - params = self._shared_params()._replace( - variable_update='parameter_server', xla_compile=True, use_fp16=True) - self._run_benchmark(params) - - -class DeepSpeech2Benchmarks(BenchmarkBase): - """Benchmarks for DeepSpeech2 model.""" - - def _shared_params(self): - return BenchmarkBase._shared_params(self)._replace( - model='deepspeech2', batch_size=32, num_gpus=1, data_name='librispeech') - - def benchmark_synth_1gpu_gpuparams(self): - params = self._shared_params()._replace(variable_update='parameter_server') - self._run_benchmark(params) - - def benchmark_xla_synth_1gpu_gpuparams(self): - params = self._shared_params()._replace( - variable_update='parameter_server', xla=True) - self._run_benchmark(params) - - def benchmark_xla_compile_synth_1gpu_gpuparams(self): - params = self._shared_params()._replace( - variable_update='parameter_server', xla_compile=True) - self._run_benchmark(params) - - -class SsdBenchmarks(BenchmarkBase): - """Benchmarks for SSD model.""" - - def _cudnn_version(self): - if sys.platform == 'win32': - return None - - lib = ctypes.cdll.LoadLibrary(None) - if hasattr(lib, 'cudnnGetErrorString'): - version = lib.cudnnGetVersion() - return version - - return None - - def _shared_params(self): - cudnn_version = self._cudnn_version() - if cudnn_version is None or cudnn_version < 7300: - raise RuntimeError( - 'Needs at least cuDNN 7.3 to work with fp16 (b/112048183). ' - 'Build with --define=use_experimental_cudnn=1') - - return BenchmarkBase._shared_params(self)._replace( - # TODO(b/115672206): Replace backbone model and data dir with replicated - # placer location for better performance. - backbone_model_path=platforms_util.get_ssd_backborn_model_file(), # pylint: disable=line-too-long - data_dir=platforms_util.get_ssd_backboard_data_dir(), - batch_size=128, - data_name='coco', - model='ssd300', - num_batches=10, - num_warmup_batches=1, - num_gpus=1, - optimizer='momentum', - momentum=0.9, - weight_decay=5e-4, - loss_type_to_report='base_loss', - single_l2_loss_op=True, - compute_lr_on_cpu=True, - ) - - def benchmark_xla_compile_real_1gpu_gpuparams(self): - params = self._shared_params()._replace( - num_gpus=1, - xla_compile=True, - ) - self._run_benchmark(params) - - def benchmark_real_1gpu_gpuparams(self): - params = self._shared_params()._replace(num_gpus=1,) - self._run_benchmark(params) - - def benchmark_xla_compile_fp16_real_1gpu_gpuparams(self): - params = self._shared_params()._replace( - num_gpus=1, xla_compile=True, use_fp16=True) - self._run_benchmark(params) - - def benchmark_fp16_real_1gpu_gpuparams(self): - params = self._shared_params()._replace(num_gpus=1, use_fp16=True) - self._run_benchmark(params) - - def benchmark_xla_compile_real_8gpu_gpuparams(self): - params = self._shared_params()._replace( - num_gpus=8, - xla_compile=True, - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2, - num_batches=50, - ) - self._run_benchmark(params) - - def benchmark_real_8gpu_gpuparams(self): - params = self._shared_params()._replace( - num_gpus=8, - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2, - num_batches=50, - ) - self._run_benchmark(params) - - def benchmark_xla_compile_fp16_real_8gpu_gpuparams(self): - params = self._shared_params()._replace( - num_gpus=8, - xla_compile=True, - use_fp16=True, - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2, - num_batches=50, - ) - self._run_benchmark(params) - - def benchmark_fp16_real_8gpu_gpuparams(self): - params = self._shared_params()._replace( - num_gpus=8, - use_fp16=True, - variable_update='replicated', - all_reduce_spec='nccl', - gradient_repacking=2, - num_batches=50, - ) - self._run_benchmark(params) - - -if __name__ == '__main__': - tf.disable_v2_behavior() - tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/mlperf.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/mlperf.py deleted file mode 100644 index 932f3136e1b5d4abb5afefebaf3c9512a7b0ca15..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/mlperf.py +++ /dev/null @@ -1,260 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains functions related to MLPerf compliance. - -MLPerf requires submissions to log what the benchmark does, in order to verify -that the benchmark meets the MLPerf requirements. This module contains a global -object `logger` that is used by other files to log what tf_cnn_benchmarks does -for compliance. - -By default, `logger` does nothing, as the MLPerf compliance logs are verbose and -unnecessary if one is not concerned about MLPerf compliance. The logger can be -enabled by using the `mlperf_logger` context manager. - -To enable the logger with `mlperf_logger`, the MLPerf compliance library at -https://github.com/mlperf/training/tree/master/compliance is required. If -the logger is not enabled, the library is not needed. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - - -from collections import namedtuple -import contextlib -import os -import sys - -import tensorflow.compat.v1 as tf - -# pylint: disable=g-import-not-at-top -try: - # Not all users have the MLPerf compliance library, so we don't want to - # unconditionally crash if these imports fail. - from mlperf_compliance import mlperf_log - from mlperf_compliance import resnet_log_helper - from mlperf_compliance import tags - import_successful = True -except ImportError: - # The logger cannot be enabled in this case since the MLPerf library isn't - # found. We return empty strings from the `tags` attribute so that - # the benchmark can still run without crashing. This empty tags are passed - # to an instance of `NullMlPerfLogger`, which does not log anything and - # ignores the tag values. - - class _Tags(object): - - def __getattr__(self, item): - return '' - tags = _Tags() - import_successful = False -# pylint: enable=g-import-not-at-top - - -_ModelInfo = namedtuple('_ModelInfo', ['print_fn', 'tag_set', - 'mlperf_model_name']) - - -_MLPERF_LOG_PREFIX = ':::MLPv0.5.0' - - -class MlPerfLogger(object): - """Logs various aspects about a benchmark run for MLPerf compliance.""" - - def __init__(self, model): - self._root_dir = os.path.split(os.path.abspath(__file__))[0] - mlperf_log.ROOT_DIR_RESNET = self._root_dir - mlperf_log.ROOT_DIR_SSD = self._root_dir - self.model = model - model_to_info = { - 'resnet50_v1.5': _ModelInfo(mlperf_log.resnet_print, - mlperf_log.RESNET_TAG_SET, tags.RESNET), - 'ssd300': _ModelInfo(mlperf_log.ssd_print, mlperf_log.SSD_TAG_SET, - tags.SSD) - } - - try: - self._log_fn, self.tag_set, self.mlperf_model_name = model_to_info[model] - except KeyError: - raise ValueError('--ml_perf_compliance_logging is only compatible when ' - '--model is one of the following: ' + - ', '.join(model_to_info.keys())) - - def log(self, key, value=None, stack_offset=2): - if key in self.tag_set: - self._log_fn(key, value, stack_offset) - else: - print('Ignoring MLPerf logging item key=%s, value=%s for model %s' % - (key, value, self.model)) - - def log_deferred_tensor_value(self, key, tensor_value, global_step, - stack_offset=2, every_n=1): - """Logs the value of a tensor when the graph is run.""" - caller = '(%s)' % mlperf_log.get_caller(stack_offset, self._root_dir) - def create_print_op(): - return tf.print(_MLPERF_LOG_PREFIX, self.mlperf_model_name, - tf.timestamp(), caller, key, - ': { "deferred": true, "value":', tensor_value, '}', - output_stream=sys.stdout) - maybe_print = tf.cond(tf.equal(global_step % every_n, 0), create_print_op, - tf.no_op) - with tf.control_dependencies([maybe_print]): - return tf.identity(tensor_value) - - def log_max_pool(self, input_tensor, output_tensor): - if self.model == 'resnet50_v1.5': - resnet_log_helper.log_max_pool(input_tensor, output_tensor) - - def log_begin_block(self, input_tensor, block_type): - if self.model == 'resnet50_v1.5': - resnet_log_helper.log_begin_block(input_tensor, block_type) - - def log_end_block(self, output_tensor): - if self.model == 'resnet50_v1.5': - resnet_log_helper.log_end_block(output_tensor) - - def log_projection(self, input_tensor, output_tensor): - if self.model == 'resnet50_v1.5': - resnet_log_helper.log_projection(input_tensor, output_tensor) - - def log_conv2d(self, input_tensor, output_tensor, stride_height, stride_width, - filters, initializer, use_bias): - """Log a conv2d call.""" - if self.model == 'resnet50_v1.5': - assert stride_height == stride_width, ( - '--ml_perf_compliance_logging does not support convolutions where ' - 'the stride height is not equal to the stride width. ' - 'stride_height=%d, stride_width=%d' % (stride_height, stride_width)) - if isinstance(initializer, tf.truncated_normal_initializer) or ( - isinstance(initializer, tf.variance_scaling_initializer) and - initializer.distribution == 'truncated_normal'): - initializer = tags.TRUNCATED_NORMAL - elif (isinstance(initializer, tf.glorot_uniform_initializer) or - initializer is None): - initializer = 'glorot_uniform' - resnet_log_helper.log_conv2d(input_tensor, output_tensor, stride_width, - filters, initializer, use_bias) - - def log_batch_norm(self, input_tensor, output_tensor, momentum, epsilon, - center, scale, training): - if self.model == 'resnet50_v1.5': - resnet_log_helper.log_batch_norm(input_tensor, output_tensor, momentum, - epsilon, center, scale, training) - - def log_train_epochs(self, num_epochs): - """Logs all the TRAIN_EPOCHs log lines.""" - num_epochs_int = int(num_epochs) - for i in range(num_epochs_int): - # MLPerf allows us to print all the train epochs at once instead of - # printing them as we do them. - self.log(key=mlperf_log.TRAIN_EPOCH, value=i, stack_offset=3) - if num_epochs_int != num_epochs: - value = (str(num_epochs_int) + - ', but this epoch only has {}% of the examples of a normal epoch' - .format(100 * (num_epochs - num_epochs_int))) - self.log(key=mlperf_log.TRAIN_EPOCH, value=value, stack_offset=3) - - def log_input_resize_aspect_preserving(self, height, width, scale_factor): - assert height == width, ( - '--ml_perf_compliance_logging does not support models with nonsquare ' - 'images. Cannot process image with height=%d and width=%d' % - (height, width)) - self.log(key=tags.INPUT_RESIZE_ASPECT_PRESERVING, - value={'min': int(height * scale_factor)}) - - def log_eval_epoch(self, tag, global_step, batch_size, stack_offset=2): - if self.model == 'resnet50_v1.5': - self.log(key=tag, stack_offset=stack_offset+1) - elif self.model == 'ssd300': - epoch = int(global_step * batch_size / 118287) - self.log(key=tag, value=epoch, stack_offset=stack_offset+1) - - def log_eval_accuracy(self, accuracy, global_step, batch_size, - examples_per_epoch, stack_offset=2): - """Logs eval accuracy.""" - epoch = int(global_step * batch_size / examples_per_epoch) - eval_accuracy = {'epoch': epoch, 'value': accuracy} - eval_iteration_accuracy = {'iteration': global_step, 'value': accuracy} - self.log(key=tags.EVAL_ACCURACY, value=eval_accuracy, - stack_offset=stack_offset+1) - self.log(key=tags.EVAL_ITERATION_ACCURACY, - value=eval_iteration_accuracy, - stack_offset=stack_offset+1) - - -def _empty_fn(*args, **kwargs): - del args, kwargs - - -class NullMlPerfLogger(object): - """A version of `MlPerfLogger` that does not log anything. - - This class has the same interface as `MlPerfLogger`, but does not actually do - anything. This is used when logging is disabled, which is the default - behavior. - """ - - def __getattr__(self, item): - return _empty_fn - - def log_deferred_tensor_value(self, key, tensor_value, *args, **kwargs): - del key, args, kwargs - return tensor_value - - -# A global singleton logger. By default, it's the null logger but can be -# switched to an MlPerfLogger with `mlperf_logger()`. -logger = NullMlPerfLogger() - - -@contextlib.contextmanager -def mlperf_logger(use_mlperf_logger, model): - """Optionally enable the mlperf logger. - - If `use_mlperf_logger` is True, sets the `logger` global variable to an - instance of MlPerfLogger that will print logs for MLPerf compliance. If - `use_mlperf_logger` is False, does nothing. - - Args: - use_mlperf_logger: If True, enables the mlperf logger. If False, this - function does nothing. - model: The model that will be logged. Required, because different models - must log different things for MLPerf compliance. - - Yields: - Nothing. - - Raises: - ImportError: If `use_mlperf_logger` is True but the MLPerf compliance - library cannot be imported - """ - global logger - if use_mlperf_logger: - if not import_successful: - raise ImportError('Failed to import MLPerf compliance library, which is ' - 'required when --ml_perf_compliance_logging is ' - 'specified. Clone this repo and add this directory ' - 'https://github.com/mlperf/training/tree/master/' - 'compliance to the PYTHONPATH environmental variable.') - logger_ = MlPerfLogger(model) - old_logger = logger - try: - logger = logger_ - yield - finally: - logger = old_logger - else: - yield diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/mlperf_test.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/mlperf_test.py deleted file mode 100644 index 7e83fc29603580b24466c22db2de3732f3d6c13e..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/mlperf_test.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Contains tests related to MLPerf. - -Note this test only passes if the MLPerf compliance library is installed. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from collections import Counter -import logging -import re - -import six -import tensorflow.compat.v1 as tf -import benchmark_cnn -import datasets -import mlperf -import test_util -from models import model -from mlperf_compliance import mlperf_log - - -class _MlPerfTestModel(model.CNNModel): - """A model to test the MLPerf compliance logging on.""" - - def __init__(self): - super(_MlPerfTestModel, self).__init__( - 'mlperf_test_model', image_size=224, batch_size=2, learning_rate=1) - - def add_inference(self, cnn): - assert cnn.top_layer.shape[1:] == (3, 224, 224) - cnn.conv(1, 1, 1, 1, 1, use_batch_norm=True) - cnn.mpool(1, 1, 1, 1, num_channels_in=1) - cnn.reshape([-1, 224 * 224]) - cnn.affine(1, activation=None) - - # Assert that the batch norm variables are filtered out for L2 loss. - variables = tf.global_variables() + tf.local_variables() - assert len(variables) > len(self.filter_l2_loss_vars(variables)) - - -class MlPerfComplianceTest(tf.test.TestCase): - """Tests the MLPerf compliance logs. - - This serves as a quick check that we probably didn't break the compliance - logging. It is not mean to be as comprehensive as the official MLPerf - compliance checker will be. - """ - - def setUp(self): - super(MlPerfComplianceTest, self).setUp() - benchmark_cnn.setup(benchmark_cnn.make_params()) - - # Map between regex and the number of times we expect to see that regex in the - # logs. Entry commented out with the comment FIXME indicate that - # tf_cnn_benchmarks currently fails compliance in that regard, and needs to be - # fixed to be MLPerf compliant. - EXPECTED_LOG_REGEXES = { - # Preprocessing tags - mlperf.tags.INPUT_ORDER: 2, # 1 for training, 1 for eval - # We pass --tf_random_seed=9876 in the test. - r'%s: 9876' % mlperf.tags.RUN_SET_RANDOM_SEED: 2, - # The Numpy random seed is hardcoded to 4321. - r'%s: 4321' % mlperf.tags.RUN_SET_RANDOM_SEED: 2, - r'%s: %d' % (mlperf.tags.PREPROC_NUM_TRAIN_EXAMPLES, - datasets.IMAGENET_NUM_TRAIN_IMAGES): 1, - r'%s: %d' % (mlperf.tags.PREPROC_NUM_EVAL_EXAMPLES, - datasets.IMAGENET_NUM_VAL_IMAGES): 1, - mlperf.tags.PREPROC_NUM_EVAL_EXAMPLES + '.*': 1, - mlperf.tags.INPUT_DISTORTED_CROP_MIN_OBJ_COV + '.*': 1, - mlperf.tags.INPUT_DISTORTED_CROP_RATIO_RANGE + '.*': 1, - mlperf.tags.INPUT_DISTORTED_CROP_AREA_RANGE + '.*': 1, - mlperf.tags.INPUT_DISTORTED_CROP_MAX_ATTEMPTS + '.*': 1, - mlperf.tags.INPUT_RANDOM_FLIP + '.*': 1, - r'%s: \[224, 224\].*' % mlperf.tags.INPUT_CENTRAL_CROP: 1, - - r'%s: \[123.68, 116.78, 103.94\].*' % mlperf.tags.INPUT_MEAN_SUBTRACTION: - 2, - - r'%s: {"min": 256}.*' % mlperf.tags.INPUT_RESIZE_ASPECT_PRESERVING: 1, - - # 1 for training, 1 for eval - r'%s: \[224, 224\].*' % mlperf.tags.INPUT_RESIZE: 2, - - # Resnet model tags - mlperf.tags.MODEL_HP_BATCH_NORM + '.*': 2, - # 2 for training, 2 for eval. Although there's only 1 conv2d, each conv2d - # produces 2 logs. - mlperf.tags.MODEL_HP_CONV2D_FIXED_PADDING + '.*': 4, - mlperf.tags.MODEL_HP_RELU + '.*': 2, - mlperf.tags.MODEL_HP_INITIAL_MAX_POOL + '.*': 2, - mlperf.tags.MODEL_HP_DENSE + '.*': 4, - mlperf.tags.MODEL_HP_DENSE + '.*': 4, - - # Note that tags our test model does not emit, like MODEL_HP_SHORTCUT_ADD, - # are omitted here. - - r'%s: "categorical_cross_entropy".*' % mlperf.tags.MODEL_HP_LOSS_FN: 1, - - # 1 for training, 2 because the _MlPerfTestModel calls this when building - # the model for both training and eval - r'%s: true' % mlperf.tags.MODEL_EXCLUDE_BN_FROM_L2: 3, - - r'%s: 0.5.*' % mlperf.tags.MODEL_L2_REGULARIZATION: 1, - - # Note we do not handle OPT_LR, since that is printed to stderr using - # tf.Print, which we cannot easily intercept. - - # Other tags - '%s: "%s"' % (mlperf.tags.OPT_NAME, mlperf.tags.SGD_WITH_MOMENTUM): 1, - '%s: 0.5' % mlperf.tags.OPT_MOMENTUM: 1, - mlperf.tags.RUN_START: 1, - '%s: 2' % mlperf.tags.INPUT_BATCH_SIZE: 1, - mlperf.tags.TRAIN_LOOP: 1, - mlperf.tags.TRAIN_EPOCH + '.*': 1, - '%s: 2' % mlperf.tags.INPUT_SIZE: 2, - mlperf.tags.EVAL_START: 2, - mlperf.tags.EVAL_STOP: 2, - '%s: 6' % mlperf.tags.EVAL_SIZE: 2, - mlperf.tags.EVAL_ACCURACY + '.*': 2, - '%s: 2.0' % mlperf.tags.EVAL_TARGET: 2, - mlperf.tags.RUN_STOP + '.*': 1, - mlperf.tags.RUN_FINAL: 1 - } - EXPECTED_LOG_REGEXES = Counter({re.compile(k): v for - k, v in EXPECTED_LOG_REGEXES.items()}) - - def testMlPerfCompliance(self): - string_io = six.StringIO() - handler = logging.StreamHandler(string_io) - data_dir = test_util.create_black_and_white_images() - try: - mlperf_log.LOGGER.addHandler(handler) - params = benchmark_cnn.make_params(data_dir=data_dir, - data_name='imagenet', - batch_size=2, - num_warmup_batches=0, - num_batches=2, - num_eval_batches=3, - eval_during_training_every_n_steps=1, - distortions=False, - weight_decay=0.5, - optimizer='momentum', - momentum=0.5, - stop_at_top_1_accuracy=2.0, - tf_random_seed=9876, - ml_perf=True) - with mlperf.mlperf_logger(use_mlperf_logger=True, model='resnet50_v1.5'): - bench_cnn = benchmark_cnn.BenchmarkCNN(params, model=_MlPerfTestModel()) - bench_cnn.run() - logs = string_io.getvalue().splitlines() - log_regexes = Counter() - for log in logs: - for regex in self.EXPECTED_LOG_REGEXES: - if regex.search(log): - log_regexes[regex] += 1 - if log_regexes != self.EXPECTED_LOG_REGEXES: - diff_counter = Counter(log_regexes) - diff_counter.subtract(self.EXPECTED_LOG_REGEXES) - differences = [] - for regex in (k for k in diff_counter.keys() if diff_counter[k]): - found_count = log_regexes[regex] - expected_count = self.EXPECTED_LOG_REGEXES[regex] - differences.append(' For regex %s: Found %d lines matching but ' - 'expected to find %d' % - (regex.pattern, found_count, expected_count)) - raise AssertionError('Logs did not match expected logs. Differences:\n' - '%s' % '\n'.join(differences)) - finally: - mlperf_log.LOGGER.removeHandler(handler) - -if __name__ == '__main__': - tf.disable_v2_behavior() - tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/alexnet_model.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/alexnet_model.py deleted file mode 100644 index 2f4611fd60d19a3dd704e47323e7fa9a5320f596..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/alexnet_model.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Alexnet model configuration. - -References: - Krizhevsky, Alex, Ilya Sutskever, and Geoffrey E. Hinton - ImageNet Classification with Deep Convolutional Neural Networks - Advances in Neural Information Processing Systems. 2012 -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow.compat.v1 as tf -from models import model - - -class AlexnetModel(model.CNNModel): - """Alexnet cnn model.""" - - def __init__(self, params=None): - super(AlexnetModel, self).__init__( - 'alexnet', 224 + 3, 512, 0.005, params=params) - - def add_inference(self, cnn): - # Note: VALID requires padding the images by 3 in width and height - cnn.conv(64, 11, 11, 4, 4, 'VALID') - cnn.mpool(3, 3, 2, 2) - cnn.conv(192, 5, 5) - cnn.mpool(3, 3, 2, 2) - cnn.conv(384, 3, 3) - cnn.conv(384, 3, 3) - cnn.conv(256, 3, 3) - cnn.mpool(3, 3, 2, 2) - cnn.reshape([-1, 256 * 6 * 6]) - cnn.affine(4096) - cnn.dropout() - cnn.affine(4096) - cnn.dropout() - - -class AlexnetCifar10Model(model.CNNModel): - """Alexnet cnn model for cifar datasets. - - The model architecture follows the one defined in the tensorflow tutorial - model. - - Reference model: tensorflow/models/tutorials/image/cifar10/cifar10.py - Paper: http://www.cs.toronto.edu/~kriz/learning-features-2009-TR.pdf - """ - - def __init__(self, params=None): - super(AlexnetCifar10Model, self).__init__( - 'alexnet', 32, 128, 0.1, params=params) - - def add_inference(self, cnn): - cnn.conv(64, 5, 5, 1, 1, 'SAME', stddev=5e-2) - cnn.mpool(3, 3, 2, 2, mode='SAME') - cnn.lrn(depth_radius=4, bias=1.0, alpha=0.001 / 9.0, beta=0.75) - cnn.conv(64, 5, 5, 1, 1, 'SAME', bias=0.1, stddev=5e-2) - cnn.lrn(depth_radius=4, bias=1.0, alpha=0.001 / 9.0, beta=0.75) - cnn.mpool(3, 3, 2, 2, mode='SAME') - shape = cnn.top_layer.get_shape().as_list() - flat_dim = shape[1] * shape[2] * shape[3] - cnn.reshape([-1, flat_dim]) - cnn.affine(384, stddev=0.04, bias=0.1) - cnn.affine(192, stddev=0.04, bias=0.1) - - def get_learning_rate(self, global_step, batch_size): - num_examples_per_epoch = 50000 - num_epochs_per_decay = 100 - decay_steps = ( - num_epochs_per_decay * num_examples_per_epoch // batch_size) - decay_factor = 0.1 - return tf.train.exponential_decay( - self.learning_rate, - global_step, - decay_steps, - decay_factor, - staircase=True) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/densenet_model.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/densenet_model.py deleted file mode 100644 index cb61b9b3f3332587daa2e308ba6d722cba408e1b..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/densenet_model.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Densenet model configuration. - -References: - "Densely Connected Convolutional Networks": https://arxiv.org/pdf/1608.06993 -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow.compat.v1 as tf -from models import model as model_lib - - -class DensenetCifar10Model(model_lib.CNNModel): - """Densenet cnn network configuration.""" - - def __init__(self, model, layer_counts, growth_rate, params=None): - self.growth_rate = growth_rate - super(DensenetCifar10Model, self).__init__( - model, 32, 64, 0.1, layer_counts=layer_counts, params=params) - self.batch_norm_config = {'decay': 0.9, 'epsilon': 1e-5, 'scale': True} - - def dense_block(self, cnn, growth_rate): - input_layer = cnn.top_layer - c = cnn.batch_norm(input_layer, **self.batch_norm_config) - c = tf.nn.relu(c) - c = cnn.conv(growth_rate, 3, 3, 1, 1, stddev=np.sqrt(2.0/9/growth_rate), - activation=None, input_layer=c) - channel_index = 3 if cnn.channel_pos == 'channels_last' else 1 - cnn.top_layer = tf.concat([input_layer, c], channel_index) - cnn.top_size += growth_rate - - def transition_layer(self, cnn): - in_size = cnn.top_size - cnn.batch_norm(**self.batch_norm_config) - cnn.top_layer = tf.nn.relu(cnn.top_layer) - cnn.conv(in_size, 1, 1, 1, 1, stddev=np.sqrt(2.0/9/in_size)) - cnn.apool(2, 2, 2, 2) - - def add_inference(self, cnn): - if self.layer_counts is None: - raise ValueError('Layer counts not specified for %s' % self.get_model()) - if self.growth_rate is None: - raise ValueError('Growth rate not specified for %s' % self.get_model()) - - cnn.conv(16, 3, 3, 1, 1, activation=None) - # Block 1 - for _ in xrange(self.layer_counts[0]): - self.dense_block(cnn, self.growth_rate) - self.transition_layer(cnn) - # Block 2 - for _ in xrange(self.layer_counts[1]): - self.dense_block(cnn, self.growth_rate) - self.transition_layer(cnn) - # Block 3 - for _ in xrange(self.layer_counts[2]): - self.dense_block(cnn, self.growth_rate) - cnn.batch_norm(**self.batch_norm_config) - cnn.top_layer = tf.nn.relu(cnn.top_layer) - channel_index = 3 if cnn.channel_pos == 'channels_last' else 1 - cnn.top_size = cnn.top_layer.get_shape().as_list()[channel_index] - cnn.spatial_mean() - - def get_learning_rate(self, global_step, batch_size): - num_batches_per_epoch = 50000 // batch_size - boundaries = num_batches_per_epoch * np.array([150, 225, 300], - dtype=np.int64) - boundaries = [x for x in boundaries] - values = [0.1, 0.01, 0.001, 0.0001] - return tf.train.piecewise_constant(global_step, boundaries, values) - - -def create_densenet40_k12_model(): - return DensenetCifar10Model('densenet40_k12', (12, 12, 12), 12) - - -def create_densenet100_k12_model(): - return DensenetCifar10Model('densenet100_k12', (32, 32, 32), 12) - - -def create_densenet100_k24_model(): - return DensenetCifar10Model('densenet100_k24', (32, 32, 32), 24) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/experimental/deepspeech.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/experimental/deepspeech.py deleted file mode 100644 index 9475e36527ca84c869c6e36cd14f7cc3f1e51339..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/experimental/deepspeech.py +++ /dev/null @@ -1,449 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""DeepSpeech2 model configuration. - -References: - https://arxiv.org/abs/1512.02595 - Deep Speech 2: End-to-End Speech Recognition in English and Mandarin -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools - -import numpy as np -from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow.compat.v1 as tf -import constants -from cnn_util import log_fn -from models import model as model_lib -from tensorflow.python.ops import variables # pylint: disable=g-direct-tensorflow-import - - -class DeepSpeechDecoder(object): - """Greedy decoder implementation for Deep Speech model.""" - - def __init__(self, labels, blank_index=28): - """Decoder initialization. - - Args: - labels: a string specifying the speech labels for the decoder to use. - blank_index: an integer specifying index for the blank character. Defaults - to 28. - """ - self.labels = labels - self.blank_index = blank_index - self.int_to_char = dict([(i, c) for (i, c) in enumerate(labels)]) - - def convert_to_string(self, sequence): - """Convert a sequence of indexes into corresponding string.""" - return ''.join([self.int_to_char[i] for i in sequence]) - - def wer(self, decode, target): - """Computes the Word Error Rate (WER). - - WER is defined as the edit distance between the two provided sentences after - tokenizing to words. - - Args: - decode: string of the decoded output. - target: a string for the ground truth label. - - Returns: - A float number for the WER of the current decode-target pair. - """ - try: - from nltk.metrics import distance # pylint: disable=g-import-not-at-top - except ImportError as e: - if 'nltk.metrics' not in e.message: - raise - raise ImportError('To use the experimental deepspeech model, you must ' - 'pip install -U nltk') - - # Map each word to a new char. - words = set(decode.split() + target.split()) - word2char = dict(zip(words, range(len(words)))) - - new_decode = [chr(word2char[w]) for w in decode.split()] - new_target = [chr(word2char[w]) for w in target.split()] - - return distance.edit_distance(''.join(new_decode), ''.join(new_target)) - - def cer(self, decode, target): - """Computes the Character Error Rate (CER). - - CER is defined as the edit distance between the two given strings. - - Args: - decode: a string of the decoded output. - target: a string for the ground truth label. - - Returns: - A float number denoting the CER for the current sentence pair. - """ - try: - from nltk.metrics import distance # pylint: disable=g-import-not-at-top - except ImportError as e: - if 'nltk.metrics' not in e.message: - raise - raise ImportError('To use the experimental deepspeech model, you must ' - 'pip install -U nltk') - return distance.edit_distance(decode, target) - - def decode(self, char_indexes): - """Decode the best guess from logits using greedy algorithm.""" - # Merge repeated chars. - merge = [k for k, _ in itertools.groupby(char_indexes)] - # Remove the blank index in the decoded sequence. - merge_remove_blank = [] - for k in merge: - if k != self.blank_index: - merge_remove_blank.append(k) - - return self.convert_to_string(merge_remove_blank) - - def decode_logits(self, logits): - """Decode the best guess from logits using greedy algorithm.""" - # Choose the class with maximimum probability. - best = list(np.argmax(logits, axis=1)) - return self.decode(best) - - -class DeepSpeech2Model(model_lib.Model): - """Define DeepSpeech2 model.""" - - # Supported rnn cells. - SUPPORTED_RNNS = { - 'lstm': tf.nn.rnn_cell.BasicLSTMCell, - 'rnn': tf.nn.rnn_cell.RNNCell, - 'gru': tf.nn.rnn_cell.GRUCell, - } - - # Parameters for batch normalization. - BATCH_NORM_EPSILON = 1e-5 - BATCH_NORM_DECAY = 0.997 - - # Filters of convolution layer - CONV_FILTERS = 32 - - def __init__(self, - num_rnn_layers=5, - rnn_type='lstm', - is_bidirectional=True, - rnn_hidden_size=800, - use_bias=True, - params=None): - """Initialize DeepSpeech2 model. - - Args: - num_rnn_layers: an integer, the number of rnn layers (default: 5). - rnn_type: a string, one of the supported rnn cells: gru, rnn or lstm. - is_bidirectional: a boolean to indicate if the rnn layer is bidirectional. - rnn_hidden_size: an integer for the number of hidden units in the RNN - cell. - use_bias: a boolean specifying whether to use a bias in the last fc layer. - params: the params from BenchmarkCNN. - """ - super(DeepSpeech2Model, self).__init__( - 'deepspeech2', - batch_size=128, - learning_rate=0.0005, - fp16_loss_scale=128, - params=params) - self.num_rnn_layers = num_rnn_layers - self.rnn_type = rnn_type - self.is_bidirectional = is_bidirectional - self.rnn_hidden_size = rnn_hidden_size - self.use_bias = use_bias - self.num_feature_bins = 161 - self.max_time_steps = 3494 - self.max_label_length = 576 - - def _batch_norm(self, inputs, training): - """Batch normalization layer. - - Note that the momentum to use will affect validation accuracy over time. - Batch norm has different behaviors during training/evaluation. With a large - momentum, the model takes longer to get a near-accurate estimation of the - moving mean/variance over the entire training dataset, which means we need - more iterations to see good evaluation results. If the training data is - evenly distributed over the feature space, we can also try setting a smaller - momentum (such as 0.1) to get good evaluation result sooner. - - Args: - inputs: input data for batch norm layer. - training: a boolean to indicate if it is in training stage. - - Returns: - tensor output from batch norm layer. - """ - return tf.layers.batch_normalization( - inputs=inputs, - momentum=DeepSpeech2Model.BATCH_NORM_DECAY, - epsilon=DeepSpeech2Model.BATCH_NORM_EPSILON, - fused=True, - training=training) - - def _conv_bn_layer(self, inputs, padding, filters, kernel_size, strides, - layer_id, training): - """Defines 2D convolutional + batch normalization layer. - - Args: - inputs: input data for convolution layer. - padding: padding to be applied before convolution layer. - filters: an integer, number of output filters in the convolution. - kernel_size: a tuple specifying the height and width of the 2D convolution - window. - strides: a tuple specifying the stride length of the convolution. - layer_id: an integer specifying the layer index. - training: a boolean to indicate which stage we are in (training/eval). - - Returns: - tensor output from the current layer. - """ - # Perform symmetric padding on the feature dimension of time_step - # This step is required to avoid issues when RNN output sequence is shorter - # than the label length. - inputs = tf.pad( - inputs, - [[0, 0], [padding[0], padding[0]], [padding[1], padding[1]], [0, 0]]) - inputs = tf.layers.conv2d( - inputs=inputs, - filters=filters, - kernel_size=kernel_size, - strides=strides, - padding='valid', - use_bias=False, - activation=tf.nn.relu6, - name='cnn_{}'.format(layer_id)) - return self._batch_norm(inputs, training) - - def _rnn_layer(self, inputs, rnn_cell, rnn_hidden_size, layer_id, - use_batch_norm, is_bidirectional, training): - """Defines a batch normalization + rnn layer. - - Args: - inputs: input tensors for the current layer. - rnn_cell: RNN cell instance to use. - rnn_hidden_size: an integer for the dimensionality of the rnn output - space. - layer_id: an integer for the index of current layer. - use_batch_norm: a boolean specifying whether to perform batch - normalization on input states. - is_bidirectional: a boolean specifying whether the rnn layer is - bi-directional. - training: a boolean to indicate which stage we are in (training/eval). - - Returns: - tensor output for the current layer. - """ - if use_batch_norm: - inputs = self._batch_norm(inputs, training) - - # Construct forward/backward RNN cells. - fw_cell = rnn_cell( - num_units=rnn_hidden_size, name='rnn_fw_{}'.format(layer_id)) - - if is_bidirectional: - bw_cell = rnn_cell( - num_units=rnn_hidden_size, name='rnn_bw_{}'.format(layer_id)) - outputs, _ = tf.nn.bidirectional_dynamic_rnn( - cell_fw=fw_cell, - cell_bw=bw_cell, - inputs=inputs, - dtype=tf.float32, - swap_memory=True) - rnn_outputs = tf.concat(outputs, -1) - else: - rnn_outputs = tf.nn.dynamic_rnn( - fw_cell, inputs, dtype=tf.float32, swap_memory=True) - - return rnn_outputs - - def get_input_data_types(self, subset): - """Returns the list of data types of the inputs.""" - del subset # Same data types for both train and validation subsets. - return [self.data_type, tf.int32, tf.int32, tf.int32] - - def get_input_shapes(self, subset): - """Returns the list of shapes of the padded inputs.""" - del subset # Same shapes for both train and validation subsets - return [ - [self.batch_size, self.max_time_steps, self.num_feature_bins, 1], - [self.batch_size, self.max_label_length], - [self.batch_size, 1], - [self.batch_size, 1], - ] - - def get_synthetic_inputs(self, input_name, nclass): - inputs = tf.random_uniform(self.get_input_shapes('train')[0], - dtype=self.get_input_data_types('train')[0]) - inputs = variables.VariableV1(inputs, trainable=False, - collections=[tf.GraphKeys.LOCAL_VARIABLES], - name=input_name) - labels = tf.convert_to_tensor( - np.random.randint(28, size=[self.batch_size, self.max_label_length])) - input_lengths = tf.convert_to_tensor( - [self.max_time_steps] * self.batch_size) - label_lengths = tf.convert_to_tensor( - [self.max_label_length] * self.batch_size) - return [inputs, labels, input_lengths, label_lengths] - - # TODO(laigd): support fp16. - # TODO(laigd): support multiple gpus. - def build_network(self, inputs, phase_train=True, nclass=29): - """Builds the forward pass of the deepspeech2 model. - - Args: - inputs: The input list of the model. - phase_train: True during training. False during evaluation. - nclass: Number of classes that the input spectrogram can belong to. - - Returns: - A BuildNetworkResult which contains the logits and model-specific extra - information. - """ - inputs = inputs[0] # Get the spectrogram feature. - - # Two cnn layers. - inputs = self._conv_bn_layer( - inputs, - padding=(20, 5), - filters=DeepSpeech2Model.CONV_FILTERS, - kernel_size=(41, 11), - strides=(2, 2), - layer_id=1, - training=phase_train) - - inputs = self._conv_bn_layer( - inputs, - padding=(10, 5), - filters=DeepSpeech2Model.CONV_FILTERS, - kernel_size=(21, 11), - strides=(2, 1), - layer_id=2, - training=phase_train) - - # output of conv_layer2 with the shape of - # [batch_size (N), times (T), features (F), channels (C)]. - # Convert the conv output to rnn input. - - # batch_size = tf.shape(inputs)[0] - feat_size = inputs.get_shape().as_list()[2] - inputs = tf.reshape( - inputs, - [self.batch_size, -1, feat_size * DeepSpeech2Model.CONV_FILTERS]) - - # RNN layers. - rnn_cell = DeepSpeech2Model.SUPPORTED_RNNS[self.rnn_type] - for layer_counter in xrange(self.num_rnn_layers): - # No batch normalization on the first layer. - use_batch_norm = (layer_counter != 0) - inputs = self._rnn_layer(inputs, rnn_cell, self.rnn_hidden_size, - layer_counter + 1, use_batch_norm, - self.is_bidirectional, phase_train) - - # FC layer with batch norm. - inputs = self._batch_norm(inputs, phase_train) - logits = tf.layers.dense(inputs, nclass, use_bias=self.use_bias) - - return model_lib.BuildNetworkResult(logits=logits, extra_info=None) - - def loss_function(self, inputs, build_network_result): - """Computes the ctc loss for the current batch of predictions. - - Args: - inputs: the input list of the model. - build_network_result: a BuildNetworkResult returned by build_network(). - - Returns: - The loss tensor of the model. - """ - logits = build_network_result.logits - actual_time_steps = inputs[2] - probs = tf.nn.softmax(logits) - ctc_time_steps = tf.shape(probs)[1] - ctc_input_length = tf.to_float( - tf.multiply(actual_time_steps, ctc_time_steps)) - ctc_input_length = tf.to_int32( - tf.floordiv(ctc_input_length, tf.to_float(self.max_time_steps))) - - label_length = inputs[3] - label_length = tf.to_int32(tf.squeeze(label_length)) - ctc_input_length = tf.to_int32(tf.squeeze(ctc_input_length)) - - labels = inputs[1] - sparse_labels = tf.to_int32( - tf.keras.backend.ctc_label_dense_to_sparse(labels, label_length)) - y_pred = tf.log( - tf.transpose(probs, perm=[1, 0, 2]) + tf.keras.backend.epsilon()) - - losses = tf.expand_dims( - tf.nn.ctc_loss( - labels=sparse_labels, - inputs=y_pred, - sequence_length=ctc_input_length, - ignore_longer_outputs_than_inputs=True), - axis=1) - loss = tf.reduce_mean(losses) - return loss - - PROBABILITY_TENSOR = 'deepspeech2_prob' - LABEL_TENSOR = 'deepspeech2_label' - - def accuracy_function(self, inputs, logits): - """Returns the ops to evaluate the model performance.""" - # Get probabilities of each predicted class - probs = tf.nn.softmax(logits) - assert probs.shape.as_list()[0] == self.batch_size - return { - (constants.UNREDUCED_ACCURACY_OP_PREFIX + self.PROBABILITY_TENSOR): - probs, - (constants.UNREDUCED_ACCURACY_OP_PREFIX + self.LABEL_TENSOR): - inputs[1], - } - - def postprocess(self, results): - """Postprocess results returned from model in Python.""" - probs = results[self.PROBABILITY_TENSOR] - - total_wer, total_cer = 0, 0 - speech_labels = " abcdefghijklmnopqrstuvwxyz'-" - greedy_decoder = DeepSpeechDecoder(speech_labels) - - # Evaluate the performance using WER (Word Error Rate) and CER (Character - # Error Rate) as metrics. - targets = results[self.LABEL_TENSOR] # The ground truth transcript - for i in range(self.batch_size): - # Decode string. - predicted_str = greedy_decoder.decode_logits(probs[i]) - expected_str = greedy_decoder.decode(targets[i]) - # Compute CER. - total_cer += (greedy_decoder.cer(predicted_str, expected_str) / - len(expected_str)) - # Compute WER. - total_wer += (greedy_decoder.wer(predicted_str, expected_str) / - len(expected_str.split())) - - # Get mean value - total_cer /= self.batch_size - total_wer /= self.batch_size - - log_fn('total CER: {:f}; total WER: {:f}; total example: {:d}.'.format( - total_cer, total_wer, self.batch_size)) - # TODO(laigd): get rid of top_N_accuracy bindings in benchmark_cnn.py - return {'top_1_accuracy': 0., 'top_5_accuracy': 0.} diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/experimental/official_ncf_model.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/experimental/official_ncf_model.py deleted file mode 100644 index 9e6ca513f9f0c3f9b7c67bc7a072ed0b35fd4f5a..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/experimental/official_ncf_model.py +++ /dev/null @@ -1,172 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Wrap the official recommendation model in a tf_cnn_benchmarks Model. - -This allows the recommendation NCF model to be used in tf_cnn_benchmarks. -Currently, the implementation is fairly hacky, because tf_cnn_benchmarks is -intended to be used only with CNNs. - -Only synthetic data with 1 GPU is currently supported. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow.compat.v1 as tf - -from models import model - - -# Obtained by running the official NCF model with the following command: -# python ncf_main.py --dataset ml-20m -# and printing the number of users and items here: -# https://github.com/tensorflow/models/blob/d089975f630a8a01be63e45ef08a31be14bb96b4/official/recommendation/data_preprocessing.py#L68 -_NUM_USERS_20M = 138493 -_NUM_ITEMS_20M = 26744 - - -# TODO(reedwm): Support multi-GPU. Currently keras layers, which this model -# uses, ignore variable_scopes, which we rely on for multi-GPU support. -# TODO(reedwm): Support real data. This will require a significant refactor. -# TODO(reedwm): All-reduce IndexedSlices more effectively. -# TODO(reedwm): Support the 1M variant of this model. - - -class NcfModel(model.Model): - r"""A model.Model wrapper around the official NCF recommendation model. - - To do an NCF run with synthetic data that roughly matches what the official - model does, run: - - python tf_cnn_benchmarks.py --optimizer=adam --model=ncf --batch_size=65536 \ - --weight_decay=0 --sparse_to_dense_grads - """ - - def __init__(self, params=None): - super(NcfModel, self).__init__( - 'official_ncf', batch_size=2048, learning_rate=0.0005, - fp16_loss_scale=128, params=params) - if self.fp16_vars: - raise ValueError('NCF model only supports float32 variables for now.') - - def build_network(self, inputs, phase_train=True, nclass=1001): - try: - from official.recommendation import neumf_model # pylint: disable=g-import-not-at-top - except ImportError as e: - if 'neumf_model' not in e.message: - raise - raise ImportError('To use the experimental NCF model, you must clone the ' - 'repo https://github.com/tensorflow/models and add ' - 'tensorflow/models to the PYTHONPATH.') - del nclass - - users, items, _ = inputs - params = { - 'num_users': _NUM_USERS_20M, - 'num_items': _NUM_ITEMS_20M, - 'model_layers': (256, 256, 128, 64), - 'mf_dim': 64, - 'mf_regularization': 0, - 'mlp_reg_layers': (0, 0, 0, 0), - 'use_tpu': False - } - user_input = tf.keras.layers.Input(tensor=users, name='user_input') - item_input = tf.keras.layers.Input(tensor=items, name='item_input') - if self.data_type == tf.float32: - keras_model = neumf_model.construct_model(user_input, item_input, params) - logits = keras_model.output - else: - assert self.data_type == tf.float16 - old_floatx = tf.keras.backend.floatx() - try: - tf.keras.backend.set_floatx('float16') - # We cannot rely on the variable_scope's fp16 custom getter here, - # because the NCF model uses keras layers, which ignore variable scopes. - # So we use a variable_creator_scope instead. - with tf.variable_creator_scope(_fp16_variable_creator): - keras_model = neumf_model.construct_model(user_input, item_input, - params) - logits = tf.cast(keras_model.output, tf.float32) - finally: - tf.keras.backend.set_floatx(old_floatx) - return model.BuildNetworkResult(logits=logits, extra_info=None) - - def loss_function(self, inputs, build_network_result): - logits = build_network_result.logits - - # Softmax with the first column of ones is equivalent to sigmoid. - # TODO(reedwm): Actually, the first column should be zeros to be equivalent - # to sigmoid. But, we keep it at ones to match the official models. - logits = tf.concat([tf.ones(logits.shape, dtype=logits.dtype), logits], - axis=1) - - return tf.losses.sparse_softmax_cross_entropy( - labels=inputs[2], - logits=logits - ) - - def get_synthetic_inputs(self, input_name, nclass): - """Returns the ops to generate synthetic inputs and labels.""" - def users_init_val(): - return tf.random_uniform((self.batch_size, 1), minval=0, - maxval=_NUM_USERS_20M, dtype=tf.int32) - users = tf.Variable(users_init_val, dtype=tf.int32, trainable=False, - collections=[tf.GraphKeys.LOCAL_VARIABLES], - name='synthetic_users') - def items_init_val(): - return tf.random_uniform((self.batch_size, 1), minval=0, - maxval=_NUM_ITEMS_20M, dtype=tf.int32) - items = tf.Variable(items_init_val, dtype=tf.int32, trainable=False, - collections=[tf.GraphKeys.LOCAL_VARIABLES], - name='synthetic_items') - - def labels_init_val(): - return tf.random_uniform((self.batch_size,), minval=0, maxval=2, - dtype=tf.int32) - labels = tf.Variable(labels_init_val, dtype=tf.int32, trainable=False, - collections=[tf.GraphKeys.LOCAL_VARIABLES], - name='synthetic_labels') - - return [users, items, labels] - - def get_input_shapes(self, subset): - del subset - return [[self.batch_size, 1], [self.batch_size, 1], [self.batch_size]] - - def get_input_data_types(self, subset): - del subset - return [self.int32, tf.int32, tf.int32] - - -def _fp16_variable_creator(next_creator, **kwargs): - """Variable creator to create variables in fp32 and cast them to fp16.""" - dtype = kwargs.get('dtype', None) - initial_value = kwargs.get('initial_value', None) - if dtype is None: - if initial_value is not None and not callable(initial_value): - dtype = initial_value.dtype - if dtype == tf.float16: - if callable(initial_value): - new_initial_value = lambda: tf.cast(initial_value(), tf.float32) - else: - new_initial_value = tf.cast(initial_value, tf.float32) - kwargs['dtype'] = tf.float32 - kwargs['initial_value'] = new_initial_value - var = next_creator(**kwargs) - return tf.cast(var, dtype=tf.float16) - else: - return next_creator(**kwargs) - diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/googlenet_model.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/googlenet_model.py deleted file mode 100644 index 3505594ec933cc05cb96b00eeac81cbc4334693c..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/googlenet_model.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Googlenet model configuration. - -References: - Szegedy, Christian, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, - Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, and Andrew Rabinovich - Going deeper with convolutions - arXiv preprint arXiv:1409.4842 (2014) -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from models import model - - -class GooglenetModel(model.CNNModel): - """GoogLeNet.""" - - def __init__(self, params=None): - super(GooglenetModel, self).__init__( - 'googlenet', 224, 32, 0.005, params=params) - - def add_inference(self, cnn): - - def inception_v1(cnn, k, l, m, n, p, q): - cols = [[('conv', k, 1, 1)], [('conv', l, 1, 1), ('conv', m, 3, 3)], - [('conv', n, 1, 1), ('conv', p, 5, 5)], - [('mpool', 3, 3, 1, 1, 'SAME'), ('conv', q, 1, 1)]] - cnn.inception_module('incept_v1', cols) - - cnn.conv(64, 7, 7, 2, 2) - cnn.mpool(3, 3, 2, 2, mode='SAME') - cnn.conv(64, 1, 1) - cnn.conv(192, 3, 3) - cnn.mpool(3, 3, 2, 2, mode='SAME') - inception_v1(cnn, 64, 96, 128, 16, 32, 32) - inception_v1(cnn, 128, 128, 192, 32, 96, 64) - cnn.mpool(3, 3, 2, 2, mode='SAME') - inception_v1(cnn, 192, 96, 208, 16, 48, 64) - inception_v1(cnn, 160, 112, 224, 24, 64, 64) - inception_v1(cnn, 128, 128, 256, 24, 64, 64) - inception_v1(cnn, 112, 144, 288, 32, 64, 64) - inception_v1(cnn, 256, 160, 320, 32, 128, 128) - cnn.mpool(3, 3, 2, 2, mode='SAME') - inception_v1(cnn, 256, 160, 320, 32, 128, 128) - inception_v1(cnn, 384, 192, 384, 48, 128, 128) - cnn.apool(7, 7, 1, 1, mode='VALID') - cnn.reshape([-1, 1024]) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/inception_model.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/inception_model.py deleted file mode 100644 index b8835edb88cb57fde2b67bc8cb5fb2caffa0527f..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/inception_model.py +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Inception model configuration. - -Includes multiple models: inception3, inception4, inception-resnet2. - -References: - Christian Szegedy, Sergey Ioffe, Vincent Vanhoucke, Alex Alemi - Inception-v4, Inception-ResNet and the Impact of Residual Connections on - Learning - - Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, - Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, Andrew Rabinovich - Going Deeper with Convolutions - http://arxiv.org/pdf/1409.4842v1.pdf - - Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, - Zbigniew Wojna - Rethinking the Inception Architecture for Computer Vision - arXiv preprint arXiv:1512.00567 (2015) - - Inception v3 model: http://arxiv.org/abs/1512.00567 - - Inception v4 and Resnet V2 architectures: http://arxiv.org/abs/1602.07261 -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from six.moves import xrange # pylint: disable=redefined-builtin -from models import model - - -class Inceptionv3Model(model.CNNModel): - """InceptionV3.""" - - def __init__(self, auxiliary=False, params=None): - self._auxiliary = auxiliary - super(Inceptionv3Model, self).__init__( - 'inception3', 299, 32, 0.005, params=params) - - def add_inference(self, cnn): - def inception_v3_a(cnn, n): - cols = [[('conv', 64, 1, 1)], [('conv', 48, 1, 1), ('conv', 64, 5, 5)], - [('conv', 64, 1, 1), ('conv', 96, 3, 3), ('conv', 96, 3, 3)], - [('apool', 3, 3, 1, 1, 'SAME'), ('conv', n, 1, 1)]] - cnn.inception_module('incept_v3_a', cols) - - def inception_v3_b(cnn): - cols = [[('conv', 384, 3, 3, 2, 2, 'VALID')], - [('conv', 64, 1, 1), - ('conv', 96, 3, 3), - ('conv', 96, 3, 3, 2, 2, 'VALID')], - [('mpool', 3, 3, 2, 2, 'VALID')]] - cnn.inception_module('incept_v3_b', cols) - - def inception_v3_c(cnn, n): - cols = [[('conv', 192, 1, 1)], - [('conv', n, 1, 1), ('conv', n, 1, 7), ('conv', 192, 7, 1)], - [('conv', n, 1, 1), ('conv', n, 7, 1), ('conv', n, 1, 7), - ('conv', n, 7, 1), ('conv', 192, 1, 7)], - [('apool', 3, 3, 1, 1, 'SAME'), ('conv', 192, 1, 1)]] - cnn.inception_module('incept_v3_c', cols) - - def inception_v3_d(cnn): - cols = [[('conv', 192, 1, 1), ('conv', 320, 3, 3, 2, 2, 'VALID')], - [('conv', 192, 1, 1), ('conv', 192, 1, 7), ('conv', 192, 7, 1), - ('conv', 192, 3, 3, 2, 2, 'VALID')], - [('mpool', 3, 3, 2, 2, 'VALID')]] - cnn.inception_module('incept_v3_d', cols) - - def inception_v3_e(cnn, pooltype): - cols = [[('conv', 320, 1, 1)], [('conv', 384, 1, 1), ('conv', 384, 1, 3)], - [('share',), ('conv', 384, 3, 1)], - [('conv', 448, 1, 1), ('conv', 384, 3, 3), ('conv', 384, 1, 3)], - [('share',), ('share',), ('conv', 384, 3, 1)], - [('mpool' if pooltype == 'max' else 'apool', 3, 3, 1, 1, 'SAME'), - ('conv', 192, 1, 1)]] - cnn.inception_module('incept_v3_e', cols) - - def incept_v3_aux(cnn): - assert cnn.aux_top_layer is None - cnn.aux_top_layer = cnn.top_layer - cnn.aux_top_size = cnn.top_size - with cnn.switch_to_aux_top_layer(): - cnn.apool(5, 5, 3, 3, mode='VALID') - cnn.conv(128, 1, 1, mode='SAME') - cnn.conv(768, 5, 5, mode='VALID', stddev=0.01) - cnn.reshape([-1, 768]) - - cnn.use_batch_norm = True - cnn.conv(32, 3, 3, 2, 2, mode='VALID') # 299 x 299 x 3 - cnn.conv(32, 3, 3, 1, 1, mode='VALID') # 149 x 149 x 32 - cnn.conv(64, 3, 3, 1, 1, mode='SAME') # 147 x 147 x 64 - cnn.mpool(3, 3, 2, 2, mode='VALID') # 147 x 147 x 64 - cnn.conv(80, 1, 1, 1, 1, mode='VALID') # 73 x 73 x 80 - cnn.conv(192, 3, 3, 1, 1, mode='VALID') # 71 x 71 x 192 - cnn.mpool(3, 3, 2, 2, 'VALID') # 35 x 35 x 192 - inception_v3_a(cnn, 32) # 35 x 35 x 256 mixed. - inception_v3_a(cnn, 64) # 35 x 35 x 288 mixed_1. - inception_v3_a(cnn, 64) # 35 x 35 x 288 mixed_2 - inception_v3_b(cnn) # 17 x 17 x 768 mixed_3 - inception_v3_c(cnn, 128) # 17 x 17 x 768 mixed_4 - inception_v3_c(cnn, 160) # 17 x 17 x 768 mixed_5 - inception_v3_c(cnn, 160) # 17 x 17 x 768 mixed_6 - inception_v3_c(cnn, 192) # 17 x 17 x 768 mixed_7 - if self._auxiliary: - incept_v3_aux(cnn) # Auxillary Head logits - inception_v3_d(cnn) # 17 x 17 x 1280 mixed_8 - inception_v3_e(cnn, 'avg') # 8 x 8 x 2048 mixed_9 - inception_v3_e(cnn, 'max') # 8 x 8 x 2048 mixed_10 - cnn.apool(8, 8, 1, 1, 'VALID') # 8 x 8 x 2048 - cnn.reshape([-1, 2048]) # 1 x 1 x 2048 - - -# Stem functions -def inception_v4_sa(cnn): - cols = [[('mpool', 3, 3, 2, 2, 'VALID')], [('conv', 96, 3, 3, 2, 2, 'VALID')]] - cnn.inception_module('incept_v4_sa', cols) - - -def inception_v4_sb(cnn): - cols = [[('conv', 64, 1, 1), ('conv', 96, 3, 3, 1, 1, 'VALID')], - [('conv', 64, 1, 1), ('conv', 64, 7, 1), ('conv', 64, 1, 7), - ('conv', 96, 3, 3, 1, 1, 'VALID')]] - cnn.inception_module('incept_v4_sb', cols) - - -def inception_v4_sc(cnn): - cols = [[('conv', 192, 3, 3, 2, 2, 'VALID')], - [('mpool', 3, 3, 2, 2, 'VALID')]] - cnn.inception_module('incept_v4_sc', cols) - - -# Reduction functions -def inception_v4_ra(cnn, k, l, m, n): - cols = [ - [('mpool', 3, 3, 2, 2, 'VALID')], [('conv', n, 3, 3, 2, 2, 'VALID')], - [('conv', k, 1, 1), ('conv', l, 3, 3), ('conv', m, 3, 3, 2, 2, 'VALID')] - ] - cnn.inception_module('incept_v4_ra', cols) - - -def inception_v4_rb(cnn): - cols = [[('mpool', 3, 3, 2, 2, 'VALID')], - [('conv', 192, 1, 1), ('conv', 192, 3, 3, 2, 2, 'VALID')], - [('conv', 256, 1, 1), ('conv', 256, 1, 7), ('conv', 320, 7, 1), - ('conv', 320, 3, 3, 2, 2, 'VALID')]] - cnn.inception_module('incept_v4_rb', cols) - - -class Inceptionv4Model(model.CNNModel): - """Inceptionv4.""" - - def __init__(self, params=None): - super(Inceptionv4Model, self).__init__( - 'inception4', 299, 32, 0.005, params=params) - - def add_inference(self, cnn): - def inception_v4_a(cnn): - cols = [[('apool', 3, 3, 1, 1, 'SAME'), ('conv', 96, 1, 1)], - [('conv', 96, 1, 1)], [('conv', 64, 1, 1), ('conv', 96, 3, 3)], - [('conv', 64, 1, 1), ('conv', 96, 3, 3), ('conv', 96, 3, 3)]] - cnn.inception_module('incept_v4_a', cols) - - def inception_v4_b(cnn): - cols = [[('apool', 3, 3, 1, 1, 'SAME'), ('conv', 128, 1, 1)], - [('conv', 384, 1, 1)], - [('conv', 192, 1, 1), ('conv', 224, 1, 7), ('conv', 256, 7, 1)], - [('conv', 192, 1, 1), ('conv', 192, 1, 7), ('conv', 224, 7, 1), - ('conv', 224, 1, 7), ('conv', 256, 7, 1)]] - cnn.inception_module('incept_v4_b', cols) - - def inception_v4_c(cnn): - cols = [[('apool', 3, 3, 1, 1, 'SAME'), ('conv', 256, 1, 1)], - [('conv', 256, 1, 1)], [('conv', 384, 1, 1), ('conv', 256, 1, 3)], - [('share',), ('conv', 256, 3, 1)], - [('conv', 384, 1, 1), ('conv', 448, 1, 3), ('conv', 512, 3, 1), - ('conv', 256, 3, 1)], [('share',), ('share',), ('share',), - ('conv', 256, 1, 3)]] - cnn.inception_module('incept_v4_c', cols) - - cnn.use_batch_norm = True - cnn.conv(32, 3, 3, 2, 2, mode='VALID') - cnn.conv(32, 3, 3, 1, 1, mode='VALID') - cnn.conv(64, 3, 3) - inception_v4_sa(cnn) - inception_v4_sb(cnn) - inception_v4_sc(cnn) - for _ in xrange(4): - inception_v4_a(cnn) - inception_v4_ra(cnn, 192, 224, 256, 384) - for _ in xrange(7): - inception_v4_b(cnn) - inception_v4_rb(cnn) - for _ in xrange(3): - inception_v4_c(cnn) - cnn.spatial_mean() - cnn.dropout(0.8) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/lenet_model.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/lenet_model.py deleted file mode 100644 index 0218daaeb2b016b7bfcc886af813e92aee25f521..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/lenet_model.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Lenet model configuration. - -References: - LeCun, Yann, Leon Bottou, Yoshua Bengio, and Patrick Haffner - Gradient-based learning applied to document recognition - Proceedings of the IEEE (1998) -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from models import model - - -class Lenet5Model(model.CNNModel): - """Lenet5.""" - - def __init__(self, params=None): - super(Lenet5Model, self).__init__('lenet5', 28, 32, 0.005, params=params) - - def add_inference(self, cnn): - # Note: This matches TF's MNIST tutorial model - cnn.conv(32, 5, 5) - cnn.mpool(2, 2) - cnn.conv(64, 5, 5) - cnn.mpool(2, 2) - cnn.reshape([-1, 64 * 7 * 7]) - cnn.affine(512) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/model.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/model.py deleted file mode 100644 index 3db13081917f9582704428c6c26956cbd652ae77..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/model.py +++ /dev/null @@ -1,340 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Base model configuration for CNN benchmarks.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from collections import namedtuple - -import tensorflow.compat.v1 as tf - -import convnet_builder -import mlperf -from tensorflow.python.ops import variables as variables_module # pylint: disable=g-direct-tensorflow-import - -# BuildNetworkResult encapsulate the result (e.g. logits) of a -# Model.build_network() call. -BuildNetworkResult = namedtuple( - 'BuildNetworkResult', - [ - 'logits', # logits of the network - 'extra_info', # Model specific extra information - ]) - - -class Model(object): - """Base model config for DNN benchmarks.""" - - def __init__(self, - model_name, - batch_size, - learning_rate, - fp16_loss_scale, - params=None): - self.model_name = model_name - self.batch_size = batch_size - self.default_batch_size = batch_size - self.learning_rate = learning_rate - # TODO(reedwm) Set custom loss scales for each model instead of using the - # default of 128. - self.fp16_loss_scale = fp16_loss_scale - - # use_tf_layers specifies whether to build the model using tf.layers. - # fp16_vars specifies whether to create the variables in float16. - if params: - self.use_tf_layers = params.use_tf_layers - self.fp16_vars = params.fp16_vars - self.data_type = tf.float16 if params.use_fp16 else tf.float32 - else: - self.use_tf_layers = True - self.fp16_vars = False - self.data_type = tf.float32 - - def get_model_name(self): - return self.model_name - - def get_batch_size(self): - return self.batch_size - - def set_batch_size(self, batch_size): - self.batch_size = batch_size - - def get_default_batch_size(self): - return self.default_batch_size - - def get_fp16_loss_scale(self): - return self.fp16_loss_scale - - def filter_l2_loss_vars(self, variables): - """Filters out variables that the L2 loss should not be computed for. - - By default, this filters out batch normalization variables and keeps all - other variables. This behavior can be overridden by subclasses. - - Args: - variables: A list of the trainable variables. - - Returns: - A list of variables that the L2 loss should be computed for. - """ - mlperf.logger.log(key=mlperf.tags.MODEL_EXCLUDE_BN_FROM_L2, - value=True) - return [v for v in variables if 'batchnorm' not in v.name] - - def get_learning_rate(self, global_step, batch_size): - del global_step - del batch_size - return self.learning_rate - - def get_input_shapes(self, subset): - """Returns the list of expected shapes of all the inputs to this model.""" - del subset - raise NotImplementedError('Must be implemented in derived classes') - - def get_input_data_types(self, subset): - """Returns the list of data types of all the inputs to this model.""" - del subset - raise NotImplementedError('Must be implemented in derived classes') - - def get_synthetic_inputs(self, input_name, nclass): - """Returns the ops to generate synthetic inputs.""" - raise NotImplementedError('Must be implemented in derived classes') - - def build_network(self, inputs, phase_train, nclass): - """Builds the forward pass of the model. - - Args: - inputs: The list of inputs, including labels - phase_train: True during training. False during evaluation. - nclass: Number of classes that the inputs can belong to. - - Returns: - A BuildNetworkResult which contains the logits and model-specific extra - information. - """ - raise NotImplementedError('Must be implemented in derived classes') - - def loss_function(self, inputs, build_network_result): - """Returns the op to measure the loss of the model. - - Args: - inputs: the input list of the model. - build_network_result: a BuildNetworkResult returned by build_network(). - - Returns: - The loss tensor of the model. - """ - raise NotImplementedError('Must be implemented in derived classes') - - # TODO(laigd): have accuracy_function() take build_network_result instead. - def accuracy_function(self, inputs, logits): - """Returns the ops to measure the accuracy of the model.""" - raise NotImplementedError('Must be implemented in derived classes') - - def postprocess(self, results): - """Postprocess results returned from model in Python.""" - return results - - def reached_target(self): - """Define custom methods to stop training when model's target is reached.""" - return False - - -class CNNModel(Model): - """Base model configuration for CNN benchmarks.""" - - # TODO(laigd): reduce the number of parameters and read everything from - # params. - def __init__(self, - model, - image_size, - batch_size, - learning_rate, - layer_counts=None, - fp16_loss_scale=128, - params=None): - super(CNNModel, self).__init__( - model, batch_size, learning_rate, fp16_loss_scale, - params=params) - self.image_size = image_size - self.layer_counts = layer_counts - self.depth = 3 - self.params = params - self.data_format = params.data_format if params else 'NCHW' - - def get_layer_counts(self): - return self.layer_counts - - def skip_final_affine_layer(self): - """Returns if the caller of this class should skip the final affine layer. - - Normally, this class adds a final affine layer to the model after calling - self.add_inference(), to generate the logits. If a subclass override this - method to return True, the caller should not add the final affine layer. - - This is useful for tests. - """ - return False - - def add_backbone_saver(self): - """Creates a tf.train.Saver as self.backbone_saver for loading backbone. - - A tf.train.Saver must be created and saved in self.backbone_saver before - calling load_backbone_model, with correct variable name mapping to load - variables from checkpoint correctly into the current model. - """ - raise NotImplementedError(self.getName() + ' does not have backbone model.') - - def load_backbone_model(self, sess, backbone_model_path): - """Loads variable values from a pre-trained backbone model. - - This should be used at the beginning of the training process for transfer - learning models using checkpoints of base models. - - Args: - sess: session to train the model. - backbone_model_path: path to backbone model checkpoint file. - """ - del sess, backbone_model_path - raise NotImplementedError(self.getName() + ' does not have backbone model.') - - def add_inference(self, cnn): - """Adds the core layers of the CNN's forward pass. - - This should build the forward pass layers, except for the initial transpose - of the images and the final Dense layer producing the logits. The layers - should be build with the ConvNetBuilder `cnn`, so that when this function - returns, `cnn.top_layer` and `cnn.top_size` refer to the last layer and the - number of units of the layer layer, respectively. - - Args: - cnn: A ConvNetBuilder to build the forward pass layers with. - """ - del cnn - raise NotImplementedError('Must be implemented in derived classes') - - def get_input_data_types(self, subset): - """Return data types of inputs for the specified subset.""" - del subset # Same types for both 'train' and 'validation' subsets. - return [self.data_type, tf.int32] - - def get_input_shapes(self, subset): - """Return data shapes of inputs for the specified subset.""" - del subset # Same shapes for both 'train' and 'validation' subsets. - # Each input is of shape [batch_size, height, width, depth] - # Each label is of shape [batch_size] - return [[self.batch_size, self.image_size, self.image_size, self.depth], - [self.batch_size]] - - def get_synthetic_inputs(self, input_name, nclass): - # Synthetic input should be within [0, 255]. - image_shape, label_shape = self.get_input_shapes('train') - inputs = tf.truncated_normal( - image_shape, - dtype=self.data_type, - mean=127, - stddev=60, - name=self.model_name + '_synthetic_inputs') - inputs = variables_module.VariableV1( - inputs, trainable=False, collections=[tf.GraphKeys.LOCAL_VARIABLES], - name=input_name) - labels = tf.random_uniform( - label_shape, - minval=0, - maxval=nclass - 1, - dtype=tf.int32, - name=self.model_name + '_synthetic_labels') - return (inputs, labels) - - def gpu_preprocess_nhwc(self, images, phase_train=True): - del phase_train - return images - - def build_network(self, - inputs, - phase_train=True, - nclass=1001): - """Returns logits from input images. - - Args: - inputs: The input images and labels - phase_train: True during training. False during evaluation. - nclass: Number of classes that the images can belong to. - - Returns: - A BuildNetworkResult which contains the logits and model-specific extra - information. - """ - images = inputs[0] - images = self.gpu_preprocess_nhwc(images, phase_train) - if self.data_format == 'NCHW': - images = tf.transpose(images, [0, 3, 1, 2]) - var_type = tf.float32 - if self.data_type == tf.float16 and self.fp16_vars: - var_type = tf.float16 - network = convnet_builder.ConvNetBuilder( - images, self.depth, phase_train, self.use_tf_layers, self.data_format, - self.data_type, var_type) - with tf.variable_scope('cg', custom_getter=network.get_custom_getter()): - self.add_inference(network) - # Add the final fully-connected class layer - logits = ( - network.affine(nclass, activation='linear') - if not self.skip_final_affine_layer() else network.top_layer) - mlperf.logger.log(key=mlperf.tags.MODEL_HP_FINAL_SHAPE, - value=logits.shape.as_list()[1:]) - aux_logits = None - if network.aux_top_layer is not None: - with network.switch_to_aux_top_layer(): - aux_logits = network.affine(nclass, activation='linear', stddev=0.001) - if self.data_type == tf.float16: - # TODO(reedwm): Determine if we should do this cast here. - logits = tf.cast(logits, tf.float32) - if aux_logits is not None: - aux_logits = tf.cast(aux_logits, tf.float32) - return BuildNetworkResult( - logits=logits, extra_info=None if aux_logits is None else aux_logits) - - def loss_function(self, inputs, build_network_result): - """Returns the op to measure the loss of the model.""" - logits = build_network_result.logits - _, labels = inputs - # TODO(laigd): consider putting the aux logit in the Inception model, - # which could call super.loss_function twice, once with the normal logits - # and once with the aux logits. - aux_logits = build_network_result.extra_info - with tf.name_scope('xentropy'): - mlperf.logger.log(key=mlperf.tags.MODEL_HP_LOSS_FN, value=mlperf.tags.CCE) - cross_entropy = tf.losses.sparse_softmax_cross_entropy( - logits=logits, labels=labels) - loss = tf.reduce_mean(cross_entropy, name='xentropy_mean') - if aux_logits is not None: - with tf.name_scope('aux_xentropy'): - aux_cross_entropy = tf.losses.sparse_softmax_cross_entropy( - logits=aux_logits, labels=labels) - aux_loss = 0.4 * tf.reduce_mean(aux_cross_entropy, name='aux_loss') - loss = tf.add_n([loss, aux_loss]) - return loss - - def accuracy_function(self, inputs, logits): - """Returns the ops to measure the accuracy of the model.""" - _, labels = inputs - top_1_op = tf.reduce_sum( - tf.cast(tf.nn.in_top_k(logits, labels, 1), self.data_type)) - top_5_op = tf.reduce_sum( - tf.cast(tf.nn.in_top_k(logits, labels, 5), self.data_type)) - return {'top_1_accuracy': top_1_op, 'top_5_accuracy': top_5_op} diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/model_config.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/model_config.py deleted file mode 100644 index 962bb164e4a9178d952b9bad93eb65f4f12ae3e5..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/model_config.py +++ /dev/null @@ -1,181 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Model configurations for CNN benchmarks. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from functools import partial - -from models import alexnet_model -from models import densenet_model -from models import googlenet_model -from models import inception_model -from models import lenet_model -from models import official_resnet_model -from models import overfeat_model -from models import resnet_model -from models import trivial_model -from models import vgg_model -from models.experimental import deepspeech -from models.experimental import official_ncf_model - - -_model_name_to_imagenet_model = { - 'vgg11': vgg_model.Vgg11Model, - 'vgg16': vgg_model.Vgg16Model, - 'vgg19': vgg_model.Vgg19Model, - 'lenet': lenet_model.Lenet5Model, - 'googlenet': googlenet_model.GooglenetModel, - 'overfeat': overfeat_model.OverfeatModel, - 'alexnet': alexnet_model.AlexnetModel, - 'trivial': trivial_model.TrivialModel, - 'inception3': inception_model.Inceptionv3Model, - 'inception4': inception_model.Inceptionv4Model, - 'official_resnet18_v2': - partial(official_resnet_model.ImagenetResnetModel, 18), - 'official_resnet34_v2': - partial(official_resnet_model.ImagenetResnetModel, 34), - 'official_resnet50_v2': - partial(official_resnet_model.ImagenetResnetModel, 50), - 'official_resnet101_v2': - partial(official_resnet_model.ImagenetResnetModel, 101), - 'official_resnet152_v2': - partial(official_resnet_model.ImagenetResnetModel, 152), - 'official_resnet200_v2': - partial(official_resnet_model.ImagenetResnetModel, 200), - 'official_resnet18': - partial(official_resnet_model.ImagenetResnetModel, 18, version=1), - 'official_resnet34': - partial(official_resnet_model.ImagenetResnetModel, 34, version=1), - 'official_resnet50': - partial(official_resnet_model.ImagenetResnetModel, 50, version=1), - 'official_resnet101': - partial(official_resnet_model.ImagenetResnetModel, 101, version=1), - 'official_resnet152': - partial(official_resnet_model.ImagenetResnetModel, 152, version=1), - 'official_resnet200': - partial(official_resnet_model.ImagenetResnetModel, 200, version=1), - 'resnet50': resnet_model.create_resnet50_model, - 'resnet50_v1.5': resnet_model.create_resnet50_v1_5_model, - 'resnet50_v2': resnet_model.create_resnet50_v2_model, - 'resnet101': resnet_model.create_resnet101_model, - 'resnet101_v2': resnet_model.create_resnet101_v2_model, - 'resnet152': resnet_model.create_resnet152_model, - 'resnet152_v2': resnet_model.create_resnet152_v2_model, - 'ncf': official_ncf_model.NcfModel, -} - - -_model_name_to_cifar_model = { - 'alexnet': alexnet_model.AlexnetCifar10Model, - 'resnet20': resnet_model.create_resnet20_cifar_model, - 'resnet20_v2': resnet_model.create_resnet20_v2_cifar_model, - 'resnet32': resnet_model.create_resnet32_cifar_model, - 'resnet32_v2': resnet_model.create_resnet32_v2_cifar_model, - 'resnet44': resnet_model.create_resnet44_cifar_model, - 'resnet44_v2': resnet_model.create_resnet44_v2_cifar_model, - 'resnet56': resnet_model.create_resnet56_cifar_model, - 'resnet56_v2': resnet_model.create_resnet56_v2_cifar_model, - 'resnet110': resnet_model.create_resnet110_cifar_model, - 'resnet110_v2': resnet_model.create_resnet110_v2_cifar_model, - 'trivial': trivial_model.TrivialCifar10Model, - 'densenet40_k12': densenet_model.create_densenet40_k12_model, - 'densenet100_k12': densenet_model.create_densenet100_k12_model, - 'densenet100_k24': densenet_model.create_densenet100_k24_model, -} - - -_model_name_to_object_detection_model = { - 'trivial': trivial_model.TrivialSSD300Model, -} - - -def _get_model_map(dataset_name): - """Get name to model map for specified dataset.""" - if dataset_name == 'cifar10': - return _model_name_to_cifar_model - elif dataset_name in ('imagenet', 'synthetic'): - return _model_name_to_imagenet_model - elif dataset_name == 'librispeech': - return {'deepspeech2': deepspeech.DeepSpeech2Model} - elif dataset_name == 'coco': - return _model_name_to_object_detection_model - else: - raise ValueError('Invalid dataset name: %s' % dataset_name) - - -# A model map dict can have this string as a value when TF2 is used, to indicate -# the model is only available in TF1. -_TF1_ONLY_STRING = 'TF1_ONLY' - - -def get_model_config(model_name, dataset, params): - """Map model name to model network configuration.""" - model_map = _get_model_map(dataset.name) - if model_name not in model_map: - raise ValueError('Invalid model name \'%s\' for dataset \'%s\'' % - (model_name, dataset.name)) - model = model_map[model_name](params=params) - if model == 'TF1_ONLY': - raise ValueError('Model \'%s\' can only be used with TensorFlow 1' - % (model_name,)) - return model - - -def register_model(model_name, dataset_name, model_func): - """Register a new model that can be obtained with `get_model_config`.""" - model_map = _get_model_map(dataset_name) - if model_name in model_map: - raise ValueError('Model "%s" is already registered for dataset "%s"' % - (model_name, dataset_name)) - model_map[model_name] = model_func - - -# pylint: disable=g-import-not-at-top -try: - from tensorflow.contrib import slim # pylint: disable=unused-import - can_import_contrib = True -except ImportError: - can_import_contrib = False - - -def register_tf1_models(): - """Registers all the TensorFlow 1-only models. - - TF 1-only models use contrib, which was removed in TF 2. If contrib can be - imported, the TF 1-only models are registered normally. If contrib cannot be - imported, the models are registered with the 'TF1_ONLY' string instead, which - will cause an error to be thrown if these models are used. - """ - if can_import_contrib: - from models.tf1_only import mobilenet_v2 - from models.tf1_only import nasnet_model - from models.tf1_only import ssd_model - register_model('mobilenet', 'imagenet', mobilenet_v2.MobilenetModel) - register_model('nasnet', 'imagenet', nasnet_model.NasnetModel) - register_model('nasnetlarge', 'imagenet', nasnet_model.NasnetLargeModel) - register_model('nasnet', 'cifar10', nasnet_model.NasnetCifarModel) - register_model('ssd300', 'coco', ssd_model.SSD300Model) - else: - register_model('mobilenet', 'imagenet', 'TF1_ONLY') - register_model('nasnet', 'imagenet', 'TF1_ONLY') - register_model('nasnetlarge', 'imagenet', 'TF1_ONLY') - register_model('nasnet', 'cifar10', 'TF1_ONLY') - register_model('ssd300', 'coco', 'TF1_ONLY') - diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/official_resnet_model.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/official_resnet_model.py deleted file mode 100644 index 997ee03e14d91e98b526ba665b55c46c39f4ef64..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/official_resnet_model.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Import official resnet models.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow.compat.v1 as tf -import datasets -from models import model as model_lib - - -class ImagenetResnetModel(model_lib.CNNModel): - """Official resnet models.""" - - def __init__(self, resnet_size, version=2, params=None): - """These are the parameters that work for Imagenet data. - - Args: - resnet_size: The number of convolutional layers needed in the model. - version: 1 or 2 for v1 or v2, respectively. - params: params passed by BenchmarkCNN. - """ - default_batch_sizes = { - 50: 128, - 101: 32, - 152: 32 - } - batch_size = default_batch_sizes.get(resnet_size, 32) - default_learning_rate = 0.0125 * batch_size / 32 - model_name = 'official_resnet_{}_v{}'.format(resnet_size, version) - super(ImagenetResnetModel, self).__init__( - model_name, 224, batch_size, default_learning_rate, params=params) - self.resnet_size = resnet_size - self.version = version - - def get_learning_rate(self, global_step, batch_size): - num_batches_per_epoch = ( - float(datasets.IMAGENET_NUM_TRAIN_IMAGES) / batch_size) - boundaries = [int(num_batches_per_epoch * x) for x in [30, 60, 80, 90]] - values = [1, 0.1, 0.01, 0.001, 0.0001] - adjusted_learning_rate = ( - self.learning_rate / self.default_batch_size * batch_size) - values = [v * adjusted_learning_rate for v in values] - return tf.train.piecewise_constant(global_step, boundaries, values) - - def build_network(self, images, phase_train=True, nclass=1001, - data_type=tf.float32): - # pylint: disable=g-import-not-at-top - try: - from official.r1.resnet.imagenet_main import ImagenetModel - except ImportError: - tf.logging.fatal('Please include tensorflow/models to the PYTHONPATH.') - raise - images = tf.cast(images, data_type) - model_class = ImagenetModel(resnet_size=self.resnet_size, - resnet_version=self.version, - # The official model dtype seems to be ignored, - # as the dtype it uses is the dtype of the input - # images. Doesn't hurt to set it though. - dtype=data_type) - logits = model_class(images, phase_train) - logits = tf.cast(logits, tf.float32) - return model_lib.BuildNetworkResult(logits=logits, extra_info=None) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/overfeat_model.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/overfeat_model.py deleted file mode 100644 index 7483bcbf3221f719e31baad4b9c93a4f52b0f629..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/overfeat_model.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Overfeat model configuration. - -References: - OverFeat: Integrated Recognition, Localization and Detection using - Convolutional Networks - Pierre Sermanet, David Eigen, Xiang Zhang, Michael Mathieu, Rob Fergus, - Yann LeCun, 2014 - http://arxiv.org/abs/1312.6229 -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from models import model - - -class OverfeatModel(model.CNNModel): - """OverfeatModel.""" - - def __init__(self, params=None): - super(OverfeatModel, self).__init__( - 'overfeat', 231, 32, 0.005, params=params) - - def add_inference(self, cnn): - # Note: VALID requires padding the images by 3 in width and height - cnn.conv(96, 11, 11, 4, 4, mode='VALID') - cnn.mpool(2, 2) - cnn.conv(256, 5, 5, 1, 1, mode='VALID') - cnn.mpool(2, 2) - cnn.conv(512, 3, 3) - cnn.conv(1024, 3, 3) - cnn.conv(1024, 3, 3) - cnn.mpool(2, 2) - cnn.reshape([-1, 1024 * 6 * 6]) - cnn.affine(3072) - cnn.dropout() - cnn.affine(4096) - cnn.dropout() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/resnet_model.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/resnet_model.py deleted file mode 100644 index 3b770f7a2cd24bdfecc399ce0414a16419cc1268..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/resnet_model.py +++ /dev/null @@ -1,480 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Resnet model configuration. - -References: - Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun - Deep Residual Learning for Image Recognition - arXiv:1512.03385 (2015) - - Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun - Identity Mappings in Deep Residual Networks - arXiv:1603.05027 (2016) - - Liang-Chieh Chen, George Papandreou, Iasonas Kokkinos, Kevin Murphy, - Alan L. Yuille - DeepLab: Semantic Image Segmentation with Deep Convolutional Nets, - Atrous Convolution, and Fully Connected CRFs - arXiv:1606.00915 (2016) -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy as np -from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow.compat.v1 as tf -import datasets -import mlperf -from models import model as model_lib - - -def bottleneck_block_v1(cnn, depth, depth_bottleneck, stride): - """Bottleneck block with identity short-cut for ResNet v1. - - Args: - cnn: the network to append bottleneck blocks. - depth: the number of output filters for this bottleneck block. - depth_bottleneck: the number of bottleneck filters for this block. - stride: Stride used in the first layer of the bottleneck block. - """ - input_layer = cnn.top_layer - in_size = cnn.top_size - name_key = 'resnet_v1' - name = name_key + str(cnn.counts[name_key]) - cnn.counts[name_key] += 1 - - with tf.variable_scope(name): - if depth == in_size: - if stride == 1: - shortcut = input_layer - else: - shortcut = cnn.apool( - 1, 1, stride, stride, input_layer=input_layer, - num_channels_in=in_size) - mlperf.logger.log_projection(input_tensor=input_layer, - output_tensor=shortcut) - else: - shortcut = cnn.conv( - depth, 1, 1, stride, stride, activation=None, - use_batch_norm=True, input_layer=input_layer, - num_channels_in=in_size, bias=None) - cnn.conv(depth_bottleneck, 1, 1, stride, stride, - input_layer=input_layer, num_channels_in=in_size, - use_batch_norm=True, bias=None) - cnn.conv(depth_bottleneck, 3, 3, 1, 1, mode='SAME_RESNET', - use_batch_norm=True, bias=None) - res = cnn.conv(depth, 1, 1, 1, 1, activation=None, - use_batch_norm=True, bias=None) - mlperf.logger.log(key=mlperf.tags.MODEL_HP_SHORTCUT_ADD) - mlperf.logger.log(key=mlperf.tags.MODEL_HP_RELU) - output = tf.nn.relu(shortcut + res) - cnn.top_layer = output - cnn.top_size = depth - - -def bottleneck_block_v1_5(cnn, depth, depth_bottleneck, stride): - """Bottleneck block with identity short-cut for ResNet v1.5. - - ResNet v1.5 is the informal name for ResNet v1 where stride 2 is used in the - first 3x3 convolution of each block instead of the first 1x1 convolution. - - First seen at https://github.com/facebook/fb.resnet.torch. Used in the paper - "Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour" - (arXiv:1706.02677v2) and by fast.ai to train to accuracy in 45 epochs using - multiple image sizes. - - Args: - cnn: the network to append bottleneck blocks. - depth: the number of output filters for this bottleneck block. - depth_bottleneck: the number of bottleneck filters for this block. - stride: Stride used in the first layer of the bottleneck block. - """ - input_layer = cnn.top_layer - in_size = cnn.top_size - name_key = 'resnet_v1.5' - name = name_key + str(cnn.counts[name_key]) - cnn.counts[name_key] += 1 - - with tf.variable_scope(name): - if depth == in_size: - if stride == 1: - shortcut = input_layer - else: - shortcut = cnn.apool( - 1, 1, stride, stride, input_layer=input_layer, - num_channels_in=in_size) - mlperf.logger.log_projection(input_tensor=input_layer, - output_tensor=shortcut) - else: - shortcut = cnn.conv( - depth, 1, 1, stride, stride, activation=None, - use_batch_norm=True, input_layer=input_layer, - num_channels_in=in_size, bias=None) - mlperf.logger.log_projection(input_tensor=input_layer, - output_tensor=shortcut) - cnn.conv(depth_bottleneck, 1, 1, 1, 1, - input_layer=input_layer, num_channels_in=in_size, - use_batch_norm=True, bias=None) - cnn.conv(depth_bottleneck, 3, 3, stride, stride, mode='SAME_RESNET', - use_batch_norm=True, bias=None) - res = cnn.conv(depth, 1, 1, 1, 1, activation=None, - use_batch_norm=True, bias=None) - mlperf.logger.log(key=mlperf.tags.MODEL_HP_SHORTCUT_ADD) - mlperf.logger.log(key=mlperf.tags.MODEL_HP_RELU) - output = tf.nn.relu(shortcut + res) - cnn.top_layer = output - cnn.top_size = depth - - -def bottleneck_block_v2(cnn, depth, depth_bottleneck, stride): - """Bottleneck block with identity short-cut for ResNet v2. - - The main difference from v1 is that a batch norm and relu are done at the - start of the block, instead of the end. This initial batch norm and relu is - collectively called a pre-activation. - - Args: - cnn: the network to append bottleneck blocks. - depth: the number of output filters for this bottleneck block. - depth_bottleneck: the number of bottleneck filters for this block. - stride: Stride used in the first layer of the bottleneck block. - """ - input_layer = cnn.top_layer - in_size = cnn.top_size - name_key = 'resnet_v2' - name = name_key + str(cnn.counts[name_key]) - cnn.counts[name_key] += 1 - - preact = cnn.batch_norm() - mlperf.logger.log(key=mlperf.tags.MODEL_HP_RELU) - preact = tf.nn.relu(preact) - with tf.variable_scope(name): - if depth == in_size: - if stride == 1: - shortcut = input_layer - else: - shortcut = cnn.apool( - 1, 1, stride, stride, input_layer=input_layer, - num_channels_in=in_size) - mlperf.logger.log_projection(input_tensor=input_layer, - output_tensor=shortcut) - else: - shortcut = cnn.conv( - depth, 1, 1, stride, stride, activation=None, use_batch_norm=False, - input_layer=preact, num_channels_in=in_size, bias=None) - cnn.conv(depth_bottleneck, 1, 1, stride, stride, - input_layer=preact, num_channels_in=in_size, - use_batch_norm=True, bias=None) - cnn.conv(depth_bottleneck, 3, 3, 1, 1, mode='SAME_RESNET', - use_batch_norm=True, bias=None) - res = cnn.conv(depth, 1, 1, 1, 1, activation=None, - use_batch_norm=False, bias=None) - mlperf.logger.log(key=mlperf.tags.MODEL_HP_SHORTCUT_ADD) - output = shortcut + res - cnn.top_layer = output - cnn.top_size = depth - - -def bottleneck_block(cnn, depth, depth_bottleneck, stride, version): - """Bottleneck block with identity short-cut. - - Args: - cnn: the network to append bottleneck blocks. - depth: the number of output filters for this bottleneck block. - depth_bottleneck: the number of bottleneck filters for this block. - stride: Stride used in the first layer of the bottleneck block. - version: version of ResNet to build. - """ - mlperf.logger.log(key=mlperf.tags.MODEL_HP_BLOCK_TYPE, - value=mlperf.tags.BOTTLENECK_BLOCK) - mlperf.logger.log_begin_block( - input_tensor=cnn.top_layer, block_type=mlperf.tags.BOTTLENECK_BLOCK) - if version == 'v2': - bottleneck_block_v2(cnn, depth, depth_bottleneck, stride) - elif version == 'v1.5': - bottleneck_block_v1_5(cnn, depth, depth_bottleneck, stride) - else: - bottleneck_block_v1(cnn, depth, depth_bottleneck, stride) - mlperf.logger.log_end_block(output_tensor=cnn.top_layer) - - -def residual_block(cnn, depth, stride, version, projection_shortcut=False): - """Residual block with identity short-cut. - - Args: - cnn: the network to append residual blocks. - depth: the number of output filters for this residual block. - stride: Stride used in the first layer of the residual block. - version: version of ResNet to build. - projection_shortcut: indicator of using projection shortcut, even if top - size and depth are equal - """ - pre_activation = True if version == 'v2' else False - input_layer = cnn.top_layer - in_size = cnn.top_size - - if projection_shortcut: - shortcut = cnn.conv( - depth, 1, 1, stride, stride, activation=None, - use_batch_norm=True, input_layer=input_layer, - num_channels_in=in_size, bias=None) - elif in_size != depth: - # Plan A of shortcut. - shortcut = cnn.apool(1, 1, stride, stride, - input_layer=input_layer, - num_channels_in=in_size) - padding = (depth - in_size) // 2 - if cnn.channel_pos == 'channels_last': - shortcut = tf.pad( - shortcut, [[0, 0], [0, 0], [0, 0], [padding, padding]]) - else: - shortcut = tf.pad( - shortcut, [[0, 0], [padding, padding], [0, 0], [0, 0]]) - else: - shortcut = input_layer - if pre_activation: - res = cnn.batch_norm(input_layer) - res = tf.nn.relu(res) - else: - res = input_layer - cnn.conv(depth, 3, 3, stride, stride, - input_layer=res, num_channels_in=in_size, - use_batch_norm=True, bias=None) - if pre_activation: - res = cnn.conv(depth, 3, 3, 1, 1, activation=None, - use_batch_norm=False, bias=None) - output = shortcut + res - else: - res = cnn.conv(depth, 3, 3, 1, 1, activation=None, - use_batch_norm=True, bias=None) - output = tf.nn.relu(shortcut + res) - cnn.top_layer = output - cnn.top_size = depth - - -class ResnetModel(model_lib.CNNModel): - """Resnet cnn network configuration.""" - - def __init__(self, model, layer_counts, params=None): - default_batch_sizes = { - 'resnet50': 64, - 'resnet101': 32, - 'resnet152': 32, - 'resnet50_v1.5': 64, - 'resnet101_v1.5': 32, - 'resnet152_v1.5': 32, - 'resnet50_v2': 64, - 'resnet101_v2': 32, - 'resnet152_v2': 32, - } - batch_size = default_batch_sizes.get(model, 32) - # The ResNet paper uses a starting lr of .1 at bs=256. - self.base_lr_batch_size = 256 - base_lr = 0.128 - if params and params.resnet_base_lr: - base_lr = params.resnet_base_lr - - super(ResnetModel, self).__init__(model, 224, batch_size, base_lr, - layer_counts, params=params) - if 'v2' in model: - self.version = 'v2' - elif 'v1.5' in model: - self.version = 'v1.5' - else: - self.version = 'v1' - - def add_inference(self, cnn): - if self.layer_counts is None: - raise ValueError('Layer counts not specified for %s' % self.get_model()) - # Drop batch size from shape logging. - mlperf.logger.log(key=mlperf.tags.MODEL_HP_INITIAL_SHAPE, - value=cnn.top_layer.shape.as_list()[1:]) - cnn.use_batch_norm = True - cnn.batch_norm_config = {'decay': 0.9, 'epsilon': 1e-5, 'scale': True} - cnn.conv(64, 7, 7, 2, 2, mode='SAME_RESNET', use_batch_norm=True) - cnn.mpool(3, 3, 2, 2, mode='SAME') - for _ in xrange(self.layer_counts[0]): - bottleneck_block(cnn, 256, 64, 1, self.version) - for i in xrange(self.layer_counts[1]): - stride = 2 if i == 0 else 1 - bottleneck_block(cnn, 512, 128, stride, self.version) - for i in xrange(self.layer_counts[2]): - stride = 2 if i == 0 else 1 - bottleneck_block(cnn, 1024, 256, stride, self.version) - for i in xrange(self.layer_counts[3]): - stride = 2 if i == 0 else 1 - bottleneck_block(cnn, 2048, 512, stride, self.version) - if self.version == 'v2': - cnn.batch_norm() - cnn.top_layer = tf.nn.relu(cnn.top_layer) - cnn.spatial_mean() - - def get_learning_rate(self, global_step, batch_size): - rescaled_lr = self.get_scaled_base_learning_rate(batch_size) - num_batches_per_epoch = ( - datasets.IMAGENET_NUM_TRAIN_IMAGES / batch_size) - boundaries = [int(num_batches_per_epoch * x) for x in [30, 60, 80, 90]] - values = [1, 0.1, 0.01, 0.001, 0.0001] - values = [rescaled_lr * v for v in values] - lr = tf.train.piecewise_constant(global_step, boundaries, values) - warmup_steps = int(num_batches_per_epoch * 5) - mlperf.logger.log(key=mlperf.tags.OPT_LR_WARMUP_STEPS, value=warmup_steps) - warmup_lr = ( - rescaled_lr * tf.cast(global_step, tf.float32) / tf.cast( - warmup_steps, tf.float32)) - return tf.cond(global_step < warmup_steps, lambda: warmup_lr, lambda: lr) - - def get_scaled_base_learning_rate(self, batch_size): - """Calculates base learning rate for creating lr schedule. - - In replicated mode, gradients are summed rather than averaged which, with - the sgd and momentum optimizers, increases the effective learning rate by - lr * num_gpus. Dividing the base lr by num_gpus negates the increase. - - Args: - batch_size: Total batch-size. - - Returns: - Base learning rate to use to create lr schedule. - """ - base_lr = self.learning_rate - if self.params.variable_update == 'replicated': - base_lr = self.learning_rate / self.params.num_gpus - scaled_lr = base_lr * (batch_size / self.base_lr_batch_size) - return scaled_lr - - -def create_resnet50_model(params): - return ResnetModel('resnet50', (3, 4, 6, 3), params=params) - - -def create_resnet50_v1_5_model(params): - return ResnetModel('resnet50_v1.5', (3, 4, 6, 3), params=params) - - -def create_resnet50_v2_model(params): - return ResnetModel('resnet50_v2', (3, 4, 6, 3), params=params) - - -def create_resnet101_model(params): - return ResnetModel('resnet101', (3, 4, 23, 3), params=params) - - -def create_resnet101_v2_model(params): - return ResnetModel('resnet101_v2', (3, 4, 23, 3), params=params) - - -def create_resnet152_model(params): - return ResnetModel('resnet152', (3, 8, 36, 3), params=params) - - -def create_resnet152_v2_model(params): - return ResnetModel('resnet152_v2', (3, 8, 36, 3), params=params) - - -class ResnetCifar10Model(model_lib.CNNModel): - """Resnet cnn network configuration for Cifar 10 dataset. - - V1 model architecture follows the one defined in the paper: - https://arxiv.org/pdf/1512.03385.pdf. - - V2 model architecture follows the one defined in the paper: - https://arxiv.org/pdf/1603.05027.pdf. - """ - - def __init__(self, model, layer_counts, params=None): - if 'v2' in model: - self.version = 'v2' - else: - self.version = 'v1' - super(ResnetCifar10Model, self).__init__( - model, 32, 128, 0.1, layer_counts, params=params) - - def add_inference(self, cnn): - if self.layer_counts is None: - raise ValueError('Layer counts not specified for %s' % self.get_model()) - - cnn.use_batch_norm = True - cnn.batch_norm_config = {'decay': 0.9, 'epsilon': 1e-5, 'scale': True} - if self.version == 'v2': - cnn.conv(16, 3, 3, 1, 1, use_batch_norm=True) - else: - cnn.conv(16, 3, 3, 1, 1, activation=None, use_batch_norm=True) - for i in xrange(self.layer_counts[0]): - # reshape to batch_size x 16 x 32 x 32 - residual_block(cnn, 16, 1, self.version) - for i in xrange(self.layer_counts[1]): - # Subsampling is performed at the first convolution with a stride of 2 - stride = 2 if i == 0 else 1 - # reshape to batch_size x 32 x 16 x 16 - residual_block(cnn, 32, stride, self.version) - for i in xrange(self.layer_counts[2]): - stride = 2 if i == 0 else 1 - # reshape to batch_size x 64 x 8 x 8 - residual_block(cnn, 64, stride, self.version) - if self.version == 'v2': - cnn.batch_norm() - cnn.top_layer = tf.nn.relu(cnn.top_layer) - cnn.spatial_mean() - - def get_learning_rate(self, global_step, batch_size): - num_batches_per_epoch = int(50000 / batch_size) - boundaries = num_batches_per_epoch * np.array([82, 123, 300], - dtype=np.int64) - boundaries = [x for x in boundaries] - values = [0.1, 0.01, 0.001, 0.0002] - return tf.train.piecewise_constant(global_step, boundaries, values) - - -def create_resnet20_cifar_model(params): - return ResnetCifar10Model('resnet20', (3, 3, 3), params=params) - - -def create_resnet20_v2_cifar_model(params): - return ResnetCifar10Model('resnet20_v2', (3, 3, 3), params=params) - - -def create_resnet32_cifar_model(params): - return ResnetCifar10Model('resnet32', (5, 5, 5), params=params) - - -def create_resnet32_v2_cifar_model(params): - return ResnetCifar10Model('resnet32_v2', (5, 5, 5), params=params) - - -def create_resnet44_cifar_model(params): - return ResnetCifar10Model('resnet44', (7, 7, 7), params=params) - - -def create_resnet44_v2_cifar_model(params): - return ResnetCifar10Model('resnet44_v2', (7, 7, 7), params=params) - - -def create_resnet56_cifar_model(params): - return ResnetCifar10Model('resnet56', (9, 9, 9), params=params) - - -def create_resnet56_v2_cifar_model(params): - return ResnetCifar10Model('resnet56_v2', (9, 9, 9), params=params) - - -def create_resnet110_cifar_model(params): - return ResnetCifar10Model('resnet110', (18, 18, 18), params=params) - - -def create_resnet110_v2_cifar_model(params): - return ResnetCifar10Model('resnet110_v2', (18, 18, 18), params=params) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/resnet_model_test.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/resnet_model_test.py deleted file mode 100644 index b4052fcd2e996c7f02458b6754dfa6dd52635a94..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/resnet_model_test.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for resnet_model.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import mock -import tensorflow.compat.v1 as tf - -from models import resnet_model - - -class ResNetModelTest(tf.test.TestCase): - - def testGetScaledBaseLearningRateOneGpuLrFromParams(self): - """Verifies setting params.resnet_base_lr pipes through.""" - lr = self._get_scaled_base_learning_rate(1, - 'parameter_server', - 256, - base_lr=.050) - self.assertEqual(lr, .050) - - def testGetScaledBaseLearningRateOneGpu(self): - lr = self._get_scaled_base_learning_rate(1, 'parameter_server', 128) - self.assertEqual(lr, .064) - - def testGetScaledBaseLearningRateEightGpuReplicated(self): - lr = self._get_scaled_base_learning_rate(8, 'replicated', 256 * 8) - self.assertEqual(lr, .128) - - def testGetScaledBaseLearningRateTwoGpuParameter(self): - lr = self._get_scaled_base_learning_rate(2, 'parameter_server', 256 * 2) - self.assertEqual(lr, .256) - - def testGetScaledBaseLearningRateTwoGpuUneven(self): - lr = self._get_scaled_base_learning_rate(2, 'replicated', 13) - self.assertEqual(lr, 0.0032500000000000003) - - def _get_scaled_base_learning_rate(self, - num_gpus, - variable_update, - batch_size, - base_lr=None): - """Simplifies testing different learning rate calculations. - - Args: - num_gpus: Number of GPUs to be used. - variable_update: Type of variable update used. - batch_size: Total batch size. - base_lr: Base learning rate before scaling. - - Returns: - Base learning rate that would be used to create lr schedule. - """ - params = mock.Mock() - params.num_gpus = num_gpus - params.variable_update = variable_update - if base_lr: - params.resnet_base_lr = base_lr - resnet50_model = resnet_model.ResnetModel('resnet50', 50, params=params) - return resnet50_model.get_scaled_base_learning_rate(batch_size) - - -if __name__ == '__main__': - tf.disable_v2_behavior() - tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/mobilenet.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/mobilenet.py deleted file mode 100644 index c56393c2127865e758bd83121c8b38f26f79c620..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/mobilenet.py +++ /dev/null @@ -1,467 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Mobilenet Base Class, branched from slim for fp16 performance study.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -import collections -import contextlib -import copy -import os - -import tensorflow.compat.v1 as tf -from tensorflow.contrib import slim as contrib_slim - -slim = contrib_slim - - -@slim.add_arg_scope -def apply_activation(x, name=None, activation_fn=None): - return activation_fn(x, name=name) if activation_fn else x - - -def _fixed_padding(inputs, kernel_size, rate=1): - """Pads the input along the spatial dimensions independently of input size. - - Pads the input such that if it was used in a convolution with 'VALID' padding, - the output would have the same dimensions as if the unpadded input was used - in a convolution with 'SAME' padding. - - Args: - inputs: A tensor of size [batch, height_in, width_in, channels]. - kernel_size: The kernel to be used in the conv2d or max_pool2d operation. - rate: An integer, rate for atrous convolution. - - Returns: - output: A tensor of size [batch, height_out, width_out, channels] with the - input, either intact (if kernel_size == 1) or padded (if kernel_size > 1). - """ - kernel_size_effective = [kernel_size[0] + (kernel_size[0] - 1) * (rate - 1), - kernel_size[0] + (kernel_size[0] - 1) * (rate - 1)] - pad_total = [kernel_size_effective[0] - 1, kernel_size_effective[1] - 1] - pad_beg = [pad_total[0] // 2, pad_total[1] // 2] - pad_end = [pad_total[0] - pad_beg[0], pad_total[1] - pad_beg[1]] - padded_inputs = tf.pad(inputs, [[0, 0], [pad_beg[0], pad_end[0]], - [pad_beg[1], pad_end[1]], [0, 0]]) - return padded_inputs - - -def _make_divisible(v, divisor, min_value=None): - if min_value is None: - min_value = divisor - new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) - # Make sure that round down does not go down by more than 10%. - if new_v < 0.9 * v: - new_v += divisor - return new_v - - -@contextlib.contextmanager -def _set_arg_scope_defaults(defaults): - """Sets arg scope defaults for all items present in defaults. - - Args: - defaults: dictionary/list of pairs, containing a mapping from - function to a dictionary of default args. - - Yields: - context manager where all defaults are set. - """ - if hasattr(defaults, 'items'): - items = list(defaults.items()) - else: - items = defaults - if not items: - yield - else: - func, default_arg = items[0] - with slim.arg_scope(func, **default_arg): - with _set_arg_scope_defaults(items[1:]): - yield - - -@slim.add_arg_scope -def depth_multiplier(output_params, - multiplier, - divisible_by=8, - min_depth=8, - **unused_kwargs): - if 'num_outputs' not in output_params: - return - d = output_params['num_outputs'] - output_params['num_outputs'] = _make_divisible(d * multiplier, divisible_by, - min_depth) - - -_Op = collections.namedtuple('Op', ['op', 'params', 'multiplier_func']) - - -def op(opfunc, **params): - multiplier = params.pop('multiplier_transorm', depth_multiplier) - return _Op(opfunc, params=params, multiplier_func=multiplier) - - -class NoOpScope(object): - """No-op context manager.""" - - def __enter__(self): - return - - def __exit__(self, exc_type, exc_value, traceback): - return False - - -def safe_arg_scope(funcs, **kwargs): - """Returns `slim.arg_scope` with all None arguments removed. - - Args: - funcs: Functions to pass to `arg_scope`. - **kwargs: Arguments to pass to `arg_scope`. - - Returns: - arg_scope or No-op context manager. - - Note: can be useful if None value should be interpreted as "do not overwrite - this parameter value". - """ - filtered_args = {name: value for name, value in kwargs.items() - if value is not None} - if filtered_args: - return slim.arg_scope(funcs, **filtered_args) - else: - return NoOpScope() - - -@slim.add_arg_scope -def mobilenet_base( # pylint: disable=invalid-name - inputs, - conv_defs, - multiplier=1.0, - final_endpoint=None, - output_stride=None, - use_explicit_padding=False, - scope=None, - is_training=False): - """Mobilenet base network. - - Constructs a network from inputs to the given final endpoint. By default - the network is constructed in inference mode. To create network - in training mode use: - - with slim.arg_scope(mobilenet.training_scope()): - logits, endpoints = mobilenet_base(...) - - Args: - inputs: a tensor of shape [batch_size, height, width, channels]. - conv_defs: A list of op(...) layers specifying the net architecture. - multiplier: Float multiplier for the depth (number of channels) - for all convolution ops. The value must be greater than zero. Typical - usage will be to set this value in (0, 1) to reduce the number of - parameters or computation cost of the model. - final_endpoint: The name of last layer, for early termination for - for V1-based networks: last layer is "layer_14", for V2: "layer_20" - output_stride: An integer that specifies the requested ratio of input to - output spatial resolution. If not None, then we invoke atrous convolution - if necessary to prevent the network from reducing the spatial resolution - of the activation maps. Allowed values are 1 or any even number, excluding - zero. Typical values are 8 (accurate fully convolutional mode), 16 - (fast fully convolutional mode), and 32 (classification mode). - - NOTE- output_stride relies on all consequent operators to support dilated - operators via "rate" parameter. This might require wrapping non-conv - operators to operate properly. - - use_explicit_padding: Use 'VALID' padding for convolutions, but prepad - inputs so that the output dimensions are the same as if 'SAME' padding - were used. - scope: optional variable scope. - is_training: How to setup batch_norm and other ops. Note: most of the time - this does not need be set directly. Use mobilenet.training_scope() to set - up training instead. This parameter is here for backward compatibility - only. It is safe to set it to the value matching - training_scope(is_training=...). It is also safe to explicitly set - it to False, even if there is outer training_scope set to to training. - (The network will be built in inference mode). If this is set to None, - no arg_scope is added for slim.batch_norm's is_training parameter. - - Returns: - tensor_out: output tensor. - end_points: a set of activations for external use, for example summaries or - losses. - - Raises: - ValueError: depth_multiplier <= 0, or the target output_stride is not - allowed. - """ - if multiplier <= 0: - raise ValueError('multiplier is not greater than zero.') - - # Set conv defs defaults and overrides. - conv_defs_defaults = conv_defs.get('defaults', {}) - conv_defs_overrides = conv_defs.get('overrides', {}) - if use_explicit_padding: - conv_defs_overrides = copy.deepcopy(conv_defs_overrides) - conv_defs_overrides[ - (slim.conv2d, slim.separable_conv2d)] = {'padding': 'VALID'} - - if output_stride is not None: - if output_stride == 0 or (output_stride > 1 and output_stride % 2): - raise ValueError('Output stride must be None, 1 or a multiple of 2.') - - # a) Set the tensorflow scope - # b) set padding to default: note we might consider removing this - # since it is also set by mobilenet_scope - # c) set all defaults - # d) set all extra overrides. - with _scope_all(scope, default_scope='Mobilenet'), \ - safe_arg_scope([slim.batch_norm], is_training=is_training), \ - _set_arg_scope_defaults(conv_defs_defaults), \ - _set_arg_scope_defaults(conv_defs_overrides): - # The current_stride variable keeps track of the output stride of the - # activations, i.e., the running product of convolution strides up to the - # current network layer. This allows us to invoke atrous convolution - # whenever applying the next convolution would result in the activations - # having output stride larger than the target output_stride. - current_stride = 1 - - # The atrous convolution rate parameter. - rate = 1 - - net = inputs - # Insert default parameters before the base scope which includes - # any custom overrides set in mobilenet. - end_points = {} - scopes = {} - for i, opdef in enumerate(conv_defs['spec']): - params = dict(opdef.params) - opdef.multiplier_func(params, multiplier) - stride = params.get('stride', 1) - if output_stride is not None and current_stride == output_stride: - # If we have reached the target output_stride, then we need to employ - # atrous convolution with stride=1 and multiply the atrous rate by the - # current unit's stride for use in subsequent layers. - layer_stride = 1 - layer_rate = rate - rate *= stride - else: - layer_stride = stride - layer_rate = 1 - current_stride *= stride - # Update params. - params['stride'] = layer_stride - # Only insert rate to params if rate > 1. - if layer_rate > 1: - params['rate'] = layer_rate - # Set padding - if use_explicit_padding: - if 'kernel_size' in params: - net = _fixed_padding(net, params['kernel_size'], layer_rate) - else: - params['use_explicit_padding'] = True - - end_point = 'layer_%d' % (i + 1) - try: - net = opdef.op(net, **params) - except Exception: - print('Failed to create op %i: %r params: %r' % (i, opdef, params)) - raise - end_points[end_point] = net - scope = os.path.dirname(net.name) - scopes[scope] = end_point - if final_endpoint is not None and end_point == final_endpoint: - break - - # Add all tensors that end with 'output' to - # endpoints - for t in net.graph.get_operations(): - scope = os.path.dirname(t.name) - bn = os.path.basename(t.name) - if scope in scopes and t.name.endswith('output'): - end_points[scopes[scope] + '/' + bn] = t.outputs[0] - return net, end_points - - -@contextlib.contextmanager -def _scope_all(scope, default_scope=None): - with tf.variable_scope(scope, default_name=default_scope) as s,\ - tf.name_scope(s.original_name_scope): - yield s - - -@slim.add_arg_scope -def mobilenet(inputs, - num_classes=1001, - prediction_fn=slim.softmax, - reuse=None, - scope='Mobilenet', - base_only=False, - **mobilenet_args): - """Mobilenet model for classification, supports both V1 and V2. - - Note: default mode is inference, use mobilenet.training_scope to create - training network. - - - Args: - inputs: a tensor of shape [batch_size, height, width, channels]. - num_classes: number of predicted classes. If 0 or None, the logits layer - is omitted and the input features to the logits layer (before dropout) - are returned instead. - prediction_fn: a function to get predictions out of logits - (default softmax). - reuse: whether or not the network and its variables should be reused. To be - able to reuse 'scope' must be given. - scope: Optional variable_scope. - base_only: if True will only create the base of the network (no pooling - and no logits). - **mobilenet_args: passed to mobilenet_base verbatim. - - conv_defs: list of conv defs - - multiplier: Float multiplier for the depth (number of channels) - for all convolution ops. The value must be greater than zero. Typical - usage will be to set this value in (0, 1) to reduce the number of - parameters or computation cost of the model. - - output_stride: will ensure that the last layer has at most total stride. - If the architecture calls for more stride than that provided - (e.g. output_stride=16, but the architecture has 5 stride=2 operators), - it will replace output_stride with fractional convolutions using Atrous - Convolutions. - - Returns: - logits: the pre-softmax activations, a tensor of size - [batch_size, num_classes] - end_points: a dictionary from components of the network to the corresponding - activation tensor. - - Raises: - ValueError: Input rank is invalid. - """ - is_training = mobilenet_args.get('is_training', False) - input_shape = inputs.get_shape().as_list() - if len(input_shape) != 4: - raise ValueError('Expected rank 4 input, was: %d' % len(input_shape)) - - with tf.variable_scope(scope, 'Mobilenet', reuse=reuse) as scope: - inputs = tf.identity(inputs, 'input') - net, end_points = mobilenet_base(inputs, scope=scope, **mobilenet_args) - if base_only: - return net, end_points - - net = tf.identity(net, name='embedding') - - with tf.variable_scope('Logits'): - net = global_pool(net) - end_points['global_pool'] = net - if not num_classes: - return net, end_points - net = slim.dropout(net, scope='Dropout', is_training=is_training) - # 1 x 1 x num_classes - # Note: legacy scope name. - logits = slim.conv2d( - net, - num_classes, [1, 1], - activation_fn=None, - normalizer_fn=None, - biases_initializer=tf.zeros_initializer(), - scope='Conv2d_1c_1x1') - - logits = tf.squeeze(logits, [1, 2]) - - logits = tf.identity(logits, name='output') - end_points['Logits'] = logits - if prediction_fn: - end_points['Predictions'] = prediction_fn(logits, 'Predictions') - return logits, end_points - - -def global_pool(input_tensor, pool_op=tf.nn.avg_pool): - """Applies avg pool to produce 1x1 output. - - NOTE: This function is funcitonally equivalenet to reduce_mean, but it has - baked in average pool which has better support across hardware. - - Args: - input_tensor: input tensor - pool_op: pooling op (avg pool is default) - Returns: - a tensor batch_size x 1 x 1 x depth. - """ - shape = input_tensor.get_shape().as_list() - if shape[1] is None or shape[2] is None: - kernel_size = tf.convert_to_tensor( - [1, tf.shape(input_tensor)[1], - tf.shape(input_tensor)[2], 1]) - else: - kernel_size = [1, shape[1], shape[2], 1] - output = pool_op( - input_tensor, ksize=kernel_size, strides=[1, 1, 1, 1], padding='VALID') - # Recover output shape, for unknown shape. - output.set_shape([None, 1, 1, None]) - return output - - -def training_scope(is_training=True, - weight_decay=0.00004, - stddev=0.09, - dropout_keep_prob=0.8, - bn_decay=0.997): - """Defines Mobilenet training scope. - - Usage: - with tf.contrib.slim.arg_scope(mobilenet.training_scope()): - logits, endpoints = mobilenet_v2.mobilenet(input_tensor) - - # the network created will be trainble with dropout/batch norm - # initialized appropriately. - Args: - is_training: if set to False this will ensure that all customizations are - set to non-training mode. This might be helpful for code that is reused - across both training/evaluation, but most of the time training_scope with - value False is not needed. If this is set to None, the parameters is not - added to the batch_norm arg_scope. - - weight_decay: The weight decay to use for regularizing the model. - stddev: Standard deviation for initialization, if negative uses xavier. - dropout_keep_prob: dropout keep probability (not set if equals to None). - bn_decay: decay for the batch norm moving averages (not set if equals to - None). - - Returns: - An argument scope to use via arg_scope. - """ - # Note: do not introduce parameters that would change the inference - # model here (for example whether to use bias), modify conv_def instead. - batch_norm_params = { - 'decay': bn_decay, - 'is_training': is_training - } - if stddev < 0: - weight_intitializer = slim.initializers.xavier_initializer() - else: - weight_intitializer = tf.truncated_normal_initializer(stddev=stddev) - - # Set weight_decay for weights in Conv and FC layers. - with slim.arg_scope( - [slim.conv2d, slim.fully_connected, slim.separable_conv2d], - weights_initializer=weight_intitializer, - normalizer_fn=slim.batch_norm), \ - slim.arg_scope([mobilenet_base, mobilenet], is_training=is_training),\ - safe_arg_scope([slim.batch_norm], **batch_norm_params), \ - safe_arg_scope([slim.dropout], is_training=is_training, - keep_prob=dropout_keep_prob), \ - slim.arg_scope([slim.conv2d], \ - weights_regularizer=slim.l2_regularizer(weight_decay)), \ - slim.arg_scope([slim.separable_conv2d], weights_regularizer=None) as s: - return s diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/mobilenet_conv_blocks.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/mobilenet_conv_blocks.py deleted file mode 100644 index 34016b277b6cc90700984a44247fb971ce708277..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/mobilenet_conv_blocks.py +++ /dev/null @@ -1,360 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Convolution blocks for mobilenet.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import contextlib -import functools - -import tensorflow.compat.v1 as tf -from tensorflow.contrib import slim - - -def _fixed_padding(inputs, kernel_size, rate=1): - """Pads the input along the spatial dimensions independently of input size. - - Pads the input such that if it was used in a convolution with 'VALID' padding, - the output would have the same dimensions as if the unpadded input was used - in a convolution with 'SAME' padding. - - Args: - inputs: A tensor of size [batch, height_in, width_in, channels]. - kernel_size: The kernel to be used in the conv2d or max_pool2d operation. - rate: An integer, rate for atrous convolution. - - Returns: - output: A tensor of size [batch, height_out, width_out, channels] with the - input, either intact (if kernel_size == 1) or padded (if kernel_size > 1). - """ - kernel_size_effective = [kernel_size[0] + (kernel_size[0] - 1) * (rate - 1), - kernel_size[0] + (kernel_size[0] - 1) * (rate - 1)] - pad_total = [kernel_size_effective[0] - 1, kernel_size_effective[1] - 1] - pad_beg = [pad_total[0] // 2, pad_total[1] // 2] - pad_end = [pad_total[0] - pad_beg[0], pad_total[1] - pad_beg[1]] - padded_inputs = tf.pad(inputs, [[0, 0], [pad_beg[0], pad_end[0]], - [pad_beg[1], pad_end[1]], [0, 0]]) - return padded_inputs - - -def _make_divisible(v, divisor, min_value=None): - if min_value is None: - min_value = divisor - new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) - # Make sure that round down does not go down by more than 10%. - if new_v < 0.9 * v: - new_v += divisor - return new_v - - -def _split_divisible(num, num_ways, divisible_by=8): - """Evenly splits num, num_ways so each piece is a multiple of divisible_by.""" - assert num % divisible_by == 0 - assert num // num_ways >= divisible_by - # Note: want to round down, we adjust each split to match the total. - base = num // num_ways // divisible_by * divisible_by - result = [] - accumulated = 0 - for i in range(num_ways): - r = base - while accumulated + r < num * (i + 1) // num_ways: - r += divisible_by - result.append(r) - accumulated += r - assert accumulated == num - return result - - -@contextlib.contextmanager -def _v1_compatible_scope_naming(scope): # pylint: disable=g-missing-docstring - if scope is None: # Create uniqified separable blocks. - with tf.variable_scope(None, default_name='separable') as s, \ - tf.name_scope(s.original_name_scope): - yield '' - else: - # We use scope_depthwise, scope_pointwise for compatibility with V1 ckpts. - # which provide numbered scopes. - scope += '_' - yield scope - - -@slim.add_arg_scope -def split_separable_conv2d(input_tensor, - num_outputs, - scope=None, - normalizer_fn=None, - stride=1, - rate=1, - endpoints=None, - use_explicit_padding=False): - """Separable mobilenet V1 style convolution. - - Depthwise convolution, with default non-linearity, - followed by 1x1 depthwise convolution. This is similar to - slim.separable_conv2d, but differs in tha it applies batch - normalization and non-linearity to depthwise. This matches - the basic building of Mobilenet Paper - (https://arxiv.org/abs/1704.04861) - - Args: - input_tensor: input - num_outputs: number of outputs - scope: optional name of the scope. Note if provided it will use - scope_depthwise for deptwhise, and scope_pointwise for pointwise. - normalizer_fn: which normalizer function to use for depthwise/pointwise - stride: stride - rate: output rate (also known as dilation rate) - endpoints: optional, if provided, will export additional tensors to it. - use_explicit_padding: Use 'VALID' padding for convolutions, but prepad - inputs so that the output dimensions are the same as if 'SAME' padding - were used. - - Returns: - output tesnor - """ - - with _v1_compatible_scope_naming(scope) as scope: - dw_scope = scope + 'depthwise' - endpoints = endpoints if endpoints is not None else {} - kernel_size = [3, 3] - padding = 'SAME' - if use_explicit_padding: - padding = 'VALID' - input_tensor = _fixed_padding(input_tensor, kernel_size, rate) - net = slim.separable_conv2d( - input_tensor, - None, - kernel_size, - depth_multiplier=1, - stride=stride, - rate=rate, - normalizer_fn=normalizer_fn, - padding=padding, - scope=dw_scope) - - endpoints[dw_scope] = net - - pw_scope = scope + 'pointwise' - net = slim.conv2d( - net, - num_outputs, [1, 1], - stride=1, - normalizer_fn=normalizer_fn, - scope=pw_scope) - endpoints[pw_scope] = net - return net - - -def expand_input_by_factor(n, divisible_by=8): - return lambda num_inputs, **_: _make_divisible(num_inputs * n, divisible_by) - - -@slim.add_arg_scope -def expanded_conv(input_tensor, - num_outputs, - expansion_size=expand_input_by_factor(6), - stride=1, - rate=1, - kernel_size=(3, 3), - residual=True, - normalizer_fn=None, - split_projection=1, - split_expansion=1, - expansion_transform=None, - depthwise_location='expansion', - depthwise_channel_multiplier=1, - endpoints=None, - use_explicit_padding=False, - padding='SAME', - scope=None): - """Depthwise Convolution Block with expansion. - - Builds a composite convolution that has the following structure - expansion (1x1) -> depthwise (kernel_size) -> projection (1x1) - - Args: - input_tensor: input - num_outputs: number of outputs in the final layer. - expansion_size: the size of expansion, could be a constant or a callable. - If latter it will be provided 'num_inputs' as an input. For forward - compatibility it should accept arbitrary keyword arguments. - Default will expand the input by factor of 6. - stride: depthwise stride - rate: depthwise rate - kernel_size: depthwise kernel - residual: whether to include residual connection between input - and output. - normalizer_fn: batchnorm or otherwise - split_projection: how many ways to split projection operator - (that is conv expansion->bottleneck) - split_expansion: how many ways to split expansion op - (that is conv bottleneck->expansion) ops will keep depth divisible - by this value. - expansion_transform: Optional function that takes expansion - as a single input and returns output. - depthwise_location: where to put depthwise covnvolutions supported - values None, 'input', 'output', 'expansion' - depthwise_channel_multiplier: depthwise channel multiplier: - each input will replicated (with different filters) - that many times. So if input had c channels, - output will have c x depthwise_channel_multpilier. - endpoints: An optional dictionary into which intermediate endpoints are - placed. The keys "expansion_output", "depthwise_output", - "projection_output" and "expansion_transform" are always populated, even - if the corresponding functions are not invoked. - use_explicit_padding: Use 'VALID' padding for convolutions, but prepad - inputs so that the output dimensions are the same as if 'SAME' padding - were used. - padding: Padding type to use if `use_explicit_padding` is not set. - scope: optional scope. - - Returns: - Tensor of depth num_outputs - - Raises: - TypeError: on inval - """ - with tf.variable_scope(scope, default_name='expanded_conv') as s, \ - tf.name_scope(s.original_name_scope): - prev_depth = input_tensor.get_shape().as_list()[3] - if depthwise_location not in [None, 'input', 'output', 'expansion']: - raise TypeError('%r is unknown value for depthwise_location' % - depthwise_location) - if use_explicit_padding: - if padding != 'SAME': - raise TypeError('`use_explicit_padding` should only be used with ' - '"SAME" padding.') - padding = 'VALID' - depthwise_func = functools.partial( - slim.separable_conv2d, - num_outputs=None, - kernel_size=kernel_size, - depth_multiplier=depthwise_channel_multiplier, - stride=stride, - rate=rate, - normalizer_fn=normalizer_fn, - padding=padding, - scope='depthwise') - # b1 -> b2 * r -> b2 - # i -> (o * r) (bottleneck) -> o - input_tensor = tf.identity(input_tensor, 'input') - net = input_tensor - - if depthwise_location == 'input': - if use_explicit_padding: - net = _fixed_padding(net, kernel_size, rate) - net = depthwise_func(net, activation_fn=None) - - if callable(expansion_size): - inner_size = expansion_size(num_inputs=prev_depth) - else: - inner_size = expansion_size - - if inner_size > net.shape[3]: - net = split_conv( - net, - inner_size, - num_ways=split_expansion, - scope='expand', - stride=1, - normalizer_fn=normalizer_fn) - net = tf.identity(net, 'expansion_output') - if endpoints is not None: - endpoints['expansion_output'] = net - - if depthwise_location == 'expansion': - if use_explicit_padding: - net = _fixed_padding(net, kernel_size, rate) - net = depthwise_func(net) - - net = tf.identity(net, name='depthwise_output') - if endpoints is not None: - endpoints['depthwise_output'] = net - if expansion_transform: - net = expansion_transform(expansion_tensor=net, input_tensor=input_tensor) - # Note in contrast with expansion, we always have - # projection to produce the desired output size. - net = split_conv( - net, - num_outputs, - num_ways=split_projection, - stride=1, - scope='project', - normalizer_fn=normalizer_fn, - activation_fn=tf.identity) - if endpoints is not None: - endpoints['projection_output'] = net - if depthwise_location == 'output': - if use_explicit_padding: - net = _fixed_padding(net, kernel_size, rate) - net = depthwise_func(net, activation_fn=None) - - if callable(residual): # custom residual - net = residual(input_tensor=input_tensor, output_tensor=net) - elif (residual and - # stride check enforces that we don't add residuals when spatial - # dimensions are None - stride == 1 and - # Depth matches - net.get_shape().as_list()[3] == - input_tensor.get_shape().as_list()[3]): - net += input_tensor - return tf.identity(net, name='output') - - -def split_conv(input_tensor, - num_outputs, - num_ways, - scope, - divisible_by=8, - **kwargs): - """Creates a split convolution. - - Split convolution splits the input and output into - 'num_blocks' blocks of approximately the same size each, - and only connects $i$-th input to $i$ output. - - Args: - input_tensor: input tensor - num_outputs: number of output filters - num_ways: num blocks to split by. - scope: scope for all the operators. - divisible_by: make sure that every part is divisiable by this. - **kwargs: will be passed directly into conv2d operator - Returns: - tensor - """ - b = input_tensor.get_shape().as_list()[3] - - if num_ways == 1 or min(b // num_ways, - num_outputs // num_ways) < divisible_by: - # Don't do any splitting if we end up with less than 8 filters - # on either side. - return slim.conv2d(input_tensor, num_outputs, [1, 1], scope=scope, **kwargs) - - outs = [] - input_splits = _split_divisible(b, num_ways, divisible_by=divisible_by) - output_splits = _split_divisible( - num_outputs, num_ways, divisible_by=divisible_by) - inputs = tf.split(input_tensor, input_splits, axis=3, name='split_' + scope) - base = scope - for i, (input_tensor, out_size) in enumerate(zip(inputs, output_splits)): - scope = base + '_part_%d' % (i,) - n = slim.conv2d(input_tensor, out_size, [1, 1], scope=scope, **kwargs) - n = tf.identity(n, scope + '_output') - outs.append(n) - return tf.concat(outs, 3, name=scope + '_concat') diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/mobilenet_test.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/mobilenet_test.py deleted file mode 100644 index c0b7d5345077585b99a6a6b5e305388bfcc5eaf0..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/mobilenet_test.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for mobilenet_v2, branched from slim for fp16 performance study.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import copy - -import tensorflow.compat.v1 as tf - -from models.tf1_only import mobilenet -from models.tf1_only import mobilenet_conv_blocks as ops -from models.tf1_only import mobilenet_v2 -from tensorflow.contrib import slim - - -def find_ops(optype): - """Find ops of a given type in graphdef or a graph. - - Args: - optype: operation type (e.g. Conv2D) - Returns: - List of operations. - """ - gd = tf.get_default_graph() - return [var for var in gd.get_operations() if var.type == optype] - - -class MobilenetV2Test(tf.test.TestCase): - - def setUp(self): # pylint: disable=g-missing-super-call - tf.reset_default_graph() - - def testCreation(self): - spec = dict(mobilenet_v2.V2_DEF) - _, ep = mobilenet.mobilenet( - tf.placeholder(tf.float32, (10, 224, 224, 16)), conv_defs=spec) - num_convs = len(find_ops('Conv2D')) - - # This is mostly a sanity test. No deep reason for these particular - # constants. - # - # All but first 2 and last one have two convolutions, and there is one - # extra conv that is not in the spec. (logits) - self.assertEqual(num_convs, len(spec['spec']) * 2 - 2) - # Check that depthwise are exposed. - for i in range(2, 17): - self.assertIn('layer_%d/depthwise_output' % i, ep) - - def testCreationNoClasses(self): - spec = copy.deepcopy(mobilenet_v2.V2_DEF) - net, ep = mobilenet.mobilenet( - tf.placeholder(tf.float32, (10, 224, 224, 16)), conv_defs=spec, - num_classes=None) - self.assertIs(net, ep['global_pool']) - - def testImageSizes(self): - for input_size, output_size in [(224, 7), (192, 6), (160, 5), - (128, 4), (96, 3)]: - tf.reset_default_graph() - _, ep = mobilenet_v2.mobilenet( - tf.placeholder(tf.float32, (10, input_size, input_size, 3))) - - self.assertEqual(ep['layer_18/output'].get_shape().as_list()[1:3], - [output_size] * 2) - - def testWithSplits(self): - spec = copy.deepcopy(mobilenet_v2.V2_DEF) - spec['overrides'] = { - (ops.expanded_conv,): dict(split_expansion=2), - } - _, _ = mobilenet.mobilenet( - tf.placeholder(tf.float32, (10, 224, 224, 16)), conv_defs=spec) - num_convs = len(find_ops('Conv2D')) - # All but 3 op has 3 conv operatore, the remainign 3 have one - # and there is one unaccounted. - self.assertEqual(num_convs, len(spec['spec']) * 3 - 5) - - def testWithOutputStride8(self): - out, _ = mobilenet.mobilenet_base( - tf.placeholder(tf.float32, (10, 224, 224, 16)), - conv_defs=mobilenet_v2.V2_DEF, - output_stride=8, - scope='MobilenetV2') - self.assertEqual(out.get_shape().as_list()[1:3], [28, 28]) - - def testDivisibleBy(self): - tf.reset_default_graph() - mobilenet_v2.mobilenet( - tf.placeholder(tf.float32, (10, 224, 224, 16)), - conv_defs=mobilenet_v2.V2_DEF, - divisible_by=16, - min_depth=32) - s = [op.outputs[0].get_shape().as_list()[-1] for op in find_ops('Conv2D')] - s = set(s) - self.assertSameElements([32, 64, 96, 160, 192, 320, 384, 576, 960, 1280, - 1001], s) - - def testDivisibleByWithArgScope(self): - tf.reset_default_graph() - # Verifies that depth_multiplier arg scope actually works - # if no default min_depth is provided. - with slim.arg_scope((mobilenet.depth_multiplier,), min_depth=32): - mobilenet_v2.mobilenet( - tf.placeholder(tf.float32, (10, 224, 224, 2)), - conv_defs=mobilenet_v2.V2_DEF, depth_multiplier=0.1) - s = [op.outputs[0].get_shape().as_list()[-1] for op in find_ops('Conv2D')] - s = set(s) - self.assertSameElements(s, [32, 192, 128, 1001]) - - def testFineGrained(self): - tf.reset_default_graph() - # Verifies that depth_multiplier arg scope actually works - # if no default min_depth is provided. - - mobilenet_v2.mobilenet( - tf.placeholder(tf.float32, (10, 224, 224, 2)), - conv_defs=mobilenet_v2.V2_DEF, depth_multiplier=0.01, - finegrain_classification_mode=True) - s = [op.outputs[0].get_shape().as_list()[-1] for op in find_ops('Conv2D')] - s = set(s) - # All convolutions will be 8->48, except for the last one. - self.assertSameElements(s, [8, 48, 1001, 1280]) - - def testMobilenetBase(self): - tf.reset_default_graph() - # Verifies that mobilenet_base returns pre-pooling layer. - with slim.arg_scope((mobilenet.depth_multiplier,), min_depth=32): - net, _ = mobilenet_v2.mobilenet_base( - tf.placeholder(tf.float32, (10, 224, 224, 16)), - conv_defs=mobilenet_v2.V2_DEF, depth_multiplier=0.1) - self.assertEqual(net.get_shape().as_list(), [10, 7, 7, 128]) - - def testWithOutputStride16(self): - tf.reset_default_graph() - out, _ = mobilenet.mobilenet_base( - tf.placeholder(tf.float32, (10, 224, 224, 16)), - conv_defs=mobilenet_v2.V2_DEF, - output_stride=16) - self.assertEqual(out.get_shape().as_list()[1:3], [14, 14]) - - def testWithOutputStride8AndExplicitPadding(self): - tf.reset_default_graph() - out, _ = mobilenet.mobilenet_base( - tf.placeholder(tf.float32, (10, 224, 224, 16)), - conv_defs=mobilenet_v2.V2_DEF, - output_stride=8, - use_explicit_padding=True, - scope='MobilenetV2') - self.assertEqual(out.get_shape().as_list()[1:3], [28, 28]) - - def testWithOutputStride16AndExplicitPadding(self): - tf.reset_default_graph() - out, _ = mobilenet.mobilenet_base( - tf.placeholder(tf.float32, (10, 224, 224, 16)), - conv_defs=mobilenet_v2.V2_DEF, - output_stride=16, - use_explicit_padding=True) - self.assertEqual(out.get_shape().as_list()[1:3], [14, 14]) - - def testBatchNormScopeDoesNotHaveIsTrainingWhenItsSetToNone(self): - sc = mobilenet.training_scope(is_training=None) - self.assertNotIn('is_training', sc[slim.arg_scope_func_key( - slim.batch_norm)]) - - def testBatchNormScopeDoesHasIsTrainingWhenItsNotNone(self): - sc = mobilenet.training_scope(is_training=False) - self.assertIn('is_training', sc[slim.arg_scope_func_key(slim.batch_norm)]) - sc = mobilenet.training_scope(is_training=True) - self.assertIn('is_training', sc[slim.arg_scope_func_key(slim.batch_norm)]) - sc = mobilenet.training_scope() - self.assertIn('is_training', sc[slim.arg_scope_func_key(slim.batch_norm)]) - - -if __name__ == '__main__': - tf.disable_v2_behavior() - tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/mobilenet_v2.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/mobilenet_v2.py deleted file mode 100644 index ac811470719c6a3f867fd88484aaa862bce09e76..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/mobilenet_v2.py +++ /dev/null @@ -1,198 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Mobilenet V2 model, branched from slim models for fp16 performance study. - -Architecture: https://arxiv.org/abs/1801.04381 - -The base model gives 72.2% accuracy on ImageNet, with 300MMadds, -3.4 M parameters. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import copy - -import tensorflow.compat.v1 as tf - -from models import model -from models.tf1_only import mobilenet as lib -from models.tf1_only import mobilenet_conv_blocks as ops -from tensorflow.contrib import slim - -op = lib.op - -expand_input = ops.expand_input_by_factor - -# pyformat: disable -# Architecture: https://arxiv.org/abs/1801.04381 -V2_DEF = dict( - defaults={ - # Note: these parameters of batch norm affect the architecture - # that's why they are here and not in training_scope. - (slim.batch_norm,): {'center': True, 'scale': True}, - (slim.conv2d, slim.fully_connected, slim.separable_conv2d): { - 'normalizer_fn': slim.batch_norm, 'activation_fn': tf.nn.relu6 - }, - (ops.expanded_conv,): { - 'expansion_size': expand_input(6), - 'split_expansion': 1, - 'normalizer_fn': slim.batch_norm, - 'residual': True - }, - (slim.conv2d, slim.separable_conv2d): {'padding': 'SAME'} - }, - spec=[ - op(slim.conv2d, stride=2, num_outputs=32, kernel_size=[3, 3]), - op(ops.expanded_conv, - expansion_size=expand_input(1, divisible_by=1), - num_outputs=16), - op(ops.expanded_conv, stride=2, num_outputs=24), - op(ops.expanded_conv, stride=1, num_outputs=24), - op(ops.expanded_conv, stride=2, num_outputs=32), - op(ops.expanded_conv, stride=1, num_outputs=32), - op(ops.expanded_conv, stride=1, num_outputs=32), - op(ops.expanded_conv, stride=2, num_outputs=64), - op(ops.expanded_conv, stride=1, num_outputs=64), - op(ops.expanded_conv, stride=1, num_outputs=64), - op(ops.expanded_conv, stride=1, num_outputs=64), - op(ops.expanded_conv, stride=1, num_outputs=96), - op(ops.expanded_conv, stride=1, num_outputs=96), - op(ops.expanded_conv, stride=1, num_outputs=96), - op(ops.expanded_conv, stride=2, num_outputs=160), - op(ops.expanded_conv, stride=1, num_outputs=160), - op(ops.expanded_conv, stride=1, num_outputs=160), - op(ops.expanded_conv, stride=1, num_outputs=320), - op(slim.conv2d, stride=1, kernel_size=[1, 1], num_outputs=1280) - ], -) -# pyformat: enable - - -@slim.add_arg_scope -def mobilenet(input_tensor, - num_classes=1001, - depth_multiplier=1.0, - scope='MobilenetV2', - conv_defs=None, - finegrain_classification_mode=False, - min_depth=None, - divisible_by=None, - **kwargs): - """Creates mobilenet V2 network. - - Inference mode is created by default. To create training use training_scope - below. - - with tf.contrib.slim.arg_scope(mobilenet_v2.training_scope()): - logits, endpoints = mobilenet_v2.mobilenet(input_tensor) - - Args: - input_tensor: The input tensor - num_classes: number of classes - depth_multiplier: The multiplier applied to scale number of - channels in each layer. Note: this is called depth multiplier in the - paper but the name is kept for consistency with slim's model builder. - scope: Scope of the operator - conv_defs: Allows to override default conv def. - finegrain_classification_mode: When set to True, the model - will keep the last layer large even for small multipliers. Following - https://arxiv.org/abs/1801.04381 - suggests that it improves performance for ImageNet-type of problems. - *Note* ignored if final_endpoint makes the builder exit earlier. - min_depth: If provided, will ensure that all layers will have that - many channels after application of depth multiplier. - divisible_by: If provided will ensure that all layers # channels - will be divisible by this number. - **kwargs: passed directly to mobilenet.mobilenet: - prediction_fn- what prediction function to use. - reuse-: whether to reuse variables (if reuse set to true, scope - must be given). - Returns: - logits/endpoints pair - - Raises: - ValueError: On invalid arguments - """ - if conv_defs is None: - conv_defs = V2_DEF - if 'multiplier' in kwargs: - raise ValueError('mobilenetv2 doesn\'t support generic ' - 'multiplier parameter use "depth_multiplier" instead.') - if finegrain_classification_mode: - conv_defs = copy.deepcopy(conv_defs) - if depth_multiplier < 1: - conv_defs['spec'][-1].params['num_outputs'] /= depth_multiplier - - depth_args = {} - # NB: do not set depth_args unless they are provided to avoid overriding - # whatever default depth_multiplier might have thanks to arg_scope. - if min_depth is not None: - depth_args['min_depth'] = min_depth - if divisible_by is not None: - depth_args['divisible_by'] = divisible_by - - with slim.arg_scope((lib.depth_multiplier,), **depth_args): - return lib.mobilenet( - input_tensor, - num_classes=num_classes, - conv_defs=conv_defs, - scope=scope, - multiplier=depth_multiplier, - **kwargs) - - -@slim.add_arg_scope -def mobilenet_base(input_tensor, depth_multiplier=1.0, **kwargs): - """Creates base of the mobilenet (no pooling and no logits) .""" - return mobilenet( - input_tensor, depth_multiplier=depth_multiplier, base_only=True, **kwargs) - - -def training_scope(**kwargs): - """Defines MobilenetV2 training scope. - - Usage: - with tf.contrib.slim.arg_scope(mobilenet_v2.training_scope()): - logits, endpoints = mobilenet_v2.mobilenet(input_tensor) - - with slim. - - Args: - **kwargs: Passed to mobilenet.training_scope. The following parameters - are supported: - weight_decay- The weight decay to use for regularizing the model. - stddev- Standard deviation for initialization, if negative uses xavier. - dropout_keep_prob- dropout keep probability - bn_decay- decay for the batch norm moving averages. - - Returns: - An `arg_scope` to use for the mobilenet v2 model. - """ - return lib.training_scope(**kwargs) - - -class MobilenetModel(model.CNNModel): - """Mobilenet model configuration.""" - - def __init__(self, params=None): - super(MobilenetModel, self).__init__( - 'mobilenet', 224, 32, 0.005, params=params) - - def add_inference(self, cnn): - with slim.arg_scope(training_scope(is_training=cnn.phase_train)): - cnn.top_layer, _ = mobilenet(cnn.top_layer, is_training=cnn.phase_train) - cnn.top_size = cnn.top_layer.shape[-1].value diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/nasnet_model.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/nasnet_model.py deleted file mode 100644 index 560d86bcaf88589734696748379150a6615a58fc..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/nasnet_model.py +++ /dev/null @@ -1,582 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Model configurations for nasnet. - -Paper: https://arxiv.org/abs/1707.07012 -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow.compat.v1 as tf - -from models import model -from models.tf1_only import nasnet_utils -from tensorflow.contrib import framework as contrib_framework -from tensorflow.contrib import layers as contrib_layers -from tensorflow.contrib import slim -from tensorflow.contrib import training as contrib_training - -arg_scope = contrib_framework.arg_scope - - -# Notes for training NASNet Cifar Model -# ------------------------------------- -# batch_size: 32 -# learning rate: 0.025 -# cosine (single period) learning rate decay -# auxiliary head loss weighting: 0.4 -# clip global norm of all gradients by 5 -def _cifar_config(is_training=True, data_format=None, total_steps=None): - drop_path_keep_prob = 1.0 if not is_training else 0.6 - return contrib_training.HParams( - stem_multiplier=3.0, - drop_path_keep_prob=drop_path_keep_prob, - num_cells=18, - use_aux_head=1, - num_conv_filters=32, - dense_dropout_keep_prob=1.0, - filter_scaling_rate=2.0, - num_reduction_layers=2, - skip_reduction_layer_input=0, - data_format=data_format or 'NHWC', - # 600 epochs with a batch size of 32 - # This is used for the drop path probabilities since it needs to increase - # the drop out probability over the course of training. - total_training_steps=total_steps or 937500, - ) - - -# Notes for training large NASNet model on ImageNet -# ------------------------------------- -# batch size (per replica): 16 -# learning rate: 0.015 * 100 -# learning rate decay factor: 0.97 -# num epochs per decay: 2.4 -# sync sgd with 100 replicas -# auxiliary head loss weighting: 0.4 -# label smoothing: 0.1 -# clip global norm of all gradients by 10 -def _large_imagenet_config(is_training=True, data_format=None, - total_steps=None): - drop_path_keep_prob = 1.0 if not is_training else 0.7 - return contrib_training.HParams( - stem_multiplier=3.0, - dense_dropout_keep_prob=0.5, - num_cells=18, - filter_scaling_rate=2.0, - num_conv_filters=168, - drop_path_keep_prob=drop_path_keep_prob, - use_aux_head=1, - num_reduction_layers=2, - skip_reduction_layer_input=1, - data_format=data_format or 'NHWC', - total_training_steps=total_steps or 250000, - ) - - -# Notes for training the mobile NASNet ImageNet model -# ------------------------------------- -# batch size (per replica): 32 -# learning rate: 0.04 * 50 -# learning rate scaling factor: 0.97 -# num epochs per decay: 2.4 -# sync sgd with 50 replicas -# auxiliary head weighting: 0.4 -# label smoothing: 0.1 -# clip global norm of all gradients by 10 -def _mobile_imagenet_config(data_format=None, total_steps=None): - return contrib_training.HParams( - stem_multiplier=1.0, - dense_dropout_keep_prob=0.5, - num_cells=12, - filter_scaling_rate=2.0, - drop_path_keep_prob=1.0, - num_conv_filters=44, - use_aux_head=1, - num_reduction_layers=2, - skip_reduction_layer_input=0, - data_format=data_format or 'NHWC', - total_training_steps=total_steps or 250000, - ) - - -def nasnet_cifar_arg_scope(weight_decay=5e-4, - batch_norm_decay=0.9, - batch_norm_epsilon=1e-5): - """Defines the default arg scope for the NASNet-A Cifar model. - - Args: - weight_decay: The weight decay to use for regularizing the model. - batch_norm_decay: Decay for batch norm moving average. - batch_norm_epsilon: Small float added to variance to avoid dividing by zero - in batch norm. - Returns: - An `arg_scope` to use for the NASNet Cifar Model. - """ - batch_norm_params = { - # Decay for the moving averages. - 'decay': batch_norm_decay, - # epsilon to prevent 0s in variance. - 'epsilon': batch_norm_epsilon, - 'scale': True, - 'fused': True, - } - weights_regularizer = contrib_layers.l2_regularizer(weight_decay) - weights_initializer = contrib_layers.variance_scaling_initializer( - mode='FAN_OUT') - with arg_scope( - [slim.fully_connected, slim.conv2d, slim.separable_conv2d], - weights_regularizer=weights_regularizer, - weights_initializer=weights_initializer): - with arg_scope([slim.fully_connected], activation_fn=None, scope='FC'): - with arg_scope( - [slim.conv2d, slim.separable_conv2d], - activation_fn=None, - biases_initializer=None): - with arg_scope([slim.batch_norm], **batch_norm_params) as sc: - return sc - - -def nasnet_mobile_arg_scope(weight_decay=4e-5, - batch_norm_decay=0.9997, - batch_norm_epsilon=1e-3): - """Defines the default arg scope for the NASNet-A Mobile ImageNet model. - - Args: - weight_decay: The weight decay to use for regularizing the model. - batch_norm_decay: Decay for batch norm moving average. - batch_norm_epsilon: Small float added to variance to avoid dividing by zero - in batch norm. - Returns: - An `arg_scope` to use for the NASNet Mobile Model. - """ - batch_norm_params = { - # Decay for the moving averages. - 'decay': batch_norm_decay, - # epsilon to prevent 0s in variance. - 'epsilon': batch_norm_epsilon, - 'scale': True, - 'fused': True, - } - weights_regularizer = contrib_layers.l2_regularizer(weight_decay) - weights_initializer = contrib_layers.variance_scaling_initializer( - mode='FAN_OUT') - with arg_scope( - [slim.fully_connected, slim.conv2d, slim.separable_conv2d], - weights_regularizer=weights_regularizer, - weights_initializer=weights_initializer): - with arg_scope([slim.fully_connected], activation_fn=None, scope='FC'): - with arg_scope( - [slim.conv2d, slim.separable_conv2d], - activation_fn=None, - biases_initializer=None): - with arg_scope([slim.batch_norm], **batch_norm_params) as sc: - return sc - - -def nasnet_large_arg_scope(weight_decay=5e-5, - batch_norm_decay=0.9997, - batch_norm_epsilon=1e-3): - """Defines the default arg scope for the NASNet-A Large ImageNet model. - - Args: - weight_decay: The weight decay to use for regularizing the model. - batch_norm_decay: Decay for batch norm moving average. - batch_norm_epsilon: Small float added to variance to avoid dividing by zero - in batch norm. - Returns: - An `arg_scope` to use for the NASNet Large Model. - """ - batch_norm_params = { - # Decay for the moving averages. - 'decay': batch_norm_decay, - # epsilon to prevent 0s in variance. - 'epsilon': batch_norm_epsilon, - 'scale': True, - 'fused': True, - } - weights_regularizer = contrib_layers.l2_regularizer(weight_decay) - weights_initializer = contrib_layers.variance_scaling_initializer( - mode='FAN_OUT') - with arg_scope( - [slim.fully_connected, slim.conv2d, slim.separable_conv2d], - weights_regularizer=weights_regularizer, - weights_initializer=weights_initializer): - with arg_scope([slim.fully_connected], activation_fn=None, scope='FC'): - with arg_scope( - [slim.conv2d, slim.separable_conv2d], - activation_fn=None, - biases_initializer=None): - with arg_scope([slim.batch_norm], **batch_norm_params) as sc: - return sc - - -def _build_aux_head(net, end_points, num_classes, hparams, scope): - """Auxiliary head used for all models across all datasets.""" - with tf.variable_scope(scope): - aux_logits = tf.identity(net) - with tf.variable_scope('aux_logits'): - aux_logits = slim.avg_pool2d( - aux_logits, [5, 5], stride=3, padding='VALID') - aux_logits = slim.conv2d(aux_logits, 128, [1, 1], scope='proj') - aux_logits = slim.batch_norm(aux_logits, scope='aux_bn0') - aux_logits = tf.nn.relu(aux_logits) - # Shape of feature map before the final layer. - shape = aux_logits.shape - if hparams.data_format == 'NHWC': - shape = shape[1:3] - else: - shape = shape[2:4] - aux_logits = slim.conv2d(aux_logits, 768, shape, padding='VALID') - aux_logits = slim.batch_norm(aux_logits, scope='aux_bn1') - aux_logits = tf.nn.relu(aux_logits) - aux_logits = contrib_layers.flatten(aux_logits) - aux_logits = slim.fully_connected(aux_logits, num_classes) - end_points['AuxLogits'] = aux_logits - - -def _imagenet_stem(inputs, hparams, stem_cell): - """Stem used for models trained on ImageNet.""" - num_stem_cells = 2 - - # 149 x 149 x 32 - num_stem_filters = int(32 * hparams.stem_multiplier) - net = slim.conv2d( - inputs, - num_stem_filters, [3, 3], - stride=2, - scope='conv0', - padding='VALID') - net = slim.batch_norm(net, scope='conv0_bn') - - # Run the reduction cells - cell_outputs = [None, net] - filter_scaling = 1.0 / (hparams.filter_scaling_rate**num_stem_cells) - for cell_num in range(num_stem_cells): - net = stem_cell( - net, - scope='cell_stem_{}'.format(cell_num), - filter_scaling=filter_scaling, - stride=2, - prev_layer=cell_outputs[-2], - cell_num=cell_num) - cell_outputs.append(net) - filter_scaling *= hparams.filter_scaling_rate - return net, cell_outputs - - -def _cifar_stem(inputs, hparams): - """Stem used for models trained on Cifar.""" - num_stem_filters = int(hparams.num_conv_filters * hparams.stem_multiplier) - net = slim.conv2d(inputs, num_stem_filters, 3, scope='l1_stem_3x3') - net = slim.batch_norm(net, scope='l1_stem_bn') - return net, [None, net] - - -def build_nasnet_cifar(images, - num_classes=None, - is_training=True, - data_format=None, - total_steps=None): - """Build NASNet model for the Cifar Dataset.""" - hparams = _cifar_config( - is_training=is_training, data_format=data_format, total_steps=total_steps) - - if tf.test.is_gpu_available() and hparams.data_format == 'NHWC': - tf.logging.info('A GPU is available on the machine, consider using NCHW ' - 'data format for increased speed on GPU.') - - # Calculate the total number of cells in the network - # Add 2 for the reduction cells - total_num_cells = hparams.num_cells + 2 - - normal_cell = nasnet_utils.NasNetANormalCell( - hparams.num_conv_filters, hparams.drop_path_keep_prob, total_num_cells, - hparams.total_training_steps) - reduction_cell = nasnet_utils.NasNetAReductionCell( - hparams.num_conv_filters, hparams.drop_path_keep_prob, total_num_cells, - hparams.total_training_steps) - with arg_scope( - [slim.dropout, nasnet_utils.drop_path, slim.batch_norm], - is_training=is_training): - with arg_scope( - [ - slim.avg_pool2d, slim.max_pool2d, slim.conv2d, slim.batch_norm, - slim.separable_conv2d, nasnet_utils.factorized_reduction, - nasnet_utils.global_avg_pool, nasnet_utils.get_channel_index, - nasnet_utils.get_channel_dim - ], - data_format=hparams.data_format): - return _build_nasnet_base( - images, - normal_cell=normal_cell, - reduction_cell=reduction_cell, - num_classes=num_classes, - hparams=hparams, - is_training=is_training, - stem_type='cifar') - - -build_nasnet_cifar.default_image_size = 32 - - -def build_nasnet_mobile(images, - num_classes=None, - is_training=True, - data_format=None, - total_steps=None, - final_endpoint=None): - """Build NASNet Mobile model for the ImageNet Dataset.""" - hparams = _mobile_imagenet_config( - data_format=data_format, total_steps=total_steps) - - if tf.test.is_gpu_available() and hparams.data_format == 'NHWC': - tf.logging.info('A GPU is available on the machine, consider using NCHW ' - 'data format for increased speed on GPU.') - - # Calculate the total number of cells in the network - # Add 2 for the reduction cells - total_num_cells = hparams.num_cells + 2 - # If ImageNet, then add an additional two for the stem cells - total_num_cells += 2 - - normal_cell = nasnet_utils.NasNetANormalCell( - hparams.num_conv_filters, hparams.drop_path_keep_prob, total_num_cells, - hparams.total_training_steps) - reduction_cell = nasnet_utils.NasNetAReductionCell( - hparams.num_conv_filters, hparams.drop_path_keep_prob, total_num_cells, - hparams.total_training_steps) - with arg_scope( - [slim.dropout, nasnet_utils.drop_path, slim.batch_norm], - is_training=is_training): - with arg_scope( - [ - slim.avg_pool2d, slim.max_pool2d, slim.conv2d, slim.batch_norm, - slim.separable_conv2d, nasnet_utils.factorized_reduction, - nasnet_utils.global_avg_pool, nasnet_utils.get_channel_index, - nasnet_utils.get_channel_dim - ], - data_format=hparams.data_format): - return _build_nasnet_base( - images, - normal_cell=normal_cell, - reduction_cell=reduction_cell, - num_classes=num_classes, - hparams=hparams, - is_training=is_training, - stem_type='imagenet', - final_endpoint=final_endpoint) - - -build_nasnet_mobile.default_image_size = 224 - - -def build_nasnet_large(images, - num_classes=None, - is_training=True, - data_format=None, - total_steps=None, - final_endpoint=None): - """Build NASNet Large model for the ImageNet Dataset.""" - hparams = _large_imagenet_config( - is_training=is_training, data_format=data_format, total_steps=total_steps) - - if tf.test.is_gpu_available() and hparams.data_format == 'NHWC': - tf.logging.info('A GPU is available on the machine, consider using NCHW ' - 'data format for increased speed on GPU.') - - # Calculate the total number of cells in the network - # Add 2 for the reduction cells - total_num_cells = hparams.num_cells + 2 - # If ImageNet, then add an additional two for the stem cells - total_num_cells += 2 - - normal_cell = nasnet_utils.NasNetANormalCell( - hparams.num_conv_filters, hparams.drop_path_keep_prob, total_num_cells, - hparams.total_training_steps) - reduction_cell = nasnet_utils.NasNetAReductionCell( - hparams.num_conv_filters, hparams.drop_path_keep_prob, total_num_cells, - hparams.total_training_steps) - with arg_scope( - [slim.dropout, nasnet_utils.drop_path, slim.batch_norm], - is_training=is_training): - with arg_scope( - [ - slim.avg_pool2d, slim.max_pool2d, slim.conv2d, slim.batch_norm, - slim.separable_conv2d, nasnet_utils.factorized_reduction, - nasnet_utils.global_avg_pool, nasnet_utils.get_channel_index, - nasnet_utils.get_channel_dim - ], - data_format=hparams.data_format): - return _build_nasnet_base( - images, - normal_cell=normal_cell, - reduction_cell=reduction_cell, - num_classes=num_classes, - hparams=hparams, - is_training=is_training, - stem_type='imagenet', - final_endpoint=final_endpoint) - - -build_nasnet_large.default_image_size = 331 - - -def _build_nasnet_base(images, - normal_cell, - reduction_cell, - num_classes, - hparams, - is_training, - stem_type, - final_endpoint=None): - """Constructs a NASNet image model.""" - - end_points = {} - - def add_and_check_endpoint(endpoint_name, net): - end_points[endpoint_name] = net - return final_endpoint and (endpoint_name == final_endpoint) - - # Find where to place the reduction cells or stride normal cells - reduction_indices = nasnet_utils.calc_reduction_layers( - hparams.num_cells, hparams.num_reduction_layers) - stem_cell = reduction_cell - - if stem_type == 'imagenet': - stem = lambda: _imagenet_stem(images, hparams, stem_cell) - elif stem_type == 'cifar': - stem = lambda: _cifar_stem(images, hparams) - else: - raise ValueError('Unknown stem_type: ', stem_type) - net, cell_outputs = stem() - if add_and_check_endpoint('Stem', net): - return net, end_points - - # Setup for building in the auxiliary head. - aux_head_cell_idxes = [] - if len(reduction_indices) >= 2: - aux_head_cell_idxes.append(reduction_indices[1] - 1) - - # Run the cells - filter_scaling = 1.0 - # true_cell_num accounts for the stem cells - true_cell_num = 2 if stem_type == 'imagenet' else 0 - for cell_num in range(hparams.num_cells): - stride = 1 - if hparams.skip_reduction_layer_input: - prev_layer = cell_outputs[-2] - if cell_num in reduction_indices: - filter_scaling *= hparams.filter_scaling_rate - net = reduction_cell( - net, - scope='reduction_cell_{}'.format(reduction_indices.index(cell_num)), - filter_scaling=filter_scaling, - stride=2, - prev_layer=cell_outputs[-2], - cell_num=true_cell_num) - if add_and_check_endpoint( - 'Reduction_Cell_{}'.format(reduction_indices.index(cell_num)), net): - return net, end_points - true_cell_num += 1 - cell_outputs.append(net) - if not hparams.skip_reduction_layer_input: - prev_layer = cell_outputs[-2] - net = normal_cell( - net, - scope='cell_{}'.format(cell_num), - filter_scaling=filter_scaling, - stride=stride, - prev_layer=prev_layer, - cell_num=true_cell_num) - - if add_and_check_endpoint('Cell_{}'.format(cell_num), net): - return net, end_points - true_cell_num += 1 - if (hparams.use_aux_head and cell_num in aux_head_cell_idxes and - num_classes and is_training): - aux_net = tf.nn.relu(net) - _build_aux_head( - aux_net, - end_points, - num_classes, - hparams, - scope='aux_{}'.format(cell_num)) - cell_outputs.append(net) - - # Final softmax layer - with tf.variable_scope('final_layer'): - net = tf.nn.relu(net) - net = nasnet_utils.global_avg_pool(net) - if add_and_check_endpoint('global_pool', net) or num_classes is None: - return net, end_points - net = slim.dropout(net, hparams.dense_dropout_keep_prob, scope='dropout') - logits = slim.fully_connected(net, num_classes) - - if add_and_check_endpoint('Logits', logits): - return net, end_points - - predictions = tf.nn.softmax(logits, name='predictions') - if add_and_check_endpoint('Predictions', predictions): - return net, end_points - return logits, end_points - - -class NasnetModel(model.CNNModel): - """Nasnet model configuration.""" - - def __init__(self, params=None): - super(NasnetModel, self).__init__('nasnet', 224, 32, 0.005, params=params) - - def add_inference(self, cnn): - tf.logging.info('input_image_shape: {}'.format(cnn.top_layer.shape)) - cnn.top_layer, _ = build_nasnet_mobile( - images=cnn.top_layer, - is_training=cnn.phase_train, - data_format=cnn.data_format) - cnn.top_size = cnn.top_layer.shape[-1].value - - -class NasnetLargeModel(model.CNNModel): - """Nasnet model configuration.""" - - def __init__(self, params=None): - super(NasnetLargeModel, self).__init__( - 'nasnet', 331, 16, 0.005, params=params) - - def add_inference(self, cnn): - tf.logging.info('input_image_shape: {}'.format(cnn.top_layer.shape)) - cnn.top_layer, _ = build_nasnet_large( - images=cnn.top_layer, - is_training=cnn.phase_train, - data_format=cnn.data_format) - cnn.top_size = cnn.top_layer.shape[-1].value - - -class NasnetCifarModel(model.CNNModel): - """Nasnet cifar model configuration.""" - - def __init__(self, params=None): - super(NasnetCifarModel, self).__init__( - 'nasnet', 32, 32, 0.025, params=params) - - def add_inference(self, cnn): - tf.logging.info('input_image_shape: {}'.format(cnn.top_layer.shape)) - cnn.top_layer, _ = build_nasnet_cifar( - images=cnn.top_layer, - is_training=cnn.phase_train, - data_format=cnn.data_format) - cnn.top_size = cnn.top_layer.shape[-1].value diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/nasnet_test.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/nasnet_test.py deleted file mode 100644 index 4e3bc3776e992c2688e6dd9dfeddbbf7835c6774..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/nasnet_test.py +++ /dev/null @@ -1,289 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for nasnet.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow.compat.v1 as tf - -from models.tf1_only import nasnet_model as nasnet -from tensorflow.contrib import slim - - -class NASNetTest(tf.test.TestCase): - - def testBuildLogitsCifarModel(self): - batch_size = 5 - height, width = 32, 32 - num_classes = 10 - inputs = tf.random_uniform((batch_size, height, width, 3)) - tf.train.create_global_step() - with slim.arg_scope(nasnet.nasnet_cifar_arg_scope()): - logits, end_points = nasnet.build_nasnet_cifar(inputs, num_classes) - auxlogits = end_points['AuxLogits'] - predictions = end_points['Predictions'] - self.assertListEqual(auxlogits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertListEqual(predictions.get_shape().as_list(), - [batch_size, num_classes]) - - def testBuildLogitsMobileModel(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - inputs = tf.random_uniform((batch_size, height, width, 3)) - tf.train.create_global_step() - with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): - logits, end_points = nasnet.build_nasnet_mobile(inputs, num_classes) - auxlogits = end_points['AuxLogits'] - predictions = end_points['Predictions'] - self.assertListEqual(auxlogits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertListEqual(predictions.get_shape().as_list(), - [batch_size, num_classes]) - - def testBuildLogitsLargeModel(self): - batch_size = 5 - height, width = 331, 331 - num_classes = 1000 - inputs = tf.random_uniform((batch_size, height, width, 3)) - tf.train.create_global_step() - with slim.arg_scope(nasnet.nasnet_large_arg_scope()): - logits, end_points = nasnet.build_nasnet_large(inputs, num_classes) - auxlogits = end_points['AuxLogits'] - predictions = end_points['Predictions'] - self.assertListEqual(auxlogits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertListEqual(logits.get_shape().as_list(), - [batch_size, num_classes]) - self.assertListEqual(predictions.get_shape().as_list(), - [batch_size, num_classes]) - - def testBuildPreLogitsCifarModel(self): - batch_size = 5 - height, width = 32, 32 - num_classes = None - inputs = tf.random_uniform((batch_size, height, width, 3)) - tf.train.create_global_step() - with slim.arg_scope(nasnet.nasnet_cifar_arg_scope()): - net, end_points = nasnet.build_nasnet_cifar(inputs, num_classes) - self.assertNotIn('AuxLogits', end_points) - self.assertNotIn('Predictions', end_points) - self.assertTrue(net.op.name.startswith('final_layer/Mean')) - self.assertListEqual(net.get_shape().as_list(), [batch_size, 768]) - - def testBuildPreLogitsMobileModel(self): - batch_size = 5 - height, width = 224, 224 - num_classes = None - inputs = tf.random_uniform((batch_size, height, width, 3)) - tf.train.create_global_step() - with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): - net, end_points = nasnet.build_nasnet_mobile(inputs, num_classes) - self.assertNotIn('AuxLogits', end_points) - self.assertNotIn('Predictions', end_points) - self.assertTrue(net.op.name.startswith('final_layer/Mean')) - self.assertListEqual(net.get_shape().as_list(), [batch_size, 1056]) - - def testBuildPreLogitsLargeModel(self): - batch_size = 5 - height, width = 331, 331 - num_classes = None - inputs = tf.random_uniform((batch_size, height, width, 3)) - tf.train.create_global_step() - with slim.arg_scope(nasnet.nasnet_large_arg_scope()): - net, end_points = nasnet.build_nasnet_large(inputs, num_classes) - self.assertNotIn('AuxLogits', end_points) - self.assertNotIn('Predictions', end_points) - self.assertTrue(net.op.name.startswith('final_layer/Mean')) - self.assertListEqual(net.get_shape().as_list(), [batch_size, 4032]) - - def testAllEndPointsShapesCifarModel(self): - batch_size = 5 - height, width = 32, 32 - num_classes = 10 - inputs = tf.random_uniform((batch_size, height, width, 3)) - tf.train.create_global_step() - with slim.arg_scope(nasnet.nasnet_cifar_arg_scope()): - _, end_points = nasnet.build_nasnet_cifar(inputs, num_classes) - endpoints_shapes = {'Stem': [batch_size, 32, 32, 96], - 'Cell_0': [batch_size, 32, 32, 192], - 'Cell_1': [batch_size, 32, 32, 192], - 'Cell_2': [batch_size, 32, 32, 192], - 'Cell_3': [batch_size, 32, 32, 192], - 'Cell_4': [batch_size, 32, 32, 192], - 'Cell_5': [batch_size, 32, 32, 192], - 'Cell_6': [batch_size, 16, 16, 384], - 'Cell_7': [batch_size, 16, 16, 384], - 'Cell_8': [batch_size, 16, 16, 384], - 'Cell_9': [batch_size, 16, 16, 384], - 'Cell_10': [batch_size, 16, 16, 384], - 'Cell_11': [batch_size, 16, 16, 384], - 'Cell_12': [batch_size, 8, 8, 768], - 'Cell_13': [batch_size, 8, 8, 768], - 'Cell_14': [batch_size, 8, 8, 768], - 'Cell_15': [batch_size, 8, 8, 768], - 'Cell_16': [batch_size, 8, 8, 768], - 'Cell_17': [batch_size, 8, 8, 768], - 'Reduction_Cell_0': [batch_size, 16, 16, 256], - 'Reduction_Cell_1': [batch_size, 8, 8, 512], - 'global_pool': [batch_size, 768], - # Logits and predictions - 'AuxLogits': [batch_size, num_classes], - 'Logits': [batch_size, num_classes], - 'Predictions': [batch_size, num_classes]} - self.assertCountEqual(endpoints_shapes.keys(), end_points.keys()) - for endpoint_name in endpoints_shapes: - tf.logging.info('Endpoint name: {}'.format(endpoint_name)) - expected_shape = endpoints_shapes[endpoint_name] - self.assertIn(endpoint_name, end_points) - self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), - expected_shape) - - def testAllEndPointsShapesMobileModel(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - inputs = tf.random_uniform((batch_size, height, width, 3)) - tf.train.create_global_step() - with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): - _, end_points = nasnet.build_nasnet_mobile(inputs, num_classes) - endpoints_shapes = {'Stem': [batch_size, 28, 28, 88], - 'Cell_0': [batch_size, 28, 28, 264], - 'Cell_1': [batch_size, 28, 28, 264], - 'Cell_2': [batch_size, 28, 28, 264], - 'Cell_3': [batch_size, 28, 28, 264], - 'Cell_4': [batch_size, 14, 14, 528], - 'Cell_5': [batch_size, 14, 14, 528], - 'Cell_6': [batch_size, 14, 14, 528], - 'Cell_7': [batch_size, 14, 14, 528], - 'Cell_8': [batch_size, 7, 7, 1056], - 'Cell_9': [batch_size, 7, 7, 1056], - 'Cell_10': [batch_size, 7, 7, 1056], - 'Cell_11': [batch_size, 7, 7, 1056], - 'Reduction_Cell_0': [batch_size, 14, 14, 352], - 'Reduction_Cell_1': [batch_size, 7, 7, 704], - 'global_pool': [batch_size, 1056], - # Logits and predictions - 'AuxLogits': [batch_size, num_classes], - 'Logits': [batch_size, num_classes], - 'Predictions': [batch_size, num_classes]} - self.assertCountEqual(endpoints_shapes.keys(), end_points.keys()) - for endpoint_name in endpoints_shapes: - tf.logging.info('Endpoint name: {}'.format(endpoint_name)) - expected_shape = endpoints_shapes[endpoint_name] - self.assertIn(endpoint_name, end_points) - self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), - expected_shape) - - def testAllEndPointsShapesLargeModel(self): - batch_size = 5 - height, width = 331, 331 - num_classes = 1000 - inputs = tf.random_uniform((batch_size, height, width, 3)) - tf.train.create_global_step() - with slim.arg_scope(nasnet.nasnet_large_arg_scope()): - _, end_points = nasnet.build_nasnet_large(inputs, num_classes) - endpoints_shapes = {'Stem': [batch_size, 42, 42, 336], - 'Cell_0': [batch_size, 42, 42, 1008], - 'Cell_1': [batch_size, 42, 42, 1008], - 'Cell_2': [batch_size, 42, 42, 1008], - 'Cell_3': [batch_size, 42, 42, 1008], - 'Cell_4': [batch_size, 42, 42, 1008], - 'Cell_5': [batch_size, 42, 42, 1008], - 'Cell_6': [batch_size, 21, 21, 2016], - 'Cell_7': [batch_size, 21, 21, 2016], - 'Cell_8': [batch_size, 21, 21, 2016], - 'Cell_9': [batch_size, 21, 21, 2016], - 'Cell_10': [batch_size, 21, 21, 2016], - 'Cell_11': [batch_size, 21, 21, 2016], - 'Cell_12': [batch_size, 11, 11, 4032], - 'Cell_13': [batch_size, 11, 11, 4032], - 'Cell_14': [batch_size, 11, 11, 4032], - 'Cell_15': [batch_size, 11, 11, 4032], - 'Cell_16': [batch_size, 11, 11, 4032], - 'Cell_17': [batch_size, 11, 11, 4032], - 'Reduction_Cell_0': [batch_size, 21, 21, 1344], - 'Reduction_Cell_1': [batch_size, 11, 11, 2688], - 'global_pool': [batch_size, 4032], - # Logits and predictions - 'AuxLogits': [batch_size, num_classes], - 'Logits': [batch_size, num_classes], - 'Predictions': [batch_size, num_classes]} - self.assertCountEqual(endpoints_shapes.keys(), end_points.keys()) - for endpoint_name in endpoints_shapes: - tf.logging.info('Endpoint name: {}'.format(endpoint_name)) - expected_shape = endpoints_shapes[endpoint_name] - self.assertIn(endpoint_name, end_points) - self.assertListEqual(end_points[endpoint_name].get_shape().as_list(), - expected_shape) - - def testVariablesSetDeviceMobileModel(self): - batch_size = 5 - height, width = 224, 224 - num_classes = 1000 - inputs = tf.random_uniform((batch_size, height, width, 3)) - tf.train.create_global_step() - # Force all Variables to reside on the device. - with tf.variable_scope('on_cpu'), tf.device('/cpu:0'): - with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): - nasnet.build_nasnet_mobile(inputs, num_classes) - with tf.variable_scope('on_gpu'), tf.device('/gpu:0'): - with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): - nasnet.build_nasnet_mobile(inputs, num_classes) - for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='on_cpu'): - self.assertDeviceEqual(v.device, '/cpu:0') - for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='on_gpu'): - self.assertDeviceEqual(v.device, '/gpu:0') - - def testUnknownBatchSizeMobileModel(self): - batch_size = 1 - height, width = 224, 224 - num_classes = 1000 - with self.test_session() as sess: - inputs = tf.placeholder(tf.float32, (None, height, width, 3)) - with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): - logits, _ = nasnet.build_nasnet_mobile(inputs, num_classes) - self.assertListEqual(logits.get_shape().as_list(), - [None, num_classes]) - images = tf.random_uniform((batch_size, height, width, 3)) - sess.run(tf.global_variables_initializer()) - output = sess.run(logits, {inputs: images.eval()}) - self.assertEqual(output.shape, (batch_size, num_classes)) - - def testEvaluationMobileModel(self): - batch_size = 2 - height, width = 224, 224 - num_classes = 1000 - with self.test_session() as sess: - eval_inputs = tf.random_uniform((batch_size, height, width, 3)) - with slim.arg_scope(nasnet.nasnet_mobile_arg_scope()): - logits, _ = nasnet.build_nasnet_mobile(eval_inputs, - num_classes, - is_training=False) - predictions = tf.argmax(logits, 1) - sess.run(tf.global_variables_initializer()) - output = sess.run(predictions) - self.assertEqual(output.shape, (batch_size,)) - - -if __name__ == '__main__': - tf.disable_v2_behavior() - tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/nasnet_utils.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/nasnet_utils.py deleted file mode 100644 index 9b280b3ea85c35ca9f804ebecbf300d98bda6baa..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/nasnet_utils.py +++ /dev/null @@ -1,492 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A custom module for some common operations used by NASNet. - -Functions exposed in this file: -- calc_reduction_layers -- get_channel_index -- get_channel_dim -- global_avg_pool -- factorized_reduction -- drop_path - -Classes exposed in this file: -- NasNetABaseCell -- NasNetANormalCell -- NasNetAReductionCell -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow.compat.v1 as tf -from tensorflow.contrib import framework as contrib_framework -from tensorflow.contrib import slim - -arg_scope = contrib_framework.arg_scope - -DATA_FORMAT_NCHW = 'NCHW' -DATA_FORMAT_NHWC = 'NHWC' -INVALID = 'null' - - -def calc_reduction_layers(num_cells, num_reduction_layers): - """Figure out what layers should have reductions.""" - reduction_layers = [] - for pool_num in range(1, num_reduction_layers + 1): - layer_num = (float(pool_num) / (num_reduction_layers + 1)) * num_cells - layer_num = int(layer_num) - reduction_layers.append(layer_num) - return reduction_layers - - -@contrib_framework.add_arg_scope -def get_channel_index(data_format=INVALID): - assert data_format != INVALID - axis = 3 if data_format == 'NHWC' else 1 - return axis - - -@contrib_framework.add_arg_scope -def get_channel_dim(shape, data_format=INVALID): - assert data_format != INVALID - assert len(shape) == 4 - if data_format == 'NHWC': - return int(shape[3]) - elif data_format == 'NCHW': - return int(shape[1]) - else: - raise ValueError('Not a valid data_format', data_format) - - -@contrib_framework.add_arg_scope -def global_avg_pool(x, data_format=INVALID): - """Average pool away the height and width spatial dimensions of x.""" - assert data_format != INVALID - assert data_format in ['NHWC', 'NCHW'] - assert x.shape.ndims == 4 - if data_format == 'NHWC': - return tf.reduce_mean(x, [1, 2]) - else: - return tf.reduce_mean(x, [2, 3]) - - -@contrib_framework.add_arg_scope -def factorized_reduction(net, output_filters, stride, data_format=INVALID): - """Reduces the shape of net without information loss due to striding.""" - assert output_filters % 2 == 0, ( - 'Need even number of filters when using this factorized reduction.') - assert data_format != INVALID - if stride == 1: - net = slim.conv2d(net, output_filters, 1, scope='path_conv') - net = slim.batch_norm(net, scope='path_bn') - return net - if data_format == 'NHWC': - stride_spec = [1, stride, stride, 1] - else: - stride_spec = [1, 1, stride, stride] - - # Skip path 1 - path1 = tf.nn.avg_pool( - net, [1, 1, 1, 1], stride_spec, 'VALID', data_format=data_format) - path1 = slim.conv2d(path1, int(output_filters / 2), 1, scope='path1_conv') - - # Skip path 2 - # First pad with 0's on the right and bottom, then shift the filter to - # include those 0's that were added. - if data_format == 'NHWC': - pad_arr = [[0, 0], [0, 1], [0, 1], [0, 0]] - path2 = tf.pad(net, pad_arr)[:, 1:, 1:, :] - concat_axis = 3 - else: - pad_arr = [[0, 0], [0, 0], [0, 1], [0, 1]] - path2 = tf.pad(net, pad_arr)[:, :, 1:, 1:] - concat_axis = 1 - - path2 = tf.nn.avg_pool( - path2, [1, 1, 1, 1], stride_spec, 'VALID', data_format=data_format) - path2 = slim.conv2d(path2, int(output_filters / 2), 1, scope='path2_conv') - - # Concat and apply BN - final_path = tf.concat(values=[path1, path2], axis=concat_axis) - final_path = slim.batch_norm(final_path, scope='final_path_bn') - return final_path - - -@contrib_framework.add_arg_scope -def drop_path(net, keep_prob, is_training=True): - """Drops out a whole example hiddenstate with the specified probability.""" - if is_training: - batch_size = tf.shape(net)[0] - noise_shape = [batch_size, 1, 1, 1] - keep_prob = tf.cast(keep_prob, dtype=net.dtype) - random_tensor = keep_prob - random_tensor += tf.random_uniform(noise_shape, dtype=net.dtype) - binary_tensor = tf.floor(random_tensor) - net = tf.div(net, keep_prob) * binary_tensor - return net - - -def _operation_to_filter_shape(operation): - splitted_operation = operation.split('x') - filter_shape = int(splitted_operation[0][-1]) - assert filter_shape == int( - splitted_operation[1][0]), 'Rectangular filters not supported.' - return filter_shape - - -def _operation_to_num_layers(operation): - splitted_operation = operation.split('_') - if 'x' in splitted_operation[-1]: - return 1 - return int(splitted_operation[-1]) - - -def _operation_to_info(operation): - """Takes in operation name and returns meta information. - - An example would be 'separable_3x3_4' -> (3, 4). - - Args: - operation: String that corresponds to convolution operation. - - Returns: - Tuple of (filter shape, num layers). - """ - num_layers = _operation_to_num_layers(operation) - filter_shape = _operation_to_filter_shape(operation) - return num_layers, filter_shape - - -def _stacked_separable_conv(net, stride, operation, filter_size): - """Takes in an operations and parses it to the correct sep operation.""" - num_layers, kernel_size = _operation_to_info(operation) - net_type = net.dtype - net = tf.cast(net, tf.float32) if net_type == tf.float16 else net - - for layer_num in range(num_layers - 1): - net = tf.nn.relu(net) - net = slim.separable_conv2d( - net, - filter_size, - kernel_size, - depth_multiplier=1, - scope='separable_{0}x{0}_{1}'.format(kernel_size, layer_num + 1), - stride=stride) - net = slim.batch_norm( - net, scope='bn_sep_{0}x{0}_{1}'.format(kernel_size, layer_num + 1)) - stride = 1 - net = tf.nn.relu(net) - net = slim.separable_conv2d( - net, - filter_size, - kernel_size, - depth_multiplier=1, - scope='separable_{0}x{0}_{1}'.format(kernel_size, num_layers), - stride=stride) - net = slim.batch_norm( - net, scope='bn_sep_{0}x{0}_{1}'.format(kernel_size, num_layers)) - net = tf.cast(net, net_type) - return net - - -def _operation_to_pooling_type(operation): - """Takes in the operation string and returns the pooling type.""" - splitted_operation = operation.split('_') - return splitted_operation[0] - - -def _operation_to_pooling_shape(operation): - """Takes in the operation string and returns the pooling kernel shape.""" - splitted_operation = operation.split('_') - shape = splitted_operation[-1] - assert 'x' in shape - filter_height, filter_width = shape.split('x') - assert filter_height == filter_width - return int(filter_height) - - -def _operation_to_pooling_info(operation): - """Parses the pooling operation string to return its type and shape.""" - pooling_type = _operation_to_pooling_type(operation) - pooling_shape = _operation_to_pooling_shape(operation) - return pooling_type, pooling_shape - - -def _pooling(net, stride, operation): - """Parses operation and performs the correct pooling operation on net.""" - padding = 'SAME' - pooling_type, pooling_shape = _operation_to_pooling_info(operation) - if pooling_type == 'avg': - net = slim.avg_pool2d(net, pooling_shape, stride=stride, padding=padding) - elif pooling_type == 'max': - net = slim.max_pool2d(net, pooling_shape, stride=stride, padding=padding) - else: - raise NotImplementedError('Unimplemented pooling type: ', pooling_type) - return net - - -class NasNetABaseCell(object): # pylint: disable=g-classes-have-attributes - """NASNet Cell class that is used as a 'layer' in image architectures. - - Args: - num_conv_filters: The number of filters for each convolution operation. - operations: List of operations that are performed in the NASNet Cell in - order. - used_hiddenstates: Binary array that signals if the hiddenstate was used - within the cell. This is used to determine what outputs of the cell - should be concatenated together. - hiddenstate_indices: Determines what hiddenstates should be combined - together with the specified operations to create the NASNet cell. - """ - - def __init__(self, num_conv_filters, operations, used_hiddenstates, - hiddenstate_indices, drop_path_keep_prob, total_num_cells, - total_training_steps): - self._num_conv_filters = num_conv_filters - self._operations = operations - self._used_hiddenstates = used_hiddenstates - self._hiddenstate_indices = hiddenstate_indices - self._drop_path_keep_prob = drop_path_keep_prob - self._total_num_cells = total_num_cells - self._total_training_steps = total_training_steps - - def _reduce_prev_layer(self, prev_layer, curr_layer): - """Matches dimension of prev_layer to the curr_layer.""" - # Set the prev layer to the current layer if it is none - if prev_layer is None: - return curr_layer - curr_num_filters = self._filter_size - prev_num_filters = get_channel_dim(prev_layer.shape) - curr_filter_shape = int(curr_layer.shape[2]) - prev_filter_shape = int(prev_layer.shape[2]) - if curr_filter_shape != prev_filter_shape: - prev_layer = tf.nn.relu(prev_layer) - prev_layer = factorized_reduction(prev_layer, curr_num_filters, stride=2) - elif curr_num_filters != prev_num_filters: - prev_layer = tf.nn.relu(prev_layer) - prev_layer = slim.conv2d( - prev_layer, curr_num_filters, 1, scope='prev_1x1') - prev_layer = slim.batch_norm(prev_layer, scope='prev_bn') - return prev_layer - - def _cell_base(self, net, prev_layer): - """Runs the beginning of the conv cell before the predicted ops are run.""" - num_filters = self._filter_size - - # Check to be sure prev layer stuff is setup correctly - prev_layer = self._reduce_prev_layer(prev_layer, net) - - net = tf.nn.relu(net) - net = slim.conv2d(net, num_filters, 1, scope='1x1') - net = slim.batch_norm(net, scope='beginning_bn') - split_axis = get_channel_index() - net = tf.split(axis=split_axis, num_or_size_splits=1, value=net) - for split in net: - assert int(split.shape[split_axis] == int( - self._num_conv_filters * self._filter_scaling)) - net.append(prev_layer) - return net - - def __call__(self, - net, - scope=None, - filter_scaling=1, - stride=1, - prev_layer=None, - cell_num=-1): - """Runs the conv cell.""" - self._cell_num = cell_num - self._filter_scaling = filter_scaling - self._filter_size = int(self._num_conv_filters * filter_scaling) - - i = 0 - with tf.variable_scope(scope): - net = self._cell_base(net, prev_layer) - for iteration in range(5): - with tf.variable_scope('comb_iter_{}'.format(iteration)): - left_hiddenstate_idx, right_hiddenstate_idx = ( - self._hiddenstate_indices[i], self._hiddenstate_indices[i + 1]) - original_input_left = left_hiddenstate_idx < 2 - original_input_right = right_hiddenstate_idx < 2 - h1 = net[left_hiddenstate_idx] - h2 = net[right_hiddenstate_idx] - - operation_left = self._operations[i] - operation_right = self._operations[i + 1] - i += 2 - # Apply conv operations - with tf.variable_scope('left'): - h1 = self._apply_conv_operation(h1, operation_left, stride, - original_input_left) - with tf.variable_scope('right'): - h2 = self._apply_conv_operation(h2, operation_right, stride, - original_input_right) - - # Combine hidden states using 'add'. - with tf.variable_scope('combine'): - h = h1 + h2 - - # Add hiddenstate to the list of hiddenstates we can choose from - net.append(h) - - with tf.variable_scope('cell_output'): - net = self._combine_unused_states(net) - - return net - - def _apply_conv_operation(self, net, operation, stride, - is_from_original_input): - """Applies the predicted conv operation to net.""" - # Dont stride if this is not one of the original hiddenstates - if stride > 1 and not is_from_original_input: - stride = 1 - input_filters = get_channel_dim(net.shape) - filter_size = self._filter_size - if 'separable' in operation: - net = _stacked_separable_conv(net, stride, operation, filter_size) - elif operation in ['none']: - # Check if a stride is needed, then use a strided 1x1 here - if stride > 1 or (input_filters != filter_size): - net = tf.nn.relu(net) - net = slim.conv2d(net, filter_size, 1, stride=stride, scope='1x1') - net = slim.batch_norm(net, scope='bn_1') - elif 'pool' in operation: - net = _pooling(net, stride, operation) - if input_filters != filter_size: - net = slim.conv2d(net, filter_size, 1, stride=1, scope='1x1') - net = slim.batch_norm(net, scope='bn_1') - else: - raise ValueError('Unimplemented operation', operation) - - if operation != 'none': - net = self._apply_drop_path(net) - return net - - def _combine_unused_states(self, net): - """Concatenate the unused hidden states of the cell.""" - used_hiddenstates = self._used_hiddenstates - - final_height = int(net[-1].shape[2]) - final_num_filters = get_channel_dim(net[-1].shape) - assert len(used_hiddenstates) == len(net) - for idx, used_h in enumerate(used_hiddenstates): - curr_height = int(net[idx].shape[2]) - curr_num_filters = get_channel_dim(net[idx].shape) - - # Determine if a reduction should be applied to make the number of - # filters match. - should_reduce = final_num_filters != curr_num_filters - should_reduce = (final_height != curr_height) or should_reduce - should_reduce = should_reduce and not used_h - if should_reduce: - stride = 2 if final_height != curr_height else 1 - with tf.variable_scope('reduction_{}'.format(idx)): - net[idx] = factorized_reduction(net[idx], final_num_filters, stride) - - states_to_combine = ([ - h for h, is_used in zip(net, used_hiddenstates) if not is_used - ]) - - # Return the concat of all the states - concat_axis = get_channel_index() - net = tf.concat(values=states_to_combine, axis=concat_axis) - return net - - @contrib_framework.add_arg_scope # No public API. For internal use only. - def _apply_drop_path(self, - net, - current_step=None, - use_summaries=True, - drop_connect_version='v3'): - """Apply drop_path regularization. - - Args: - net: the Tensor that gets drop_path regularization applied. - current_step: a float32 Tensor with the current global_step value, - to be divided by hparams.total_training_steps. Usually None, which - defaults to tf.train.get_or_create_global_step() properly casted. - use_summaries: a Python boolean. If set to False, no summaries are output. - drop_connect_version: one of 'v1', 'v2', 'v3', controlling whether - the dropout rate is scaled by current_step (v1), layer (v2), or - both (v3, the default). - - Returns: - The dropped-out value of `net`. - """ - drop_path_keep_prob = self._drop_path_keep_prob - if drop_path_keep_prob < 1.0: - assert drop_connect_version in ['v1', 'v2', 'v3'] - if drop_connect_version in ['v2', 'v3']: - # Scale keep prob by layer number - assert self._cell_num != -1 - # The added 2 is for the reduction cells - num_cells = self._total_num_cells - layer_ratio = (self._cell_num + 1) / float(num_cells) - if use_summaries: - with tf.device('/cpu:0'): - tf.summary.scalar('layer_ratio', layer_ratio) - drop_path_keep_prob = 1 - layer_ratio * (1 - drop_path_keep_prob) - if drop_connect_version in ['v1', 'v3']: - # Decrease the keep probability over time - if not current_step: - current_step = tf.cast(tf.train.get_or_create_global_step(), - tf.float32) - drop_path_burn_in_steps = self._total_training_steps - current_ratio = current_step / drop_path_burn_in_steps - current_ratio = tf.minimum(1.0, current_ratio) - if use_summaries: - with tf.device('/cpu:0'): - tf.summary.scalar('current_ratio', current_ratio) - drop_path_keep_prob = (1 - current_ratio * (1 - drop_path_keep_prob)) - if use_summaries: - with tf.device('/cpu:0'): - tf.summary.scalar('drop_path_keep_prob', drop_path_keep_prob) - net = drop_path(net, drop_path_keep_prob) - return net - - -class NasNetANormalCell(NasNetABaseCell): - """NASNetA Normal Cell.""" - - def __init__(self, num_conv_filters, drop_path_keep_prob, total_num_cells, - total_training_steps): - operations = [ - 'separable_5x5_2', 'separable_3x3_2', 'separable_5x5_2', - 'separable_3x3_2', 'avg_pool_3x3', 'none', 'avg_pool_3x3', - 'avg_pool_3x3', 'separable_3x3_2', 'none' - ] - used_hiddenstates = [1, 0, 0, 0, 0, 0, 0] - hiddenstate_indices = [0, 1, 1, 1, 0, 1, 1, 1, 0, 0] - super(NasNetANormalCell, self).__init__( - num_conv_filters, operations, used_hiddenstates, hiddenstate_indices, - drop_path_keep_prob, total_num_cells, total_training_steps) - - -class NasNetAReductionCell(NasNetABaseCell): - """NASNetA Reduction Cell.""" - - def __init__(self, num_conv_filters, drop_path_keep_prob, total_num_cells, - total_training_steps): - operations = [ - 'separable_5x5_2', 'separable_7x7_2', 'max_pool_3x3', 'separable_7x7_2', - 'avg_pool_3x3', 'separable_5x5_2', 'none', 'avg_pool_3x3', - 'separable_3x3_2', 'max_pool_3x3' - ] - used_hiddenstates = [1, 1, 1, 0, 0, 0, 0] - hiddenstate_indices = [0, 1, 0, 1, 0, 1, 3, 2, 2, 0] - super(NasNetAReductionCell, self).__init__( - num_conv_filters, operations, used_hiddenstates, hiddenstate_indices, - drop_path_keep_prob, total_num_cells, total_training_steps) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/ssd_model.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/ssd_model.py deleted file mode 100644 index 56d9e16d3c65aed82f10cfeb2a095e3eabb4186c..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/tf1_only/ssd_model.py +++ /dev/null @@ -1,683 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - - -"""SSD300 Model Configuration. - -References: - Wei Liu, Dragomir Anguelov, Dumitru Erhan, Christian Szegedy, Scott Reed, - Cheng-Yang Fu, Alexander C. Berg - SSD: Single Shot MultiBox Detector - arXiv:1512.02325 - -Ported from MLPerf reference implementation: - https://github.com/mlperf/reference/tree/ssd/single_stage_detector/ssd - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import multiprocessing -import os -import re -import threading -import tensorflow.compat.v1 as tf - -# pylint: disable=g-direct-tensorflow-import -import constants -import mlperf -import ssd_constants -from cnn_util import log_fn -from models import model as model_lib -from models import resnet_model -from tensorflow.contrib import layers as contrib_layers -from tensorflow.python.ops import variables - -BACKBONE_MODEL_SCOPE_NAME = 'resnet34_backbone' - - -class SSD300Model(model_lib.CNNModel): - """Single Shot Multibox Detection (SSD) model for 300x300 image datasets.""" - - def __init__(self, label_num=ssd_constants.NUM_CLASSES, batch_size=32, - learning_rate=1e-3, backbone='resnet34', params=None): - super(SSD300Model, self).__init__('ssd300', 300, batch_size, learning_rate, - params=params) - # For COCO dataset, 80 categories + 1 background = 81 labels - self.label_num = label_num - - # Currently only support ResNet-34 as backbone model - if backbone != 'resnet34': - raise ValueError('Invalid backbone model %s for SSD.' % backbone) - mlperf.logger.log(key=mlperf.tags.BACKBONE, value=backbone) - - # Number of channels and default boxes associated with the following layers: - # ResNet34 layer, Conv7, Conv8_2, Conv9_2, Conv10_2, Conv11_2 - self.out_chan = [256, 512, 512, 256, 256, 256] - mlperf.logger.log(key=mlperf.tags.LOC_CONF_OUT_CHANNELS, - value=self.out_chan) - - # Number of default boxes from layers of different scales - # 38x38x4, 19x19x6, 10x10x6, 5x5x6, 3x3x4, 1x1x4 - self.num_dboxes = [4, 6, 6, 6, 4, 4] - mlperf.logger.log(key=mlperf.tags.NUM_DEFAULTS_PER_CELL, - value=self.num_dboxes) - - # TODO(haoyuzhang): in order to correctly restore in replicated mode, need - # to create a saver for each tower before graph is finalized. Use variable - # manager for better efficiency. - self.backbone_savers = [] - - # Collected predictions for eval stage. It maps each image id in eval - # dataset to a dict containing the following information: - # source_id: raw ID of image - # raw_shape: raw shape of image - # pred_box: encoded box coordinates of prediction - # pred_scores: scores of classes in prediction - self.predictions = {} - - # Global step when predictions are collected. - self.eval_global_step = 0 - - # Average precision. In asynchronous eval mode, this is the latest AP we - # get so far and may not be the results at current eval step. - self.eval_coco_ap = 0 - - # Process, queues, and thread for asynchronous evaluation. When enabled, - # create a separate process (async_eval_process) that continuously pull - # intermediate results from the predictions queue (a multiprocessing queue), - # process them, and push final results into results queue (another - # multiprocessing queue). The main thread is responsible to push messages - # into predictions queue, and start a separate thread to continuously pull - # messages from results queue to update final results. - # Message in predictions queue should be a tuple of two elements: - # (evaluation step, predictions) - # Message in results queue should be a tuple of two elements: - # (evaluation step, final results) - self.async_eval_process = None - self.async_eval_predictions_queue = None - self.async_eval_results_queue = None - self.async_eval_results_getter_thread = None - - # The MLPerf reference uses a starting lr of 1e-3 at bs=32. - self.base_lr_batch_size = 32 - - def skip_final_affine_layer(self): - return True - - def gpu_preprocess_nhwc(self, images, phase_train=True): - try: - import ssd_dataloader # pylint: disable=g-import-not-at-top - except ImportError: - raise ImportError('To use the COCO dataset, you must clone the ' - 'repo https://github.com/tensorflow/models and add ' - 'tensorflow/models and tensorflow/models/research to ' - 'the PYTHONPATH, and compile the protobufs by ' - 'following https://github.com/tensorflow/models/blob/' - 'master/research/object_detection/g3doc/installation.md' - '#protobuf-compilation ; To evaluate using COCO' - 'metric, download and install Python COCO API from' - 'https://github.com/cocodataset/cocoapi') - - if phase_train: - images = ssd_dataloader.color_jitter( - images, brightness=0.125, contrast=0.5, saturation=0.5, hue=0.05) - images = ssd_dataloader.normalize_image(images) - return images - - def add_backbone_model(self, cnn): - # -------------------------------------------------------------------------- - # Resnet-34 backbone model -- modified for SSD - # -------------------------------------------------------------------------- - - # Input 300x300, output 150x150 - cnn.conv(64, 7, 7, 2, 2, mode='SAME_RESNET', use_batch_norm=True) - cnn.mpool(3, 3, 2, 2, mode='SAME') - - resnet34_layers = [3, 4, 6, 3] - version = 'v1' - - # ResNet-34 block group 1 - # Input 150x150, output 75x75 - for i in range(resnet34_layers[0]): - # Last argument forces residual_block to use projection shortcut, even - # though the numbers of input and output channels are equal - resnet_model.residual_block(cnn, 64, 1, version) - - # ResNet-34 block group 2 - # Input 75x75, output 38x38 - for i in range(resnet34_layers[1]): - stride = 2 if i == 0 else 1 - resnet_model.residual_block(cnn, 128, stride, version, i == 0) - - # ResNet-34 block group 3 - # This block group is modified: first layer uses stride=1 so that the image - # size does not change in group of layers - # Input 38x38, output 38x38 - for i in range(resnet34_layers[2]): - # The following line is intentionally commented out to differentiate from - # the original ResNet-34 model - # stride = 2 if i == 0 else 1 - resnet_model.residual_block(cnn, 256, stride, version, i == 0) - - # ResNet-34 block group 4: removed final block group - # The following 3 lines are intentionally commented out to differentiate - # from the original ResNet-34 model - # for i in range(resnet34_layers[3]): - # stride = 2 if i == 0 else 1 - # resnet_model.residual_block(cnn, 512, stride, version, i == 0) - - def add_inference(self, cnn): - cnn.use_batch_norm = True - cnn.batch_norm_config = {'decay': ssd_constants.BATCH_NORM_DECAY, - 'epsilon': ssd_constants.BATCH_NORM_EPSILON, - 'scale': True} - - with tf.variable_scope(BACKBONE_MODEL_SCOPE_NAME): - self.add_backbone_model(cnn) - - # -------------------------------------------------------------------------- - # SSD additional layers - # -------------------------------------------------------------------------- - - def add_ssd_layer(cnn, depth, k_size, stride, mode): - return cnn.conv( - depth, - k_size, - k_size, - stride, - stride, - mode=mode, - use_batch_norm=False, - kernel_initializer=contrib_layers.xavier_initializer()) - - # Activations for feature maps of different layers - self.activations = [cnn.top_layer] - # Conv7_1, Conv7_2 - # Input 38x38, output 19x19 - add_ssd_layer(cnn, 256, 1, 1, 'valid') - self.activations.append(add_ssd_layer(cnn, 512, 3, 2, 'same')) - - # Conv8_1, Conv8_2 - # Input 19x19, output 10x10 - add_ssd_layer(cnn, 256, 1, 1, 'valid') - self.activations.append(add_ssd_layer(cnn, 512, 3, 2, 'same')) - - # Conv9_1, Conv9_2 - # Input 10x10, output 5x5 - add_ssd_layer(cnn, 128, 1, 1, 'valid') - self.activations.append(add_ssd_layer(cnn, 256, 3, 2, 'same')) - - # Conv10_1, Conv10_2 - # Input 5x5, output 3x3 - add_ssd_layer(cnn, 128, 1, 1, 'valid') - self.activations.append(add_ssd_layer(cnn, 256, 3, 1, 'valid')) - - # Conv11_1, Conv11_2 - # Input 3x3, output 1x1 - add_ssd_layer(cnn, 128, 1, 1, 'valid') - self.activations.append(add_ssd_layer(cnn, 256, 3, 1, 'valid')) - - self.loc = [] - self.conf = [] - - for nd, ac, oc in zip(self.num_dboxes, self.activations, self.out_chan): - l = cnn.conv( - nd * 4, - 3, - 3, - 1, - 1, - input_layer=ac, - num_channels_in=oc, - activation=None, - use_batch_norm=False, - kernel_initializer=contrib_layers.xavier_initializer()) - scale = l.get_shape()[-1] - # shape = [batch_size, nd * 4, scale, scale] - l = tf.reshape(l, [self.batch_size, nd, 4, scale, scale]) - # shape = [batch_size, nd, 4, scale, scale] - l = tf.transpose(l, [0, 1, 3, 4, 2]) - # shape = [batch_size, nd, scale, scale, 4] - self.loc.append(tf.reshape(l, [self.batch_size, -1, 4])) - # shape = [batch_size, nd * scale * scale, 4] - - c = cnn.conv( - nd * self.label_num, - 3, - 3, - 1, - 1, - input_layer=ac, - num_channels_in=oc, - activation=None, - use_batch_norm=False, - kernel_initializer=contrib_layers.xavier_initializer()) - # shape = [batch_size, nd * label_num, scale, scale] - c = tf.reshape(c, [self.batch_size, nd, self.label_num, scale, scale]) - # shape = [batch_size, nd, label_num, scale, scale] - c = tf.transpose(c, [0, 1, 3, 4, 2]) - # shape = [batch_size, nd, scale, scale, label_num] - self.conf.append(tf.reshape(c, [self.batch_size, -1, self.label_num])) - # shape = [batch_size, nd * scale * scale, label_num] - - # Shape of locs: [batch_size, NUM_SSD_BOXES, 4] - # Shape of confs: [batch_size, NUM_SSD_BOXES, label_num] - locs, confs = tf.concat(self.loc, 1), tf.concat(self.conf, 1) - - # Pack location and confidence outputs into a single output layer - # Shape of logits: [batch_size, NUM_SSD_BOXES, 4+label_num] - logits = tf.concat([locs, confs], 2) - - cnn.top_layer = logits - cnn.top_size = 4 + self.label_num - - return cnn.top_layer - - def get_learning_rate(self, global_step, batch_size): - rescaled_lr = self.get_scaled_base_learning_rate(batch_size) - # Defined in MLPerf reference model - boundaries = [160000, 200000] - boundaries = [b * self.base_lr_batch_size // batch_size for b in boundaries] - decays = [1, 0.1, 0.01] - learning_rates = [rescaled_lr * d for d in decays] - lr = tf.train.piecewise_constant(global_step, boundaries, learning_rates) - warmup_steps = int(118287 / batch_size * 5) - warmup_lr = ( - rescaled_lr * tf.cast(global_step, tf.float32) / tf.cast( - warmup_steps, tf.float32)) - return tf.cond(global_step < warmup_steps, lambda: warmup_lr, lambda: lr) - - def get_scaled_base_learning_rate(self, batch_size): - """Calculates base learning rate for creating lr schedule. - - In replicated mode, gradients are summed rather than averaged which, with - the sgd and momentum optimizers, increases the effective learning rate by - lr * num_gpus. Dividing the base lr by num_gpus negates the increase. - - Args: - batch_size: Total batch-size. - - Returns: - Base learning rate to use to create lr schedule. - """ - base_lr = self.learning_rate - if self.params.variable_update == 'replicated': - base_lr = self.learning_rate / self.params.num_gpus - scaled_lr = base_lr * (batch_size / self.base_lr_batch_size) - return scaled_lr - - def _collect_backbone_vars(self): - backbone_vars = tf.get_collection( - tf.GraphKeys.GLOBAL_VARIABLES, scope='.*'+ BACKBONE_MODEL_SCOPE_NAME) - var_list = {} - - # Assume variables in the checkpoint are following the naming convention of - # a model checkpoint trained with TF official model - # TODO(haoyuzhang): the following variable name parsing is hacky and easy - # to break if there is change in naming convention of either benchmarks or - # official models. - for v in backbone_vars: - # conv2d variable example (model <-- checkpoint): - # v/cg/conv24/conv2d/kernel:0 <-- conv2d_24/kernel - if 'conv2d' in v.name: - re_match = re.search(r'conv(\d+)/conv2d/(.+):', v.name) - if re_match: - layer_id = int(re_match.group(1)) - param_name = re_match.group(2) - vname_in_ckpt = self._var_name_in_official_model_ckpt( - 'conv2d', layer_id, param_name) - var_list[vname_in_ckpt] = v - - # batchnorm varariable example: - # v/cg/conv24/batchnorm25/gamma:0 <-- batch_normalization_25/gamma - elif 'batchnorm' in v.name: - re_match = re.search(r'batchnorm(\d+)/(.+):', v.name) - if re_match: - layer_id = int(re_match.group(1)) - param_name = re_match.group(2) - vname_in_ckpt = self._var_name_in_official_model_ckpt( - 'batch_normalization', layer_id, param_name) - var_list[vname_in_ckpt] = v - - return var_list - - def _var_name_in_official_model_ckpt(self, layer_name, layer_id, param_name): - """Return variable names according to convention in TF official models.""" - vname_in_ckpt = layer_name - if layer_id > 0: - vname_in_ckpt += '_' + str(layer_id) - vname_in_ckpt += '/' + param_name - return vname_in_ckpt - - def loss_function(self, inputs, build_network_result): - logits = build_network_result.logits - - # Unpack model output back to locations and confidence scores of predictions - # Shape of pred_loc: [batch_size, NUM_SSD_BOXES, 4] - # Shape of pred_label: [batch_size, NUM_SSD_BOXES, label_num] - pred_loc, pred_label = tf.split(logits, [4, self.label_num], 2) - - # Shape of gt_loc: [batch_size, NUM_SSD_BOXES, 4] - # Shape of gt_label: [batch_size, NUM_SSD_BOXES, 1] - # Shape of num_gt: [batch_size] - _, gt_loc, gt_label, num_gt = inputs - gt_label = tf.cast(gt_label, tf.int32) - - box_loss = self._localization_loss(pred_loc, gt_loc, gt_label, num_gt) - class_loss = self._classification_loss(pred_label, gt_label, num_gt) - - tf.summary.scalar('box_loss', tf.reduce_mean(box_loss)) - tf.summary.scalar('class_loss', tf.reduce_mean(class_loss)) - return class_loss + box_loss - - def _localization_loss(self, pred_loc, gt_loc, gt_label, num_matched_boxes): - """Computes the localization loss. - - Computes the localization loss using smooth l1 loss. - Args: - pred_loc: a flatten tensor that includes all predicted locations. The - shape is [batch_size, num_anchors, 4]. - gt_loc: a tensor representing box regression targets in - [batch_size, num_anchors, 4]. - gt_label: a tensor that represents the classification groundtruth targets. - The shape is [batch_size, num_anchors, 1]. - num_matched_boxes: the number of anchors that are matched to a groundtruth - targets, used as the loss normalizater. The shape is [batch_size]. - Returns: - box_loss: a float32 representing total box regression loss. - """ - mask = tf.greater(tf.squeeze(gt_label), 0) - float_mask = tf.cast(mask, tf.float32) - - smooth_l1 = tf.reduce_sum(tf.losses.huber_loss( - gt_loc, pred_loc, - reduction=tf.losses.Reduction.NONE - ), axis=2) - smooth_l1 = tf.multiply(smooth_l1, float_mask) - box_loss = tf.reduce_sum(smooth_l1, axis=1) - - return tf.reduce_mean(box_loss / num_matched_boxes) - - def _classification_loss(self, pred_label, gt_label, num_matched_boxes): - """Computes the classification loss. - - Computes the classification loss with hard negative mining. - Args: - pred_label: a flatten tensor that includes all predicted class. The shape - is [batch_size, num_anchors, num_classes]. - gt_label: a tensor that represents the classification groundtruth targets. - The shape is [batch_size, num_anchors, 1]. - num_matched_boxes: the number of anchors that are matched to a groundtruth - targets. This is used as the loss normalizater. - - Returns: - box_loss: a float32 representing total box regression loss. - """ - cross_entropy = tf.losses.sparse_softmax_cross_entropy( - gt_label, pred_label, reduction=tf.losses.Reduction.NONE) - - mask = tf.greater(tf.squeeze(gt_label), 0) - float_mask = tf.cast(mask, tf.float32) - - # Hard example mining - neg_masked_cross_entropy = cross_entropy * (1 - float_mask) - relative_position = tf.argsort( - tf.argsort( - neg_masked_cross_entropy, direction='DESCENDING')) - num_neg_boxes = tf.minimum( - tf.to_int32(num_matched_boxes) * ssd_constants.NEGS_PER_POSITIVE, - ssd_constants.NUM_SSD_BOXES) - top_k_neg_mask = tf.cast(tf.less( - relative_position, - tf.tile(num_neg_boxes[:, tf.newaxis], (1, ssd_constants.NUM_SSD_BOXES)) - ), tf.float32) - - class_loss = tf.reduce_sum( - tf.multiply(cross_entropy, float_mask + top_k_neg_mask), axis=1) - - return tf.reduce_mean(class_loss / num_matched_boxes) - - def add_backbone_saver(self): - # Create saver with mapping from variable names in checkpoint of backbone - # model to variables in SSD model - backbone_var_list = self._collect_backbone_vars() - self.backbone_savers.append(tf.train.Saver(backbone_var_list)) - - def load_backbone_model(self, sess, backbone_model_path): - for saver in self.backbone_savers: - saver.restore(sess, backbone_model_path) - - def get_input_data_types(self, subset): - if subset == 'validation': - return [self.data_type, tf.float32, tf.float32, tf.float32, tf.int32] - return [self.data_type, tf.float32, tf.float32, tf.float32] - - def get_input_shapes(self, subset): - """Return encoded tensor shapes for train and eval data respectively.""" - if subset == 'validation': - # Validation data shapes: - # 1. images - # 2. ground truth locations of boxes - # 3. ground truth classes of objects in boxes - # 4. source image IDs - # 5. raw image shapes - return [ - [self.batch_size, self.image_size, self.image_size, self.depth], - [self.batch_size, ssd_constants.MAX_NUM_EVAL_BOXES, 4], - [self.batch_size, ssd_constants.MAX_NUM_EVAL_BOXES, 1], - [self.batch_size], - [self.batch_size, 3], - ] - - # Training data shapes: - # 1. images - # 2. ground truth locations of boxes - # 3. ground truth classes of objects in boxes - # 4. numbers of objects in images - return [ - [self.batch_size, self.image_size, self.image_size, self.depth], - [self.batch_size, ssd_constants.NUM_SSD_BOXES, 4], - [self.batch_size, ssd_constants.NUM_SSD_BOXES, 1], - [self.batch_size] - ] - - def accuracy_function(self, inputs, logits): - """Returns the ops to measure the mean precision of the model.""" - try: - import ssd_dataloader # pylint: disable=g-import-not-at-top - from object_detection.box_coders import faster_rcnn_box_coder # pylint: disable=g-import-not-at-top - from object_detection.core import box_coder # pylint: disable=g-import-not-at-top - from object_detection.core import box_list # pylint: disable=g-import-not-at-top - except ImportError: - raise ImportError('To use the COCO dataset, you must clone the ' - 'repo https://github.com/tensorflow/models and add ' - 'tensorflow/models and tensorflow/models/research to ' - 'the PYTHONPATH, and compile the protobufs by ' - 'following https://github.com/tensorflow/models/blob/' - 'master/research/object_detection/g3doc/installation.md' - '#protobuf-compilation ; To evaluate using COCO' - 'metric, download and install Python COCO API from' - 'https://github.com/cocodataset/cocoapi') - - # Unpack model output back to locations and confidence scores of predictions - # pred_locs: relative locations (coordinates) of objects in all SSD boxes - # shape: [batch_size, NUM_SSD_BOXES, 4] - # pred_labels: confidence scores of objects being of all categories - # shape: [batch_size, NUM_SSD_BOXES, label_num] - pred_locs, pred_labels = tf.split(logits, [4, self.label_num], 2) - - ssd_box_coder = faster_rcnn_box_coder.FasterRcnnBoxCoder( - scale_factors=ssd_constants.BOX_CODER_SCALES) - anchors = box_list.BoxList( - tf.convert_to_tensor(ssd_dataloader.DefaultBoxes()('ltrb'))) - pred_boxes = box_coder.batch_decode( - encoded_boxes=pred_locs, box_coder=ssd_box_coder, anchors=anchors) - - pred_scores = tf.nn.softmax(pred_labels, axis=2) - - # TODO(haoyuzhang): maybe use `gt_boxes` and `gt_classes` for visualization. - _, gt_boxes, gt_classes, source_id, raw_shape = inputs # pylint: disable=unused-variable - - return { - (constants.UNREDUCED_ACCURACY_OP_PREFIX + - ssd_constants.PRED_BOXES): pred_boxes, - (constants.UNREDUCED_ACCURACY_OP_PREFIX + - ssd_constants.PRED_SCORES): pred_scores, - # TODO(haoyuzhang): maybe use these values for visualization. - # constants.UNREDUCED_ACCURACY_OP_PREFIX+'gt_boxes': gt_boxes, - # constants.UNREDUCED_ACCURACY_OP_PREFIX+'gt_classes': gt_classes, - (constants.UNREDUCED_ACCURACY_OP_PREFIX + - ssd_constants.SOURCE_ID): source_id, - (constants.UNREDUCED_ACCURACY_OP_PREFIX + - ssd_constants.RAW_SHAPE): raw_shape - } - - def postprocess(self, results): - """Postprocess results returned from model.""" - try: - import coco_metric # pylint: disable=g-import-not-at-top - except ImportError: - raise ImportError('To use the COCO dataset, you must clone the ' - 'repo https://github.com/tensorflow/models and add ' - 'tensorflow/models and tensorflow/models/research to ' - 'the PYTHONPATH, and compile the protobufs by ' - 'following https://github.com/tensorflow/models/blob/' - 'master/research/object_detection/g3doc/installation.md' - '#protobuf-compilation ; To evaluate using COCO' - 'metric, download and install Python COCO API from' - 'https://github.com/cocodataset/cocoapi') - - pred_boxes = results[ssd_constants.PRED_BOXES] - pred_scores = results[ssd_constants.PRED_SCORES] - # TODO(haoyuzhang): maybe use these values for visualization. - # gt_boxes = results['gt_boxes'] - # gt_classes = results['gt_classes'] - source_id = results[ssd_constants.SOURCE_ID] - raw_shape = results[ssd_constants.RAW_SHAPE] - - # COCO evaluation requires processing COCO_NUM_VAL_IMAGES exactly once. Due - # to rounding errors (i.e., COCO_NUM_VAL_IMAGES % batch_size != 0), setting - # `num_eval_epochs` to 1 is not enough and will often miss some images. We - # expect user to set `num_eval_epochs` to >1, which will leave some unused - # images from previous steps in `predictions`. Here we check if we are doing - # eval at a new global step. - if results['global_step'] > self.eval_global_step: - self.eval_global_step = results['global_step'] - self.predictions.clear() - - for i, sid in enumerate(source_id): - self.predictions[int(sid)] = { - ssd_constants.PRED_BOXES: pred_boxes[i], - ssd_constants.PRED_SCORES: pred_scores[i], - ssd_constants.SOURCE_ID: source_id[i], - ssd_constants.RAW_SHAPE: raw_shape[i] - } - - # COCO metric calculates mAP only after a full epoch of evaluation. Return - # dummy results for top_N_accuracy to be compatible with benchmar_cnn.py. - if len(self.predictions) >= ssd_constants.COCO_NUM_VAL_IMAGES: - log_fn('Got results for all {:d} eval examples. Calculate mAP...'.format( - ssd_constants.COCO_NUM_VAL_IMAGES)) - - annotation_file = os.path.join(self.params.data_dir, - ssd_constants.ANNOTATION_FILE) - # Size of predictions before decoding about 15--30GB, while size after - # decoding is 100--200MB. When using async eval mode, decoding takes - # 20--30 seconds of main thread time but is necessary to avoid OOM during - # inter-process communication. - decoded_preds = coco_metric.decode_predictions(self.predictions.values()) - self.predictions.clear() - - if self.params.collect_eval_results_async: - def _eval_results_getter(): - """Iteratively get eval results from async eval process.""" - while True: - step, eval_results = self.async_eval_results_queue.get() - self.eval_coco_ap = eval_results['COCO/AP'] - mlperf.logger.log_eval_accuracy( - self.eval_coco_ap, step, self.batch_size * self.params.num_gpus, - ssd_constants.COCO_NUM_TRAIN_IMAGES) - if self.reached_target(): - # Reached target, clear all pending messages in predictions queue - # and insert poison pill to stop the async eval process. - while not self.async_eval_predictions_queue.empty(): - self.async_eval_predictions_queue.get() - self.async_eval_predictions_queue.put('STOP') - break - - if not self.async_eval_process: - # Limiting the number of messages in predictions queue to prevent OOM. - # Each message (predictions data) can potentially consume a lot of - # memory, and normally there should only be few messages in the queue. - # If often blocked on this, consider reducing eval frequency. - self.async_eval_predictions_queue = multiprocessing.Queue(2) - self.async_eval_results_queue = multiprocessing.Queue() - - # Reason to use a Process as opposed to Thread is mainly the - # computationally intensive eval runner. Python multithreading is not - # truly running in parallel, a runner thread would get significantly - # delayed (or alternatively delay the main thread). - self.async_eval_process = multiprocessing.Process( - target=coco_metric.async_eval_runner, - args=(self.async_eval_predictions_queue, - self.async_eval_results_queue, - annotation_file)) - self.async_eval_process.daemon = True - self.async_eval_process.start() - - self.async_eval_results_getter_thread = threading.Thread( - target=_eval_results_getter, args=()) - self.async_eval_results_getter_thread.daemon = True - self.async_eval_results_getter_thread.start() - - self.async_eval_predictions_queue.put( - (self.eval_global_step, decoded_preds)) - return {'top_1_accuracy': 0, 'top_5_accuracy': 0.} - - eval_results = coco_metric.compute_map(decoded_preds, annotation_file) - self.eval_coco_ap = eval_results['COCO/AP'] - ret = {'top_1_accuracy': self.eval_coco_ap, 'top_5_accuracy': 0.} - for metric_key, metric_value in eval_results.items(): - ret[constants.SIMPLE_VALUE_RESULT_PREFIX + metric_key] = metric_value - mlperf.logger.log_eval_accuracy(self.eval_coco_ap, self.eval_global_step, - self.batch_size * self.params.num_gpus, - ssd_constants.COCO_NUM_TRAIN_IMAGES) - return ret - log_fn('Got {:d} out of {:d} eval examples.' - ' Waiting for the remaining to calculate mAP...'.format( - len(self.predictions), ssd_constants.COCO_NUM_VAL_IMAGES)) - return {'top_1_accuracy': self.eval_coco_ap, 'top_5_accuracy': 0.} - - def get_synthetic_inputs(self, input_name, nclass): - """Generating synthetic data matching real data shape and type.""" - inputs = tf.random_uniform( - self.get_input_shapes('train')[0], dtype=self.data_type) - inputs = variables.VariableV1(inputs, trainable=False, - collections=[tf.GraphKeys.LOCAL_VARIABLES], - name=input_name) - boxes = tf.random_uniform( - [self.batch_size, ssd_constants.NUM_SSD_BOXES, 4], dtype=tf.float32) - classes = tf.random_uniform( - [self.batch_size, ssd_constants.NUM_SSD_BOXES, 1], dtype=tf.float32) - nboxes = tf.random_uniform( - [self.batch_size], minval=1, maxval=10, dtype=tf.float32) - return (inputs, boxes, classes, nboxes) - - def reached_target(self): - return (self.params.stop_at_top_1_accuracy and - self.eval_coco_ap >= self.params.stop_at_top_1_accuracy) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/trivial_model.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/trivial_model.py deleted file mode 100644 index 3ba84d72672c6e3c0903c9af2d0dddecdd7fa2c1..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/trivial_model.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Trivial model configuration.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow.compat.v1 as tf -from models import model - - -class TrivialModel(model.CNNModel): - """Trivial model configuration.""" - - def __init__(self, params=None): - super(TrivialModel, self).__init__( - 'trivial', 224 + 3, 32, 0.005, params=params) - - def add_inference(self, cnn): - cnn.reshape([-1, 227 * 227 * 3]) - cnn.affine(1) - cnn.affine(4096) - - -class TrivialCifar10Model(model.CNNModel): - """Trivial cifar10 model configuration.""" - - def __init__(self, params=None): - super(TrivialCifar10Model, self).__init__( - 'trivial', 32, 32, 0.005, params=params) - - def add_inference(self, cnn): - cnn.reshape([-1, 32 * 32 * 3]) - cnn.affine(1) - cnn.affine(4096) - - -class TrivialSSD300Model(model.CNNModel): - """Trivial SSD300 model configuration.""" - - def __init__(self, params=None): - super(TrivialSSD300Model, self).__init__( - 'trivial', 300, params.batch_size, 0.005, params=params) - - def add_inference(self, cnn): - cnn.reshape([-1, 300 * 300 * 3]) - cnn.affine(1) - cnn.affine(4096) - - def get_input_shapes(self, subset): - return [[self.batch_size, 300, 300, 3], - [self.batch_size, 8732, 4], - [self.batch_size, 8732, 1], - [self.batch_size]] - - def loss_function(self, inputs, build_network_result): - images, _, _, labels = inputs - labels = tf.cast(labels, tf.int32) - return super(TrivialSSD300Model, self).loss_function( - (images, labels), build_network_result) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/vgg_model.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/vgg_model.py deleted file mode 100644 index 938385c95bbc916ca8677bca232085334a48bbf4..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/models/vgg_model.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Vgg model configuration. - -Includes multiple models: vgg11, vgg16, vgg19, corresponding to - model A, D, and E in Table 1 of [1]. - -References: -[1] Simonyan, Karen, Andrew Zisserman - Very Deep Convolutional Networks for Large-Scale Image Recognition - arXiv:1409.1556 (2014) -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from six.moves import xrange # pylint: disable=redefined-builtin -from models import model - - -def _construct_vgg(cnn, num_conv_layers): - """Build vgg architecture from blocks.""" - assert len(num_conv_layers) == 5 - for _ in xrange(num_conv_layers[0]): - cnn.conv(64, 3, 3) - cnn.mpool(2, 2) - for _ in xrange(num_conv_layers[1]): - cnn.conv(128, 3, 3) - cnn.mpool(2, 2) - for _ in xrange(num_conv_layers[2]): - cnn.conv(256, 3, 3) - cnn.mpool(2, 2) - for _ in xrange(num_conv_layers[3]): - cnn.conv(512, 3, 3) - cnn.mpool(2, 2) - for _ in xrange(num_conv_layers[4]): - cnn.conv(512, 3, 3) - cnn.mpool(2, 2) - cnn.reshape([-1, 512 * 7 * 7]) - cnn.affine(4096) - cnn.dropout() - cnn.affine(4096) - cnn.dropout() - - -class Vgg11Model(model.CNNModel): - - def __init__(self, params=None): - super(Vgg11Model, self).__init__('vgg11', 224, 64, 0.005, params=params) - - def add_inference(self, cnn): - _construct_vgg(cnn, [1, 1, 2, 2, 2]) - - -class Vgg16Model(model.CNNModel): - - def __init__(self, params=None): - super(Vgg16Model, self).__init__('vgg16', 224, 64, 0.005, params=params) - - def add_inference(self, cnn): - _construct_vgg(cnn, [2, 2, 3, 3, 3]) - - -class Vgg19Model(model.CNNModel): - - def __init__(self, params=None): - super(Vgg19Model, self).__init__('vgg19', 224, 64, 0.005, params=params) - - def add_inference(self, cnn): - _construct_vgg(cnn, [2, 2, 4, 4, 4]) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/platforms/default/util.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/platforms/default/util.py deleted file mode 100644 index e64b9137fa6ccc5d12b07126dcf30265574eae41..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/platforms/default/util.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Utility code for the default platform.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import sys -import tempfile - -import cnn_util -from models import model_config - - -_ROOT_PROJECT_DIR = os.path.dirname(cnn_util.__file__) - - -def define_platform_params(): - """Defines platform-specific parameters. - - Currently there are no platform-specific parameters to be defined. - """ - pass - - -def get_cluster_manager(params, config_proto): - """Returns the cluster manager to be used.""" - return cnn_util.GrpcClusterManager(params, config_proto) - - -def get_command_to_run_python_module(module): - """Returns a command to run a Python module.""" - python_interpretter = sys.executable - if not python_interpretter: - raise ValueError('Could not find Python interpreter') - return [python_interpretter, - os.path.join(_ROOT_PROJECT_DIR, module + '.py')] - - -def get_test_output_dir(): - """Returns a directory where test outputs should be placed.""" - base_dir = os.environ.get('TEST_OUTPUTS_DIR', - '/tmp/tf_cnn_benchmarks_test_outputs') - if not os.path.exists(base_dir): - os.mkdir(base_dir) - return tempfile.mkdtemp(dir=base_dir) - - -def get_test_data_dir(): - """Returns the path to the test_data directory.""" - return os.path.join(_ROOT_PROJECT_DIR, 'test_data') - - -def get_ssd_backborn_model_file(): - raise NotImplementedError - - -def get_ssd_backboard_data_dir(): - raise NotImplementedError - - -def _initialize(params, config_proto): - del params, config_proto - model_config.register_tf1_models() - - -_is_initalized = False - - -def initialize(params, config_proto): - global _is_initalized - if _is_initalized: - return - _is_initalized = True - _initialize(params, config_proto) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/platforms/util.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/platforms/util.py deleted file mode 100644 index 9d569691bdec804080d62d11f8a200cd1ec2f2a9..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/platforms/util.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Utility code for a certain platform. - -This file simply imports everything from the default platform. To switch to a -different platform, the import statement can be changed to point to a new -platform. - -Creating a custom platform can be useful to, e.g., run some initialization code -required by the platform or register a platform-specific model. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from platforms.default.util import * # pylint: disable=unused-import,wildcard-import diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/preprocessing.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/preprocessing.py deleted file mode 100644 index 16fa71b2663a85d9ca2b413da06eab06db48b25a..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/preprocessing.py +++ /dev/null @@ -1,1339 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Image pre-processing utilities. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import math -from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow.compat.v1 as tf - -# pylint: disable=g-direct-tensorflow-import -import cnn_util -from tensorflow.python.data.ops import multi_device_iterator_ops -from tensorflow.python.framework import function -from tensorflow.python.layers import utils -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.platform import gfile -import mlperf - - -def parse_example_proto(example_serialized): - """Parses an Example proto containing a training example of an image. - - The output of the build_image_data.py image preprocessing script is a dataset - containing serialized Example protocol buffers. Each Example proto contains - the following fields: - - image/height: 462 - image/width: 581 - image/colorspace: 'RGB' - image/channels: 3 - image/class/label: 615 - image/class/synset: 'n03623198' - image/class/text: 'knee pad' - image/object/bbox/xmin: 0.1 - image/object/bbox/xmax: 0.9 - image/object/bbox/ymin: 0.2 - image/object/bbox/ymax: 0.6 - image/object/bbox/label: 615 - image/format: 'JPEG' - image/filename: 'ILSVRC2012_val_00041207.JPEG' - image/encoded: - - Args: - example_serialized: scalar Tensor tf.string containing a serialized - Example protocol buffer. - - Returns: - image_buffer: Tensor tf.string containing the contents of a JPEG file. - label: Tensor tf.int32 containing the label. - bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords] - where each coordinate is [0, 1) and the coordinates are arranged as - [ymin, xmin, ymax, xmax]. - text: Tensor tf.string containing the human-readable label. - """ - # Dense features in Example proto. - feature_map = { - 'image/encoded': tf.FixedLenFeature([], dtype=tf.string, - default_value=''), - 'image/class/label': tf.FixedLenFeature([1], dtype=tf.int64, - default_value=-1), - 'image/class/text': tf.FixedLenFeature([], dtype=tf.string, - default_value=''), - } - sparse_float32 = tf.VarLenFeature(dtype=tf.float32) - # Sparse features in Example proto. - feature_map.update( - {k: sparse_float32 for k in ['image/object/bbox/xmin', - 'image/object/bbox/ymin', - 'image/object/bbox/xmax', - 'image/object/bbox/ymax']}) - - features = tf.parse_single_example(example_serialized, feature_map) - label = tf.cast(features['image/class/label'], dtype=tf.int32) - - xmin = tf.expand_dims(features['image/object/bbox/xmin'].values, 0) - ymin = tf.expand_dims(features['image/object/bbox/ymin'].values, 0) - xmax = tf.expand_dims(features['image/object/bbox/xmax'].values, 0) - ymax = tf.expand_dims(features['image/object/bbox/ymax'].values, 0) - - # Note that we impose an ordering of (y, x) just to make life difficult. - bbox = tf.concat([ymin, xmin, ymax, xmax], 0) - - # Force the variable number of bounding boxes into the shape - # [1, num_boxes, coords]. - bbox = tf.expand_dims(bbox, 0) - bbox = tf.transpose(bbox, [0, 2, 1]) - - return features['image/encoded'], label, bbox, features['image/class/text'] - - -_RESIZE_METHOD_MAP = { - 'nearest': tf.image.ResizeMethod.NEAREST_NEIGHBOR, - 'bilinear': tf.image.ResizeMethod.BILINEAR, - 'bicubic': tf.image.ResizeMethod.BICUBIC, - 'area': tf.image.ResizeMethod.AREA -} - - -def get_image_resize_method(resize_method, batch_position=0): - """Get tensorflow resize method. - - If resize_method is 'round_robin', return different methods based on batch - position in a round-robin fashion. NOTE: If the batch size is not a multiple - of the number of methods, then the distribution of methods will not be - uniform. - - Args: - resize_method: (string) nearest, bilinear, bicubic, area, or round_robin. - batch_position: position of the image in a batch. NOTE: this argument can - be an integer or a tensor - Returns: - one of resize type defined in tf.image.ResizeMethod. - """ - - if resize_method != 'round_robin': - return _RESIZE_METHOD_MAP[resize_method] - - # return a resize method based on batch position in a round-robin fashion. - resize_methods = list(_RESIZE_METHOD_MAP.values()) - def lookup(index): - return resize_methods[index] - - def resize_method_0(): - return utils.smart_cond(batch_position % len(resize_methods) == 0, - lambda: lookup(0), resize_method_1) - - def resize_method_1(): - return utils.smart_cond(batch_position % len(resize_methods) == 1, - lambda: lookup(1), resize_method_2) - - def resize_method_2(): - return utils.smart_cond(batch_position % len(resize_methods) == 2, - lambda: lookup(2), lambda: lookup(3)) - - # NOTE(jsimsa): Unfortunately, we cannot use a single recursive function here - # because TF would not be able to construct a finite graph. - - return resize_method_0() - - -def decode_jpeg(image_buffer, scope=None): # , dtype=tf.float32): - """Decode a JPEG string into one 3-D float image Tensor. - - Args: - image_buffer: scalar string Tensor. - scope: Optional scope for op_scope. - Returns: - 3-D float Tensor with values ranging from [0, 1). - """ - # with tf.op_scope([image_buffer], scope, 'decode_jpeg'): - # with tf.name_scope(scope, 'decode_jpeg', [image_buffer]): - with tf.name_scope(scope or 'decode_jpeg'): - # Decode the string as an RGB JPEG. - # Note that the resulting image contains an unknown height and width - # that is set dynamically by decode_jpeg. In other words, the height - # and width of image is unknown at compile-time. - image = tf.image.decode_jpeg(image_buffer, channels=3, - fancy_upscaling=False, - dct_method='INTEGER_FAST') - - # image = tf.Print(image, [tf.shape(image)], 'Image shape: ') - - return image - - -_R_MEAN = 123.68 -_G_MEAN = 116.78 -_B_MEAN = 103.94 -_CHANNEL_MEANS = [_R_MEAN, _G_MEAN, _B_MEAN] - - -def normalized_image(images): - # Rescale from [0, 255] to [0, 2] - images = tf.multiply(images, 1. / 127.5) - # Rescale to [-1, 1] - mlperf.logger.log(key=mlperf.tags.INPUT_MEAN_SUBTRACTION, value=[1.0] * 3) - return tf.subtract(images, 1.0) - - -def eval_image(image, - height, - width, - batch_position, - resize_method, - summary_verbosity=0): - """Get the image for model evaluation. - - We preprocess the image simiarly to Slim, see - https://github.com/tensorflow/models/blob/master/research/slim/preprocessing/vgg_preprocessing.py - Validation images do not have bounding boxes, so to crop the image, we first - resize the image such that the aspect ratio is maintained and the resized - height and width are both at least 1.145 times `height` and `width` - respectively. Then, we do a central crop to size (`height`, `width`). - - Args: - image: 3-D float Tensor representing the image. - height: The height of the image that will be returned. - width: The width of the image that will be returned. - batch_position: position of the image in a batch, which affects how images - are distorted and resized. NOTE: this argument can be an integer or a - tensor - resize_method: one of the strings 'round_robin', 'nearest', 'bilinear', - 'bicubic', or 'area'. - summary_verbosity: Verbosity level for summary ops. Pass 0 to disable both - summaries and checkpoints. - Returns: - An image of size (output_height, output_width, 3) that is resized and - cropped as described above. - """ - # TODO(reedwm): Currently we resize then crop. Investigate if it's faster to - # crop then resize. - with tf.name_scope('eval_image'): - if summary_verbosity >= 3: - tf.summary.image( - 'original_image', tf.expand_dims(image, 0)) - - shape = tf.shape(image) - image_height = shape[0] - image_width = shape[1] - image_height_float = tf.cast(image_height, tf.float32) - image_width_float = tf.cast(image_width, tf.float32) - - # This value is chosen so that in resnet, images are cropped to a size of - # 256 x 256, which matches what other implementations do. The final image - # size for resnet is 224 x 224, and floor(224 * 1.145) = 256. - scale_factor = 1.145 - - # Compute resize_height and resize_width to be the minimum values such that - # 1. The aspect ratio is maintained (i.e. resize_height / resize_width is - # image_height / image_width), and - # 2. resize_height >= height * `scale_factor`, and - # 3. resize_width >= width * `scale_factor` - max_ratio = tf.maximum(height / image_height_float, - width / image_width_float) - resize_height = tf.cast(image_height_float * max_ratio * scale_factor, - tf.int32) - resize_width = tf.cast(image_width_float * max_ratio * scale_factor, - tf.int32) - mlperf.logger.log_input_resize_aspect_preserving(height, width, - scale_factor) - - # Resize the image to shape (`resize_height`, `resize_width`) - image_resize_method = get_image_resize_method(resize_method, batch_position) - distorted_image = tf.image.resize_images(image, - [resize_height, resize_width], - image_resize_method, - align_corners=False) - - # Do a central crop of the image to size (height, width). - # MLPerf requires us to log (height, width) with two different keys. - mlperf.logger.log(key=mlperf.tags.INPUT_CENTRAL_CROP, value=[height, width]) - mlperf.logger.log(key=mlperf.tags.INPUT_RESIZE, value=[height, width]) - total_crop_height = (resize_height - height) - crop_top = total_crop_height // 2 - total_crop_width = (resize_width - width) - crop_left = total_crop_width // 2 - distorted_image = tf.slice(distorted_image, [crop_top, crop_left, 0], - [height, width, 3]) - - distorted_image.set_shape([height, width, 3]) - if summary_verbosity >= 3: - tf.summary.image( - 'cropped_resized_image', tf.expand_dims(distorted_image, 0)) - image = distorted_image - return image - - -def train_image(image_buffer, - height, - width, - bbox, - batch_position, - resize_method, - distortions, - scope=None, - summary_verbosity=0, - distort_color_in_yiq=False, - fuse_decode_and_crop=False): - """Distort one image for training a network. - - Distorting images provides a useful technique for augmenting the data - set during training in order to make the network invariant to aspects - of the image that do not effect the label. - - Args: - image_buffer: scalar string Tensor representing the raw JPEG image buffer. - height: integer - width: integer - bbox: 3-D float Tensor of bounding boxes arranged [1, num_boxes, coords] - where each coordinate is [0, 1) and the coordinates are arranged - as [ymin, xmin, ymax, xmax]. - batch_position: position of the image in a batch, which affects how images - are distorted and resized. NOTE: this argument can be an integer or a - tensor - resize_method: round_robin, nearest, bilinear, bicubic, or area. - distortions: If true, apply full distortions for image colors. - scope: Optional scope for op_scope. - summary_verbosity: Verbosity level for summary ops. Pass 0 to disable both - summaries and checkpoints. - distort_color_in_yiq: distort color of input images in YIQ space. - fuse_decode_and_crop: fuse the decode/crop operation. - Returns: - 3-D float Tensor of distorted image used for training. - """ - # with tf.op_scope([image, height, width, bbox], scope, 'distort_image'): - # with tf.name_scope(scope, 'distort_image', [image, height, width, bbox]): - with tf.name_scope(scope or 'distort_image'): - # A large fraction of image datasets contain a human-annotated bounding box - # delineating the region of the image containing the object of interest. We - # choose to create a new bounding box for the object which is a randomly - # distorted version of the human-annotated bounding box that obeys an - # allowed range of aspect ratios, sizes and overlap with the human-annotated - # bounding box. If no box is supplied, then we assume the bounding box is - # the entire image. - min_object_covered = 0.1 - aspect_ratio_range = [0.75, 1.33] - area_range = [0.05, 1.0] - max_attempts = 100 - mlperf.logger.log(key=mlperf.tags.INPUT_DISTORTED_CROP_MIN_OBJ_COV, - value=min_object_covered) - mlperf.logger.log(key=mlperf.tags.INPUT_DISTORTED_CROP_RATIO_RANGE, - value=aspect_ratio_range) - mlperf.logger.log(key=mlperf.tags.INPUT_DISTORTED_CROP_AREA_RANGE, - value=area_range) - mlperf.logger.log(key=mlperf.tags.INPUT_DISTORTED_CROP_MAX_ATTEMPTS, - value=max_attempts) - - sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box( - tf.image.extract_jpeg_shape(image_buffer), - bounding_boxes=bbox, - min_object_covered=min_object_covered, - aspect_ratio_range=aspect_ratio_range, - area_range=area_range, - max_attempts=max_attempts, - use_image_if_no_bounding_boxes=True) - bbox_begin, bbox_size, distort_bbox = sample_distorted_bounding_box - if summary_verbosity >= 3: - image = tf.image.decode_jpeg(image_buffer, channels=3, - dct_method='INTEGER_FAST') - image = tf.image.convert_image_dtype(image, dtype=tf.float32) - image_with_distorted_box = tf.image.draw_bounding_boxes( - tf.expand_dims(image, 0), distort_bbox) - tf.summary.image( - 'images_with_distorted_bounding_box', - image_with_distorted_box) - - # Crop the image to the specified bounding box. - if fuse_decode_and_crop: - offset_y, offset_x, _ = tf.unstack(bbox_begin) - target_height, target_width, _ = tf.unstack(bbox_size) - crop_window = tf.stack([offset_y, offset_x, target_height, target_width]) - image = tf.image.decode_and_crop_jpeg( - image_buffer, crop_window, channels=3) - else: - image = tf.image.decode_jpeg(image_buffer, channels=3, - dct_method='INTEGER_FAST') - image = tf.slice(image, bbox_begin, bbox_size) - - mlperf.logger.log(key=mlperf.tags.INPUT_RANDOM_FLIP) - distorted_image = tf.image.random_flip_left_right(image) - - # This resizing operation may distort the images because the aspect - # ratio is not respected. - mlperf.logger.log(key=mlperf.tags.INPUT_RESIZE, value=[height, width]) - image_resize_method = get_image_resize_method(resize_method, batch_position) - distorted_image = tf.image.resize_images( - distorted_image, [height, width], - image_resize_method, - align_corners=False) - # Restore the shape since the dynamic slice based upon the bbox_size loses - # the third dimension. - distorted_image.set_shape([height, width, 3]) - if summary_verbosity >= 3: - tf.summary.image('cropped_resized_maybe_flipped_image', - tf.expand_dims(distorted_image, 0)) - - if distortions: - distorted_image = tf.cast(distorted_image, dtype=tf.float32) - # Images values are expected to be in [0,1] for color distortion. - distorted_image /= 255. - # Randomly distort the colors. - distorted_image = distort_color(distorted_image, batch_position, - distort_color_in_yiq=distort_color_in_yiq) - - # Note: This ensures the scaling matches the output of eval_image - distorted_image *= 255 - - if summary_verbosity >= 3: - tf.summary.image( - 'final_distorted_image', - tf.expand_dims(distorted_image, 0)) - return distorted_image - - -def distort_color(image, batch_position=0, distort_color_in_yiq=False, - scope=None): - """Distort the color of the image. - - Each color distortion is non-commutative and thus ordering of the color ops - matters. Ideally we would randomly permute the ordering of the color ops. - Rather then adding that level of complication, we select a distinct ordering - of color ops based on the position of the image in a batch. - - Args: - image: float32 Tensor containing single image. Tensor values should be in - range [0, 1]. - batch_position: the position of the image in a batch. NOTE: this argument - can be an integer or a tensor - distort_color_in_yiq: distort color of input images in YIQ space. - scope: Optional scope for op_scope. - Returns: - color-distorted image - """ - if distort_color_in_yiq: - try: - from tensorflow.contrib.image.python.ops import distort_image_ops # pylint: disable=g-import-not-at-top - except ImportError: - raise ValueError( - 'In TF2, you cannot pass --distortions unless you also pass ' - '--nodistort_color_in_yiq. This is because the random_hsv_in_yiq was ' - 'removed in TF2. --distortions does not improve accuracy on resnet ' - 'so it is not recommended. --nodistort_color_in_yiq also has no ' - 'impact on accuracy, but may hurt performance.') - - with tf.name_scope(scope or 'distort_color'): - - def distort_fn_0(image=image): - """Variant 0 of distort function.""" - image = tf.image.random_brightness(image, max_delta=32. / 255.) - if distort_color_in_yiq: - image = distort_image_ops.random_hsv_in_yiq( - image, lower_saturation=0.5, upper_saturation=1.5, - max_delta_hue=0.2 * math.pi) - else: - image = tf.image.random_saturation(image, lower=0.5, upper=1.5) - image = tf.image.random_hue(image, max_delta=0.2) - image = tf.image.random_contrast(image, lower=0.5, upper=1.5) - return image - - def distort_fn_1(image=image): - """Variant 1 of distort function.""" - image = tf.image.random_brightness(image, max_delta=32. / 255.) - image = tf.image.random_contrast(image, lower=0.5, upper=1.5) - if distort_color_in_yiq: - image = distort_image_ops.random_hsv_in_yiq( - image, lower_saturation=0.5, upper_saturation=1.5, - max_delta_hue=0.2 * math.pi) - else: - image = tf.image.random_saturation(image, lower=0.5, upper=1.5) - image = tf.image.random_hue(image, max_delta=0.2) - return image - - image = utils.smart_cond(batch_position % 2 == 0, distort_fn_0, - distort_fn_1) - # The random_* ops do not necessarily clamp. - image = tf.clip_by_value(image, 0.0, 1.0) - return image - - -class InputPreprocessor(object): - """Base class for all model preprocessors.""" - - def __init__(self, batch_size, output_shapes): - self.batch_size = batch_size - self.output_shapes = output_shapes - - def supports_datasets(self): - """Whether this preprocessor supports dataset.""" - return False - - def minibatch(self, dataset, subset, params, shift_ratio=-1): - """Returns tensors representing a minibatch of all the input.""" - raise NotImplementedError('Must be implemented by subclass.') - - # The methods added below are only supported/used if supports_datasets() - # returns True. - # TODO(laigd): refactor benchmark_cnn.py and put the logic of - # _build_input_processing() into InputPreprocessor. - - def parse_and_preprocess(self, value, batch_position): - """Function to parse and preprocess an Example proto in input pipeline.""" - raise NotImplementedError('Must be implemented by subclass.') - - # TODO(laigd): figure out how to remove these parameters, since the - # preprocessor itself has self.batch_size, self.num_splits, etc defined. - def build_multi_device_iterator(self, batch_size, num_splits, cpu_device, - params, gpu_devices, dataset, doing_eval): - """Creates a MultiDeviceIterator.""" - assert self.supports_datasets() - assert num_splits == len(gpu_devices) - with tf.name_scope('batch_processing'): - if doing_eval: - subset = 'validation' - else: - subset = 'train' - batch_size_per_split = batch_size // num_splits - ds = self.create_dataset( - batch_size, - num_splits, - batch_size_per_split, - dataset, - subset, - train=(not doing_eval), - datasets_repeat_cached_sample=params.datasets_repeat_cached_sample, - num_threads=params.datasets_num_private_threads, - datasets_use_caching=params.datasets_use_caching, - datasets_parallel_interleave_cycle_length=( - params.datasets_parallel_interleave_cycle_length), - datasets_sloppy_parallel_interleave=( - params.datasets_sloppy_parallel_interleave), - datasets_parallel_interleave_prefetch=( - params.datasets_parallel_interleave_prefetch)) - multi_device_iterator = multi_device_iterator_ops.MultiDeviceIterator( - ds, - gpu_devices, - source_device=cpu_device, - max_buffer_size=params.multi_device_iterator_max_buffer_size) - tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, - multi_device_iterator.initializer) - return multi_device_iterator - - def create_dataset(self, - batch_size, - num_splits, - batch_size_per_split, - dataset, - subset, - train, - datasets_repeat_cached_sample, - num_threads=None, - datasets_use_caching=False, - datasets_parallel_interleave_cycle_length=None, - datasets_sloppy_parallel_interleave=False, - datasets_parallel_interleave_prefetch=None): - """Creates a dataset for the benchmark.""" - raise NotImplementedError('Must be implemented by subclass.') - - def create_iterator(self, ds): - ds_iterator = tf.data.make_initializable_iterator(ds) - tf.add_to_collection(tf.GraphKeys.TABLE_INITIALIZERS, - ds_iterator.initializer) - return ds_iterator - - def minibatch_fn(self, batch_size, model_input_shapes, num_splits, - dataset, subset, train, datasets_repeat_cached_sample, - num_threads, datasets_use_caching, - datasets_parallel_interleave_cycle_length, - datasets_sloppy_parallel_interleave, - datasets_parallel_interleave_prefetch): - """Returns a function and list of args for the fn to create a minibatch.""" - assert self.supports_datasets() - batch_size_per_split = batch_size // num_splits - assert batch_size_per_split == model_input_shapes[0][0] - with tf.name_scope('batch_processing'): - ds = self.create_dataset(batch_size, num_splits, batch_size_per_split, - dataset, subset, train, - datasets_repeat_cached_sample, num_threads, - datasets_use_caching, - datasets_parallel_interleave_cycle_length, - datasets_sloppy_parallel_interleave, - datasets_parallel_interleave_prefetch) - ds_iterator = self.create_iterator(ds) - - ds_iterator_string_handle = ds_iterator.string_handle() - - @function.Defun(tf.string) - def _fn(h): - remote_iterator = tf.data.Iterator.from_string_handle( - h, ds_iterator.output_types, ds_iterator.output_shapes) - input_list = remote_iterator.get_next() - reshaped_input_list = [ - tf.reshape(input_list[i], shape=model_input_shapes[i]) - for i in range(len(input_list)) - ] - return reshaped_input_list - - return _fn, [ds_iterator_string_handle] - - -class BaseImagePreprocessor(InputPreprocessor): - """Base class for all image model preprocessors.""" - - def __init__(self, - batch_size, - output_shapes, - num_splits, - dtype, - train, - distortions, - resize_method, - shift_ratio=-1, - summary_verbosity=0, - distort_color_in_yiq=True, - fuse_decode_and_crop=True, - match_mlperf=False): - super(BaseImagePreprocessor, self).__init__(batch_size, output_shapes) - image_shape = output_shapes[0] - # image_shape is in form (batch_size, height, width, depth) - self.height = image_shape[1] - self.width = image_shape[2] - self.depth = image_shape[3] - self.num_splits = num_splits - self.dtype = dtype - self.train = train - self.resize_method = resize_method - self.shift_ratio = shift_ratio - self.distortions = distortions - self.distort_color_in_yiq = distort_color_in_yiq - self.fuse_decode_and_crop = fuse_decode_and_crop - if self.batch_size % self.num_splits != 0: - raise ValueError( - ('batch_size must be a multiple of num_splits: ' - 'batch_size %d, num_splits: %d') % - (self.batch_size, self.num_splits)) - self.batch_size_per_split = self.batch_size // self.num_splits - self.summary_verbosity = summary_verbosity - self.match_mlperf = match_mlperf - - def parse_and_preprocess(self, value, batch_position): - assert self.supports_datasets() - image_buffer, label_index, bbox, _ = parse_example_proto(value) - if self.match_mlperf: - bbox = tf.zeros((1, 0, 4), dtype=bbox.dtype) - mlperf.logger.log(key=mlperf.tags.INPUT_CROP_USES_BBOXES, value=False) - else: - mlperf.logger.log(key=mlperf.tags.INPUT_CROP_USES_BBOXES, value=True) - image = self.preprocess(image_buffer, bbox, batch_position) - return (image, label_index) - - def preprocess(self, image_buffer, bbox, batch_position): - raise NotImplementedError('Must be implemented by subclass.') - - def create_dataset(self, - batch_size, - num_splits, - batch_size_per_split, - dataset, - subset, - train, - datasets_repeat_cached_sample, - num_threads=None, - datasets_use_caching=False, - datasets_parallel_interleave_cycle_length=None, - datasets_sloppy_parallel_interleave=False, - datasets_parallel_interleave_prefetch=None): - """Creates a dataset for the benchmark.""" - assert self.supports_datasets() - glob_pattern = dataset.tf_record_pattern(subset) - file_names = gfile.Glob(glob_pattern) - if not file_names: - raise ValueError('Found no files in --data_dir matching: {}' - .format(glob_pattern)) - ds = tf.data.TFRecordDataset.list_files(file_names, shuffle=train) - ds = ds.apply( - tf.data.experimental.parallel_interleave( - tf.data.TFRecordDataset, - cycle_length=datasets_parallel_interleave_cycle_length or 10, - sloppy=datasets_sloppy_parallel_interleave, - prefetch_input_elements=datasets_parallel_interleave_prefetch)) - if datasets_repeat_cached_sample: - # Repeat a single sample element indefinitely to emulate memory-speed IO. - ds = ds.take(1).cache().repeat() - counter = tf.data.Dataset.range(batch_size) - counter = counter.repeat() - ds = tf.data.Dataset.zip((ds, counter)) - ds = ds.prefetch(buffer_size=batch_size) - if datasets_use_caching: - ds = ds.cache() - if train: - buffer_size = 10000 - mlperf.logger.log(key=mlperf.tags.INPUT_SHARD, value=buffer_size) - ds = ds.apply( - tf.data.experimental.shuffle_and_repeat(buffer_size=buffer_size)) - else: - ds = ds.repeat() - ds = ds.apply( - tf.data.experimental.map_and_batch( - map_func=self.parse_and_preprocess, - batch_size=batch_size_per_split, - num_parallel_batches=num_splits)) - ds = ds.prefetch(buffer_size=num_splits) - if num_threads: - options = tf.data.Options() - options.experimental_threading.private_threadpool_size = num_threads - ds = ds.with_options(options) - return ds - - -class RecordInputImagePreprocessor(BaseImagePreprocessor): - """Preprocessor for images with RecordInput format.""" - - def preprocess(self, image_buffer, bbox, batch_position): - """Preprocessing image_buffer as a function of its batch position.""" - if self.train: - image = train_image(image_buffer, self.height, self.width, bbox, - batch_position, self.resize_method, self.distortions, - None, summary_verbosity=self.summary_verbosity, - distort_color_in_yiq=self.distort_color_in_yiq, - fuse_decode_and_crop=self.fuse_decode_and_crop) - else: - image = tf.image.decode_jpeg( - image_buffer, channels=3, dct_method='INTEGER_FAST') - image = eval_image(image, self.height, self.width, batch_position, - self.resize_method, - summary_verbosity=self.summary_verbosity) - # Note: image is now float32 [height,width,3] with range [0, 255] - - # image = tf.cast(image, tf.uint8) # HACK TESTING - - if self.match_mlperf: - mlperf.logger.log(key=mlperf.tags.INPUT_MEAN_SUBTRACTION, - value=_CHANNEL_MEANS) - normalized = image - _CHANNEL_MEANS - else: - normalized = normalized_image(image) - return tf.cast(normalized, self.dtype) - - def minibatch(self, - dataset, - subset, - params, - shift_ratio=-1): - if shift_ratio < 0: - shift_ratio = self.shift_ratio - with tf.name_scope('batch_processing'): - # Build final results per split. - images = [[] for _ in range(self.num_splits)] - labels = [[] for _ in range(self.num_splits)] - if params.use_datasets: - ds = self.create_dataset( - self.batch_size, self.num_splits, self.batch_size_per_split, - dataset, subset, self.train, - datasets_repeat_cached_sample=params.datasets_repeat_cached_sample, - num_threads=params.datasets_num_private_threads, - datasets_use_caching=params.datasets_use_caching, - datasets_parallel_interleave_cycle_length=( - params.datasets_parallel_interleave_cycle_length), - datasets_sloppy_parallel_interleave=( - params.datasets_sloppy_parallel_interleave), - datasets_parallel_interleave_prefetch=( - params.datasets_parallel_interleave_prefetch)) - ds_iterator = self.create_iterator(ds) - for d in xrange(self.num_splits): - images[d], labels[d] = ds_iterator.get_next() - - # TODO(laigd): consider removing the --use_datasets option, it should - # always use datasets. - else: - record_input = data_flow_ops.RecordInput( - file_pattern=dataset.tf_record_pattern(subset), - seed=301, - parallelism=64, - buffer_size=10000, - batch_size=self.batch_size, - shift_ratio=shift_ratio, - name='record_input') - records = record_input.get_yield_op() - records = tf.split(records, self.batch_size, 0) - records = [tf.reshape(record, []) for record in records] - for idx in xrange(self.batch_size): - value = records[idx] - (image, label) = self.parse_and_preprocess(value, idx) - split_index = idx % self.num_splits - labels[split_index].append(label) - images[split_index].append(image) - - for split_index in xrange(self.num_splits): - if not params.use_datasets: - images[split_index] = tf.parallel_stack(images[split_index]) - labels[split_index] = tf.concat(labels[split_index], 0) - images[split_index] = tf.reshape( - images[split_index], - shape=[self.batch_size_per_split, self.height, self.width, - self.depth]) - labels[split_index] = tf.reshape(labels[split_index], - [self.batch_size_per_split]) - return images, labels - - def supports_datasets(self): - return True - - -class ImagenetPreprocessor(RecordInputImagePreprocessor): - - def preprocess(self, image_buffer, bbox, batch_position): - # pylint: disable=g-import-not-at-top - try: - from official.r1.resnet.imagenet_preprocessing import preprocess_image - except ImportError: - tf.logging.fatal('Please include tensorflow/models to the PYTHONPATH.') - raise - if self.train: - image = preprocess_image( - image_buffer, bbox, self.height, self.width, self.depth, - is_training=True) - else: - image = preprocess_image( - image_buffer, bbox, self.height, self.width, self.depth, - is_training=False) - return tf.cast(image, self.dtype) - - -class Cifar10ImagePreprocessor(BaseImagePreprocessor): - """Preprocessor for Cifar10 input images.""" - - def _distort_image(self, image): - """Distort one image for training a network. - - Adopted the standard data augmentation scheme that is widely used for - this dataset: the images are first zero-padded with 4 pixels on each side, - then randomly cropped to again produce distorted images; half of the images - are then horizontally mirrored. - - Args: - image: input image. - Returns: - distorted image. - """ - image = tf.image.resize_image_with_crop_or_pad( - image, self.height + 8, self.width + 8) - distorted_image = tf.random_crop(image, - [self.height, self.width, self.depth]) - # Randomly flip the image horizontally. - distorted_image = tf.image.random_flip_left_right(distorted_image) - if self.summary_verbosity >= 3: - tf.summary.image('distorted_image', tf.expand_dims(distorted_image, 0)) - return distorted_image - - def _eval_image(self, image): - """Get the image for model evaluation.""" - distorted_image = tf.image.resize_image_with_crop_or_pad( - image, self.width, self.height) - if self.summary_verbosity >= 3: - tf.summary.image('cropped.image', tf.expand_dims(distorted_image, 0)) - return distorted_image - - def preprocess(self, raw_image): - """Preprocessing raw image.""" - if self.summary_verbosity >= 3: - tf.summary.image('raw.image', tf.expand_dims(raw_image, 0)) - if self.train and self.distortions: - image = self._distort_image(raw_image) - else: - image = self._eval_image(raw_image) - normalized = normalized_image(image) - return tf.cast(normalized, self.dtype) - - def minibatch(self, - dataset, - subset, - params, - shift_ratio=-1): - # TODO(jsimsa): Implement datasets code path - del shift_ratio, params - with tf.name_scope('batch_processing'): - all_images, all_labels = dataset.read_data_files(subset) - all_images = tf.constant(all_images) - all_labels = tf.constant(all_labels) - input_image, input_label = tf.train.slice_input_producer( - [all_images, all_labels]) - input_image = tf.cast(input_image, self.dtype) - input_label = tf.cast(input_label, tf.int32) - # Ensure that the random shuffling has good mixing properties. - min_fraction_of_examples_in_queue = 0.4 - min_queue_examples = int(dataset.num_examples_per_epoch(subset) * - min_fraction_of_examples_in_queue) - raw_images, raw_labels = tf.train.shuffle_batch( - [input_image, input_label], batch_size=self.batch_size, - capacity=min_queue_examples + 3 * self.batch_size, - min_after_dequeue=min_queue_examples) - - images = [[] for i in range(self.num_splits)] - labels = [[] for i in range(self.num_splits)] - - # Create a list of size batch_size, each containing one image of the - # batch. Without the unstack call, raw_images[i] would still access the - # same image via a strided_slice op, but would be slower. - raw_images = tf.unstack(raw_images, axis=0) - raw_labels = tf.unstack(raw_labels, axis=0) - for i in xrange(self.batch_size): - split_index = i % self.num_splits - # The raw image read from data has the format [depth, height, width] - # reshape to the format returned by minibatch. - raw_image = tf.reshape(raw_images[i], - [dataset.depth, dataset.height, dataset.width]) - raw_image = tf.transpose(raw_image, [1, 2, 0]) - image = self.preprocess(raw_image) - images[split_index].append(image) - - labels[split_index].append(raw_labels[i]) - - for split_index in xrange(self.num_splits): - images[split_index] = tf.parallel_stack(images[split_index]) - labels[split_index] = tf.parallel_stack(labels[split_index]) - return images, labels - - -class COCOPreprocessor(BaseImagePreprocessor): - """Preprocessor for COCO dataset input images, boxes, and labels.""" - - def minibatch(self, - dataset, - subset, - params, - shift_ratio=-1): - del shift_ratio # Not used when using datasets instead of data_flow_ops - with tf.name_scope('batch_processing'): - ds = self.create_dataset( - batch_size=self.batch_size, - num_splits=self.num_splits, - batch_size_per_split=self.batch_size_per_split, - dataset=dataset, - subset=subset, - train=self.train, - datasets_repeat_cached_sample=params.datasets_repeat_cached_sample, - num_threads=params.datasets_num_private_threads, - datasets_use_caching=params.datasets_use_caching, - datasets_parallel_interleave_cycle_length=( - params.datasets_parallel_interleave_cycle_length), - datasets_sloppy_parallel_interleave=( - params.datasets_sloppy_parallel_interleave), - datasets_parallel_interleave_prefetch=( - params.datasets_parallel_interleave_prefetch)) - ds_iterator = self.create_iterator(ds) - - # Training data: 4 tuple - # Validation data: 5 tuple - # See get_input_shapes in models/ssd_model.py for details. - input_len = 4 if subset == 'train' else 5 - input_lists = [[None for _ in range(self.num_splits)] - for _ in range(input_len)] - for d in xrange(self.num_splits): - input_list = ds_iterator.get_next() - for i in range(input_len): - input_lists[i][d] = input_list[i] - return input_lists - - def preprocess(self, data): - try: - import ssd_dataloader # pylint: disable=g-import-not-at-top - import ssd_constants # pylint: disable=g-import-not-at-top - from object_detection.core import preprocessor # pylint: disable=g-import-not-at-top - except ImportError: - raise ImportError('To use the COCO dataset, you must clone the ' - 'repo https://github.com/tensorflow/models and add ' - 'tensorflow/models and tensorflow/models/research to ' - 'the PYTHONPATH, and compile the protobufs by ' - 'following https://github.com/tensorflow/models/blob/' - 'master/research/object_detection/g3doc/installation.md' - '#protobuf-compilation') - image_buffer = data['image_buffer'] - boxes = data['groundtruth_boxes'] - classes = tf.reshape(data['groundtruth_classes'], [-1, 1]) - source_id = tf.string_to_number(data['source_id']) - raw_shape = data['raw_shape'] - - ssd_encoder = ssd_dataloader.Encoder() - - # Only 80 of the 90 COCO classes are used. - class_map = tf.convert_to_tensor(ssd_constants.CLASS_MAP) - classes = tf.gather(class_map, classes) - classes = tf.cast(classes, dtype=tf.float32) - - if self.train: - image, boxes, classes = ssd_dataloader.ssd_decode_and_crop( - image_buffer, boxes, classes, raw_shape) - # ssd_crop resizes and returns image of dtype float32 and does not change - # its range (i.e., value in between 0--255). Divide by 255. converts it - # to [0, 1] range. Not doing this before cropping to avoid dtype cast - # (which incurs additional memory copy). - image /= 255. - - image, boxes = preprocessor.random_horizontal_flip( - image=image, boxes=boxes) - # Random horizontal flip probability is 50% - # See https://github.com/tensorflow/models/blob/master/research/object_detection/core/preprocessor.py # pylint: disable=line-too-long - mlperf.logger.log(key=mlperf.tags.RANDOM_FLIP_PROBABILITY, value=0.5) - - image = tf.cast(image, self.dtype) - - encoded_returns = ssd_encoder.encode_labels(boxes, classes) - encoded_classes, encoded_boxes, num_matched_boxes = encoded_returns - - # Shape of image: [width, height, channel] - # Shape of encoded_boxes: [NUM_SSD_BOXES, 4] - # Shape of encoded_classes: [NUM_SSD_BOXES, 1] - # Shape of num_matched_boxes: [1] - return (image, encoded_boxes, encoded_classes, num_matched_boxes) - - else: - image = tf.image.decode_jpeg(image_buffer) - image = tf.image.resize_images( - image, size=(ssd_constants.IMAGE_SIZE, ssd_constants.IMAGE_SIZE)) - # resize_image returns image of dtype float32 and does not change its - # range. Divide by 255 to convert image to [0, 1] range. - image /= 255. - - image = ssd_dataloader.normalize_image(image) - image = tf.cast(image, self.dtype) - - def trim_and_pad(inp_tensor): - """Limit the number of boxes, and pad if necessary.""" - inp_tensor = inp_tensor[:ssd_constants.MAX_NUM_EVAL_BOXES] - num_pad = ssd_constants.MAX_NUM_EVAL_BOXES - tf.shape(inp_tensor)[0] - inp_tensor = tf.pad(inp_tensor, [[0, num_pad], [0, 0]]) - return tf.reshape(inp_tensor, [ssd_constants.MAX_NUM_EVAL_BOXES, - inp_tensor.get_shape()[1]]) - - boxes, classes = trim_and_pad(boxes), trim_and_pad(classes) - - # Shape of boxes: [MAX_NUM_EVAL_BOXES, 4] - # Shape of classes: [MAX_NUM_EVAL_BOXES, 1] - # Shape of source_id: [] (scalar tensor) - # Shape of raw_shape: [3] - return (image, boxes, classes, source_id, raw_shape) - - def create_dataset(self, - batch_size, - num_splits, - batch_size_per_split, - dataset, - subset, - train, - datasets_repeat_cached_sample, - num_threads=None, - datasets_use_caching=False, - datasets_parallel_interleave_cycle_length=None, - datasets_sloppy_parallel_interleave=False, - datasets_parallel_interleave_prefetch=None): - """Creates a dataset for the benchmark.""" - try: - import ssd_dataloader # pylint: disable=g-import-not-at-top - except ImportError: - raise ImportError('To use the COCO dataset, you must clone the ' - 'repo https://github.com/tensorflow/models and add ' - 'tensorflow/models and tensorflow/models/research to ' - 'the PYTHONPATH, and compile the protobufs by ' - 'following https://github.com/tensorflow/models/blob/' - 'master/research/object_detection/g3doc/installation.md' - '#protobuf-compilation') - assert self.supports_datasets() - - glob_pattern = dataset.tf_record_pattern(subset) - ds = tf.data.TFRecordDataset.list_files(glob_pattern, shuffle=train) - # TODO(haoyuzhang): Enable map+filter fusion after cl/218399112 in release - # options = tf.data.Options() - # options.experimental_optimization = tf.data.experimental.OptimizationOptions() # pylint: disable=line-too-long - # options.experimental_optimization.map_and_filter_fusion = True - # ds = ds.with_options(options) - - ds = ds.apply( - tf.data.experimental.parallel_interleave( - tf.data.TFRecordDataset, - cycle_length=datasets_parallel_interleave_cycle_length or 10, - sloppy=datasets_sloppy_parallel_interleave)) - mlperf.logger.log(key=mlperf.tags.INPUT_ORDER) - if datasets_repeat_cached_sample: - # Repeat a single sample element indefinitely to emulate memory-speed IO. - ds = ds.take(1).cache().repeat() - ds = ds.prefetch(buffer_size=batch_size) - if datasets_use_caching: - ds = ds.cache() - if train: - ds = ds.apply(tf.data.experimental.shuffle_and_repeat(buffer_size=10000)) - mlperf.logger.log(key=mlperf.tags.INPUT_SHARD, value=10000) - mlperf.logger.log(key=mlperf.tags.INPUT_ORDER) - else: - ds = ds.repeat() - - ds = ds.map(ssd_dataloader.ssd_parse_example_proto, num_parallel_calls=64) - ds = ds.filter( - lambda data: tf.greater(tf.shape(data['groundtruth_boxes'])[0], 0)) - ds = ds.apply( - tf.data.experimental.map_and_batch( - map_func=self.preprocess, - batch_size=batch_size_per_split, - num_parallel_batches=num_splits, - drop_remainder=train)) - ds = ds.prefetch(buffer_size=num_splits) - if num_threads: - options = tf.data.Options() - options.experimental_threading.private_threadpool_size = num_threads - ds = ds.with_options(options) - return ds - - def supports_datasets(self): - return True - - -class TestImagePreprocessor(BaseImagePreprocessor): - """Preprocessor used for testing. - - set_fake_data() sets which images and labels will be output by minibatch(), - and must be called before minibatch(). This allows tests to easily specify - a set of images to use for training, without having to create any files. - - Queue runners must be started for this preprocessor to work. - """ - - def __init__(self, - batch_size, - output_shapes, - num_splits, - dtype, - train=None, - distortions=None, - resize_method=None, - shift_ratio=0, - summary_verbosity=0, - distort_color_in_yiq=False, - fuse_decode_and_crop=False, - match_mlperf=False): - super(TestImagePreprocessor, self).__init__( - batch_size, output_shapes, num_splits, dtype, train, distortions, - resize_method, shift_ratio, summary_verbosity=summary_verbosity, - distort_color_in_yiq=distort_color_in_yiq, - fuse_decode_and_crop=fuse_decode_and_crop, match_mlperf=match_mlperf) - self.expected_subset = None - - def set_fake_data(self, fake_images, fake_labels): - assert len(fake_images.shape) == 4 - assert len(fake_labels.shape) == 1 - num_images = fake_images.shape[0] - assert num_images == fake_labels.shape[0] - assert num_images % self.batch_size == 0 - self.fake_images = fake_images - self.fake_labels = fake_labels - - def minibatch(self, - dataset, - subset, - params, - shift_ratio=0): - """Get test image batches.""" - del dataset, params - if (not hasattr(self, 'fake_images') or - not hasattr(self, 'fake_labels')): - raise ValueError('Must call set_fake_data() before calling minibatch ' - 'on TestImagePreprocessor') - if self.expected_subset is not None: - assert subset == self.expected_subset - - shift_ratio = shift_ratio or self.shift_ratio - fake_images = cnn_util.roll_numpy_batches(self.fake_images, self.batch_size, - shift_ratio) - fake_labels = cnn_util.roll_numpy_batches(self.fake_labels, self.batch_size, - shift_ratio) - - with tf.name_scope('batch_processing'): - image_slice, label_slice = tf.train.slice_input_producer( - [fake_images, fake_labels], - shuffle=False, - name='image_slice') - raw_images, raw_labels = tf.train.batch( - [image_slice, label_slice], batch_size=self.batch_size, - name='image_batch') - images = [[] for _ in range(self.num_splits)] - labels = [[] for _ in range(self.num_splits)] - for i in xrange(self.batch_size): - split_index = i % self.num_splits - raw_image = tf.cast(raw_images[i], self.dtype) - images[split_index].append(raw_image) - labels[split_index].append(raw_labels[i]) - for split_index in xrange(self.num_splits): - images[split_index] = tf.parallel_stack(images[split_index]) - labels[split_index] = tf.parallel_stack(labels[split_index]) - - normalized = [normalized_image(part) for part in images] - return [[tf.cast(part, self.dtype) for part in normalized], labels] - - -class LibrispeechPreprocessor(InputPreprocessor): - """Preprocessor for librispeech class for all image model preprocessors.""" - - def __init__(self, batch_size, output_shapes, num_splits, dtype, train, - **kwargs): - del kwargs - super(LibrispeechPreprocessor, self).__init__(batch_size, output_shapes) - self.num_splits = num_splits - self.dtype = dtype - self.is_train = train - if self.batch_size % self.num_splits != 0: - raise ValueError(('batch_size must be a multiple of num_splits: ' - 'batch_size %d, num_splits: %d') % (self.batch_size, - self.num_splits)) - self.batch_size_per_split = self.batch_size // self.num_splits - - def create_dataset(self, - batch_size, - num_splits, - batch_size_per_split, - dataset, - subset, - train, - datasets_repeat_cached_sample, - num_threads=None, - datasets_use_caching=False, - datasets_parallel_interleave_cycle_length=None, - datasets_sloppy_parallel_interleave=False, - datasets_parallel_interleave_prefetch=None): - """Creates a dataset for the benchmark.""" - # TODO(laigd): currently the only difference between this and the one in - # BaseImagePreprocessor is, this uses map() and padded_batch() while the - # latter uses tf.data.experimental.map_and_batch(). Try to merge them. - assert self.supports_datasets() - glob_pattern = dataset.tf_record_pattern(subset) - file_names = gfile.Glob(glob_pattern) - if not file_names: - raise ValueError('Found no files in --data_dir matching: {}' - .format(glob_pattern)) - ds = tf.data.TFRecordDataset.list_files(file_names, shuffle=train) - ds = ds.apply( - tf.data.experimental.parallel_interleave( - tf.data.TFRecordDataset, - cycle_length=datasets_parallel_interleave_cycle_length or 10, - sloppy=datasets_sloppy_parallel_interleave, - prefetch_input_elements=datasets_parallel_interleave_prefetch)) - if datasets_repeat_cached_sample: - # Repeat a single sample element indefinitely to emulate memory-speed IO. - ds = ds.take(1).cache().repeat() - counter = tf.data.Dataset.range(batch_size) - counter = counter.repeat() - ds = tf.data.Dataset.zip((ds, counter)) - ds = ds.prefetch(buffer_size=batch_size) - if datasets_use_caching: - ds = ds.cache() - if train: - ds = ds.apply(tf.data.experimental.shuffle_and_repeat(buffer_size=10000)) - else: - ds = ds.repeat() - ds = ds.map(map_func=self.parse_and_preprocess, - num_parallel_calls=batch_size_per_split*num_splits) - ds = ds.padded_batch( - batch_size=batch_size_per_split, - padded_shapes=tuple([ - tf.TensorShape(output_shape[1:]) - for output_shape in self.output_shapes - ]), - drop_remainder=True) - ds = ds.prefetch(buffer_size=num_splits) - if num_threads: - options = tf.data.Options() - options.experimental_threading.private_threadpool_size = num_threads - ds = ds.with_options(options) - return ds - - def minibatch(self, dataset, subset, params, shift_ratio=-1): - assert params.use_datasets - # TODO(laigd): unify this with CNNModel's minibatch() - # TODO(laigd): in distributed mode we use shift_ratio so different workers - # won't work on same inputs, so we should respect that. - del shift_ratio - with tf.name_scope('batch_processing'): - ds = self.create_dataset( - self.batch_size, - self.num_splits, - self.batch_size_per_split, - dataset, - subset, - self.is_train, - datasets_repeat_cached_sample=params.datasets_repeat_cached_sample, - num_threads=params.datasets_num_private_threads, - datasets_use_caching=params.datasets_use_caching, - datasets_parallel_interleave_cycle_length=( - params.datasets_parallel_interleave_cycle_length), - datasets_sloppy_parallel_interleave=( - params.datasets_sloppy_parallel_interleave), - datasets_parallel_interleave_prefetch=( - params.datasets_parallel_interleave_prefetch)) - ds_iterator = self.create_iterator(ds) - - # The four lists are: input spectrogram feature, labels, input lengths, - # label lengths - input_lists = [[None for _ in range(self.num_splits)] for _ in range(4)] - for d in xrange(self.num_splits): - input_list = ds_iterator.get_next() - for i in range(4): - input_lists[i][d] = input_list[i] - - assert self.output_shapes == [ - input_lists[i][0].shape.as_list() for i in range(4) - ] - return tuple(input_lists) - - def supports_datasets(self): - return True - - def parse_and_preprocess(self, value, batch_position): - """Parse an TFRecord.""" - del batch_position - assert self.supports_datasets() - context_features = { - 'labels': tf.VarLenFeature(dtype=tf.int64), - 'input_length': tf.FixedLenFeature([], dtype=tf.int64), - 'label_length': tf.FixedLenFeature([], dtype=tf.int64), - } - sequence_features = { - 'features': tf.FixedLenSequenceFeature([161], dtype=tf.float32) - } - context_parsed, sequence_parsed = tf.parse_single_sequence_example( - serialized=value, - context_features=context_features, - sequence_features=sequence_features, - ) - - return [ - # Input - tf.expand_dims(sequence_parsed['features'], axis=2), - # Label - tf.cast( - tf.reshape( - tf.sparse_tensor_to_dense(context_parsed['labels']), [-1]), - dtype=tf.int32), - # Input length - tf.cast( - tf.reshape(context_parsed['input_length'], [1]), - dtype=tf.int32), - # Label length - tf.cast( - tf.reshape(context_parsed['label_length'], [1]), - dtype=tf.int32), - ] diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/run.sh b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/run.sh deleted file mode 100644 index 7276d791ddc5eab57099c9cc7587f2b866c4a158..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/run.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -source /public/home/qianyj/virtualenv/dtk21.10.1/dtk21.10.1_tf1.15/venv/bin/activate -export ROCM_PATH=/public/home/qianyj/package/dtk-21.10.1/dtk-21.10.1 -export HIP_PATH=${ROCM_PATH}/hip -export CPACK_INSTLL_PREFIX=$ROCM_PATH -export AMDGPU_TARGETS="gfx900;gfx906" -export PATH=${ROCM_PATH}/bin:${ROCM_PATH}/llvm/bin:${ROCM_PATH}/hip/bin:$PATH -export LD_LIBRARY_PATH=${ROCM_PATH}/lib:${ROCM_PATH}/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${ROCM_PATH}/hip/lib:${ROCM_PATH}/llvm/lib:$LD_LIBRARY_PATH -export C_INCLUDE_PATH=${ROCM_PATH}/include:${ROCM_PATH}/llvm/include${C_INCLUDE_PATH:+:${C_INCLUDE_PATH}} -export CPLUS_INCLUDE_PATH=${ROCM_PATH}/include:${ROCM_PATH}/llvm/include${CPLUS_INCLUDE_PATH:+:${CPLUS_INCLUDE_PATH}} - -export HSA_FORCE_FINE_GRAIN_PCIE=1 -export MIOPEN_FIND_MODE=3 - -export TF_CPP_MIN_VLOG_LEVEL=2 - -HIP_VISIBLE_DEVICES=0,1,2,3 numactl --cpunodebind=0,1,2,3 --membind=0,1,2,3 nohup python3 tf_cnn_benchmarks.py --data_format=NCHW --batch_size=128 --model=resnet50 --save_model_steps=20000 --optimizer=momentum --variable_update=replicated --print_training_accuracy=true --eval_during_training_every_n_epochs=1 --nodistortions --num_gpus=4 --num_epochs=90 --weight_decay=1e-4 --data_dir=/public/software/apps/DeepLearning/Data/ImageNet-tensorflow/ --use_fp16=False --data_name=imagenet --train_dir=/public/home/qianyj/TF_test/dtk21.10.1/tf1.15/benchmarks-master/scripts/checkpoint >logfile 2>&1 & diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/run_tests.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/run_tests.py deleted file mode 100644 index 5b3dcd3276c776a1a585181229fae19e691106e3..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/run_tests.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Runs the tf_cnn_benchmarks tests.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import sys -import unittest - -from absl import app -from absl import flags as absl_flags -import tensorflow.compat.v1 as tf - -import all_reduce_benchmark_test -import allreduce_test -import benchmark_cnn_distributed_test -import benchmark_cnn_test -import cnn_util_test -import variable_mgr_util_test -from models import model_config - -# Ideally, we wouldn't need this option, and run both distributed tests and non- -# distributed tests. But, TensorFlow allocates all the GPU memory by default, so -# the non-distributed tests allocate all the GPU memory. The distributed tests -# spawn processes that run TensorFlow, and cannot run if all the GPU memory is -# already allocated. If a non-distributed test is run, then a distributed test -# is run in the same process, the distributed test will fail because there is no -# more GPU memory for the spawned processes to allocate. -absl_flags.DEFINE_boolean('run_distributed_tests', False, - 'If True, run the distributed tests. If False, the' - 'non-distributed tests.') - -absl_flags.DEFINE_boolean('full_tests', False, - 'If True, all distributed or non-distributed tests ' - 'are run, which can take hours. If False, only a ' - 'subset of tests will be run. This subset runs much ' - 'faster and tests almost all the functionality as ' - 'the full set of tests, so it is recommended to keep ' - 'this option set to False.') - -FLAGS = absl_flags.FLAGS - - -def main(_): - loader = unittest.defaultTestLoader - if FLAGS.full_tests: - suite = unittest.TestSuite([ - loader.loadTestsFromModule(allreduce_test), - loader.loadTestsFromModule(cnn_util_test), - loader.loadTestsFromModule(variable_mgr_util_test), - loader.loadTestsFromModule(benchmark_cnn_test), - loader.loadTestsFromModule(all_reduce_benchmark_test), - ]) - if model_config.can_import_contrib: - from models.tf1_only import nasnet_test # pylint: disable=g-import-not-at-top - suite.addTest(loader.loadTestsFromModule(nasnet_test)) - dist_suite = unittest.TestSuite([ - loader.loadTestsFromModule(benchmark_cnn_distributed_test), - ]) - else: - suite = unittest.TestSuite([ - loader.loadTestsFromModule(allreduce_test), - loader.loadTestsFromModule(cnn_util_test), - loader.loadTestsFromModule(all_reduce_benchmark_test), - loader.loadTestsFromModule(variable_mgr_util_test), - loader.loadTestsFromTestCase(benchmark_cnn_test.TestAlexnetModel), - loader.loadTestsFromTestCase(benchmark_cnn_test.TfCnnBenchmarksTest), - loader.loadTestsFromTestCase(benchmark_cnn_test.VariableUpdateTest), - loader.loadTestsFromTestCase( - benchmark_cnn_test.VariableMgrLocalReplicatedTest), - ]) - dist_suite = unittest.TestSuite([ - loader.loadTestsFromNames([ - 'benchmark_cnn_distributed_test.DistributedVariableUpdateTest' - '.testVarUpdateDefault', - - 'benchmark_cnn_distributed_test.TfCnnBenchmarksDistributedTest' - '.testParameterServer', - ]), - ]) - - if FLAGS.run_distributed_tests: - print('Running distributed tests') - result = unittest.TextTestRunner(verbosity=2).run(dist_suite) - else: - print('Running non-distributed tests') - result = unittest.TextTestRunner(verbosity=2).run(suite) - sys.exit(not result.wasSuccessful()) - - -if __name__ == '__main__': - tf.disable_v2_behavior() - app.run(main) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/ssd_constants.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/ssd_constants.py deleted file mode 100644 index 77fa0149b79f827b4e021afa67aa0e9409620e78..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/ssd_constants.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2018 Google. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Central location for all constants related to MLPerf SSD.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -# ============================================================================== -# == Model ===================================================================== -# ============================================================================== -IMAGE_SIZE = 300 - -# TODO(taylorrobie): MLPerf uses 80, but COCO documents 90. (RetinaNet uses 90) -# Update(taylorrobie): Labels > 81 show up in the pipeline. This will need to -# be resolved. -NUM_CLASSES = 81 # Including "no class". Not all COCO classes are used. - -# Note: Zero is special. (Background class) CLASS_INV_MAP[0] must be zero. -CLASS_INV_MAP = ( - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, - 22, 23, 24, 25, 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, - 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, - 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, - 88, 89, 90) -_MAP = {j: i for i, j in enumerate(CLASS_INV_MAP)} -CLASS_MAP = tuple(_MAP.get(i, -1) for i in range(max(CLASS_INV_MAP) + 1)) - -NUM_SSD_BOXES = 8732 - -RESNET_DEPTH = 34 - -"""SSD specific""" -MIN_LEVEL = 3 -MAX_LEVEL = 8 - -FEATURE_SIZES = (38, 19, 10, 5, 3, 1) -STEPS = (8, 16, 32, 64, 100, 300) - -# https://github.com/amdegroot/ssd.pytorch/blob/master/data/config.py -SCALES = (21, 45, 99, 153, 207, 261, 315) -ASPECT_RATIOS = ((2,), (2, 3), (2, 3), (2, 3), (2,), (2,)) -NUM_DEFAULTS = (4, 6, 6, 6, 4, 4) -NUM_DEFAULTS_BY_LEVEL = {3: 4, 4: 6, 5: 6, 6: 6, 7: 4, 8: 4} -SCALE_XY = 0.1 -SCALE_HW = 0.2 -BOX_CODER_SCALES = (1 / SCALE_XY, 1 / SCALE_XY, 1 / SCALE_HW, 1 / SCALE_HW) -MATCH_THRESHOLD = 0.5 - -# https://discuss.pytorch.org/t/how-to-preprocess-input-for-pre-trained-networks/683 -NORMALIZATION_MEAN = (0.485, 0.456, 0.406) -NORMALIZATION_STD = (0.229, 0.224, 0.225) - -# SSD Cropping -NUM_CROP_PASSES = 50 -CROP_MIN_IOU_CHOICES = (0, 0.1, 0.3, 0.5, 0.7, 0.9) -P_NO_CROP_PER_PASS = 1 / (len(CROP_MIN_IOU_CHOICES) + 1) - -# Hard example mining -NEGS_PER_POSITIVE = 3 - -# Batch normalization -BATCH_NORM_DECAY = 0.997 -BATCH_NORM_EPSILON = 1e-4 - - -# ============================================================================== -# == Optimizer ================================================================= -# ============================================================================== -LEARNING_RATE_SCHEDULE = ( - (0, 1e-3), - (160000, 1e-4), - (200000, 1e-5), -) -MOMENTUM = 0.9 -WEIGHT_DECAY = 5e-4 - - -# ============================================================================== -# == Keys ====================================================================== -# ============================================================================== -BOXES = "boxes" -CLASSES = "classes" -NUM_MATCHED_BOXES = "num_matched_boxes" -IMAGE = "image" -SOURCE_ID = "source_id" -RAW_SHAPE = "raw_shape" -PRED_BOXES = "pred_boxes" -PRED_SCORES = "pred_scores" - - -# ============================================================================== -# == Evaluation ================================================================ -# ============================================================================== - -# Note: This is based on a batch size of 32 -# https://github.com/mlperf/reference/blob/master/single_stage_detector/ssd/train.py#L21-L37 -CHECKPOINT_FREQUENCY = 20000 -MAX_NUM_EVAL_BOXES = 200 -OVERLAP_CRITERIA = 0.5 # Used for nonmax supression -MIN_SCORE = 0.05 # Minimum score to be considered during evaluation. -DUMMY_SCORE = -1e5 # If no boxes are matched. - -ANNOTATION_FILE = "annotations/instances_val2017.json" -COCO_NUM_TRAIN_IMAGES = 118287 -COCO_NUM_VAL_IMAGES = 4952 diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/ssd_dataloader.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/ssd_dataloader.py deleted file mode 100644 index 907d30903735d5181abbf18b02118a5eec2540ab..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/ssd_dataloader.py +++ /dev/null @@ -1,405 +0,0 @@ -# Copyright 2018 Google. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Data loader and processing.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import itertools as it -import math - -import numpy as np -import tensorflow.compat.v1 as tf - -from object_detection.box_coders import faster_rcnn_box_coder -from object_detection.core import box_list -from object_detection.core import region_similarity_calculator -from object_detection.core import target_assigner -from object_detection.matchers import argmax_matcher -import mlperf -import ssd_constants - - -class DefaultBoxes(object): - """Default bounding boxes for 300x300 5 layer SSD. - - Default bounding boxes generation follows the order of (W, H, anchor_sizes). - Therefore, the tensor converted from DefaultBoxes has a shape of - [anchor_sizes, H, W, 4]. The last dimension is the box coordinates; 'ltrb' - is [ymin, xmin, ymax, xmax] while 'xywh' is [cy, cx, h, w]. - """ - - def __init__(self): - fk = ssd_constants.IMAGE_SIZE / np.array(ssd_constants.STEPS) - - self.default_boxes = [] - # size of feature and number of feature - for idx, feature_size in enumerate(ssd_constants.FEATURE_SIZES): - sk1 = ssd_constants.SCALES[idx] / ssd_constants.IMAGE_SIZE - sk2 = ssd_constants.SCALES[idx+1] / ssd_constants.IMAGE_SIZE - sk3 = math.sqrt(sk1*sk2) - all_sizes = [(sk1, sk1), (sk3, sk3)] - - for alpha in ssd_constants.ASPECT_RATIOS[idx]: - w, h = sk1 * math.sqrt(alpha), sk1 / math.sqrt(alpha) - all_sizes.append((w, h)) - all_sizes.append((h, w)) - - assert len(all_sizes) == ssd_constants.NUM_DEFAULTS[idx] - - for w, h in all_sizes: - for i, j in it.product(range(feature_size), repeat=2): - cx, cy = (j + 0.5) / fk[idx], (i + 0.5) / fk[idx] - box = tuple(np.clip(k, 0, 1) for k in (cy, cx, h, w)) - self.default_boxes.append(box) - - assert len(self.default_boxes) == ssd_constants.NUM_SSD_BOXES - - mlperf.logger.log(key=mlperf.tags.FEATURE_SIZES, - value=ssd_constants.FEATURE_SIZES) - mlperf.logger.log(key=mlperf.tags.STEPS, - value=ssd_constants.STEPS) - mlperf.logger.log(key=mlperf.tags.SCALES, - value=ssd_constants.SCALES) - mlperf.logger.log(key=mlperf.tags.ASPECT_RATIOS, - value=ssd_constants.ASPECT_RATIOS) - mlperf.logger.log(key=mlperf.tags.NUM_DEFAULTS, - value=ssd_constants.NUM_SSD_BOXES) - - def to_ltrb(cy, cx, h, w): - return cy - h / 2, cx - w / 2, cy + h / 2, cx + w / 2 - - # For IoU calculation - self.default_boxes_ltrb = tuple(to_ltrb(*i) for i in self.default_boxes) - - def __call__(self, order='ltrb'): - if order == 'ltrb': return self.default_boxes_ltrb - if order == 'xywh': return self.default_boxes - - -def calc_iou_tensor(boxes1, boxes2): - """Calculation of IoU based on two boxes tensor. - - Reference to https://github.com/kuangliu/pytorch-ssd - - Args: - boxes1: shape (N, 4), four coordinates of N boxes - boxes2: shape (M, 4), four coordinates of M boxes - Returns: - IoU: shape (N, M), IoU of the i-th box in `boxes1` and j-th box in `boxes2` - """ - b1_left, b1_top, b1_right, b1_bottom = tf.split(boxes1, 4, axis=1) - b2_left, b2_top, b2_right, b2_bottom = tf.split(boxes2, 4, axis=1) - - # Shape of intersect_* (N, M) - intersect_left = tf.maximum(b1_left, tf.transpose(b2_left)) - intersect_top = tf.maximum(b1_top, tf.transpose(b2_top)) - intersect_right = tf.minimum(b1_right, tf.transpose(b2_right)) - intersect_bottom = tf.minimum(b1_bottom, tf.transpose(b2_bottom)) - - boxes1_area = (b1_right - b1_left) * (b1_bottom - b1_top) - boxes2_area = (b2_right - b2_left) * (b2_bottom - b2_top) - - intersect = tf.multiply(tf.maximum((intersect_right - intersect_left), 0), - tf.maximum((intersect_bottom - intersect_top), 0)) - union = boxes1_area + tf.transpose(boxes2_area) - intersect - iou = intersect / union - - return iou - - -def ssd_parse_example_proto(example_serialized): - """Parses an Example proto containing a training example of an image. - - Each Example proto contains the following fields that we care about: - - image/encoded: - image/source_id: tf.string - image/height: tf.int64 - image/width: tf.int64 - image/object/bbox/xmin: tf.VarLenFeature(tf.float32) - image/object/bbox/xmax: tf.VarLenFeature(tf.float32) - image/object/bbox/ymin: tf.VarLenFeature(tf.float32 - image/object/bbox/ymax: tf.VarLenFeature(tf.float32) - image/object/class/label: tf.VarLenFeature(tf.int64) - image/object/class/text: tf.VarLenFeature(tf.string) - - Complete decoder can be found in: - https://github.com/tensorflow/models/blob/master/research/object_detection/data_decoders/tf_example_decoder.py - - Args: - example_serialized: scalar Tensor tf.string containing a serialized - Example protocol buffer. - - Returns: - A dictionary with the following key-values: - image_buffer: Tensor tf.string containing the contents of a JPEG file. - groundtruth_boxes: Tensor tf.float32 of shape [num_boxes, 4], containing - coordinates of object bounding boxes. - groundtruth_classeS: Tensor tf.int64 of shape [num_boxes, 1], containing - class labels of objects. - source_id: unique image identifier. - raw_shape: [height, width, 3]. - """ - feature_map = { - 'image/encoded': tf.FixedLenFeature( - (), dtype=tf.string, default_value=''), - 'image/source_id': tf.FixedLenFeature((), tf.string, default_value=''), - 'image/height': tf.FixedLenFeature((), tf.int64, default_value=1), - 'image/width': tf.FixedLenFeature((), tf.int64, default_value=1), - 'image/object/bbox/xmin': tf.VarLenFeature(dtype=tf.float32), - 'image/object/bbox/ymin': tf.VarLenFeature(dtype=tf.float32), - 'image/object/bbox/xmax': tf.VarLenFeature(dtype=tf.float32), - 'image/object/bbox/ymax': tf.VarLenFeature(dtype=tf.float32), - 'image/object/class/label': tf.VarLenFeature(dtype=tf.int64), - } - features = tf.parse_single_example(example_serialized, feature_map) - - xmin = tf.expand_dims(features['image/object/bbox/xmin'].values, 1) - ymin = tf.expand_dims(features['image/object/bbox/ymin'].values, 1) - xmax = tf.expand_dims(features['image/object/bbox/xmax'].values, 1) - ymax = tf.expand_dims(features['image/object/bbox/ymax'].values, 1) - - image_buffer = features['image/encoded'] - # Bounding box coordinates should be in ltrb order - boxes = tf.concat([ymin, xmin, ymax, xmax], 1) - classes = tf.expand_dims(features['image/object/class/label'].values, 1) - source_id = features['image/source_id'] - raw_shape = tf.stack([features['image/height'], features['image/width'], 3]) - - return {'image_buffer': image_buffer, - 'groundtruth_boxes': boxes, - 'groundtruth_classes': classes, - 'source_id': source_id, - 'raw_shape': raw_shape} - - -def ssd_decode_and_crop(image_buffer, boxes, classes, raw_shape): - """Crop image randomly and decode the cropped region. - - This function will crop an image to meet the following requirements: - 1. height to width ratio between 0.5 and 2; - 2. IoUs of some boxes exceed specified threshold; - 3. At least one box center is in the cropped region. - We defer the jpeg decoding task until after the crop to avoid wasted work. - - Reference: https://github.com/chauhan-utk/ssd.DomainAdaptation - - Args: - image_buffer: Tensor tf.string containing the contents of a JPEG file. - boxes: Tensor tf.float32 of shape [num_boxes, 4], containing coordinates of - object bounding boxes. - classes: Tensor tf.int64 of shape [num_boxes, 1], containing class labels - of objects. - raw_shape: [height, width, 3]. - - Returns: - resized_image: decoded, cropped, and resized image Tensor tf.float32 of - shape [ssd_constants.IMAGE_SIZE, ssd_constants.IMAGE_SIZE, 3], value - range 0--255. - cropped_boxes: box coordinates for objects in the cropped region. - cropped_classes: class labels for objects in the cropped region. - """ - - num_boxes = tf.shape(boxes)[0] - - def no_crop_check(): - return (tf.random_uniform(shape=(), minval=0, maxval=1, dtype=tf.float32) - < ssd_constants.P_NO_CROP_PER_PASS) - - def no_crop_proposal(): - return ( - tf.ones((), tf.bool), - tf.convert_to_tensor([0, 0, 1, 1], dtype=tf.float32), - tf.ones((num_boxes,), tf.bool), - ) - - def crop_proposal(): - rand_vec = lambda minval, maxval: tf.random_uniform( - shape=(ssd_constants.NUM_CROP_PASSES, 1), minval=minval, maxval=maxval, - dtype=tf.float32) - - width, height = rand_vec(0.3, 1), rand_vec(0.3, 1) - left, top = rand_vec(0, 1-width), rand_vec(0, 1-height) - - right = left + width - bottom = top + height - - ltrb = tf.concat([left, top, right, bottom], axis=1) - - min_iou = tf.random_shuffle(ssd_constants.CROP_MIN_IOU_CHOICES)[0] - ious = calc_iou_tensor(ltrb, boxes) - - # discard any bboxes whose center not in the cropped image - xc, yc = [tf.tile(0.5 * (boxes[:, i + 0] + boxes[:, i + 2])[tf.newaxis, :], - (ssd_constants.NUM_CROP_PASSES, 1)) for i in range(2)] - - masks = tf.reduce_all(tf.stack([ - tf.greater(xc, tf.tile(left, (1, num_boxes))), - tf.less(xc, tf.tile(right, (1, num_boxes))), - tf.greater(yc, tf.tile(top, (1, num_boxes))), - tf.less(yc, tf.tile(bottom, (1, num_boxes))), - ], axis=2), axis=2) - - # Checks of whether a crop is valid. - valid_aspect = tf.logical_and(tf.less(height/width, 2), - tf.less(width/height, 2)) - valid_ious = tf.reduce_all(tf.greater(ious, min_iou), axis=1, keepdims=True) - valid_masks = tf.reduce_any(masks, axis=1, keepdims=True) - - valid_all = tf.cast(tf.reduce_all(tf.concat( - [valid_aspect, valid_ious, valid_masks], axis=1), axis=1), tf.int32) - - # One indexed, as zero is needed for the case of no matches. - index = tf.range(1, 1 + ssd_constants.NUM_CROP_PASSES, dtype=tf.int32) - - # Either one-hot, or zeros if there is no valid crop. - selection = tf.equal(tf.reduce_max(index * valid_all), index) - - use_crop = tf.reduce_any(selection) - output_ltrb = tf.reduce_sum(tf.multiply(ltrb, tf.tile(tf.cast( - selection, tf.float32)[:, tf.newaxis], (1, 4))), axis=0) - output_masks = tf.reduce_any(tf.logical_and(masks, tf.tile( - selection[:, tf.newaxis], (1, num_boxes))), axis=0) - - return use_crop, output_ltrb, output_masks - - def proposal(*args): - return tf.cond( - pred=no_crop_check(), - true_fn=no_crop_proposal, - false_fn=crop_proposal, - ) - - _, crop_bounds, box_masks = tf.while_loop( - cond=lambda x, *_: tf.logical_not(x), - body=proposal, - loop_vars=[tf.zeros((), tf.bool), tf.zeros((4,), tf.float32), tf.zeros((num_boxes,), tf.bool)], - ) - - filtered_boxes = tf.boolean_mask(boxes, box_masks, axis=0) - - mlperf.logger.log(key=mlperf.tags.NUM_CROPPING_ITERATIONS, - value=ssd_constants.NUM_CROP_PASSES) - - # Clip boxes to the cropped region. - filtered_boxes = tf.stack([ - tf.maximum(filtered_boxes[:, 0], crop_bounds[0]), - tf.maximum(filtered_boxes[:, 1], crop_bounds[1]), - tf.minimum(filtered_boxes[:, 2], crop_bounds[2]), - tf.minimum(filtered_boxes[:, 3], crop_bounds[3]), - ], axis=1) - - left = crop_bounds[0] - top = crop_bounds[1] - width = crop_bounds[2] - left - height = crop_bounds[3] - top - - cropped_boxes = tf.stack([ - (filtered_boxes[:, 0] - left) / width, - (filtered_boxes[:, 1] - top) / height, - (filtered_boxes[:, 2] - left) / width, - (filtered_boxes[:, 3] - top) / height, - ], axis=1) - - # crop_window containing integer coordinates of cropped region. A normalized - # coordinate value of y should be mapped to the image coordinate at - # y * (height - 1). - raw_shape = tf.cast(raw_shape, tf.float32) - crop_window = tf.stack([left * (raw_shape[0] - 1), - top * (raw_shape[1] - 1), - width * raw_shape[0], - height * raw_shape[1]]) - crop_window = tf.cast(crop_window, tf.int32) - - # Fused op only decodes the cropped portion of an image - cropped_image = tf.image.decode_and_crop_jpeg( - image_buffer, crop_window, channels=3) - - # Resize converts image dtype from uint8 to float32, without rescaling values. - resized_image = tf.image.resize_images( - cropped_image, [ssd_constants.IMAGE_SIZE, ssd_constants.IMAGE_SIZE]) - mlperf.logger.log(key=mlperf.tags.INPUT_SIZE, - value=ssd_constants.IMAGE_SIZE) - - cropped_classes = tf.boolean_mask(classes, box_masks, axis=0) - - return resized_image, cropped_boxes, cropped_classes - - -def color_jitter(image, brightness=0, contrast=0, saturation=0, hue=0): - """Distort the color of the image.""" - with tf.name_scope('distort_color'): - if brightness > 0: - image = tf.image.random_brightness(image, max_delta=brightness) - if contrast > 0: - image = tf.image.random_contrast( - image, lower=1-contrast, upper=1+contrast) - if saturation > 0: - image = tf.image.random_saturation( - image, lower=1-saturation, upper=1+saturation) - if hue > 0: - image = tf.image.random_hue(image, max_delta=hue) - return image - - -def normalize_image(images): - """Normalize image to zero mean and unit variance. - - Args: - images: a tensor representing images, at least 3-D. - Returns: - images normalized by mean and stdev. - """ - data_type = images.dtype - mean = tf.constant(ssd_constants.NORMALIZATION_MEAN, data_type) - std = tf.constant(ssd_constants.NORMALIZATION_STD, data_type) - images = tf.divide(tf.subtract(images, mean), std) - - mlperf.logger.log(key=mlperf.tags.DATA_NORMALIZATION_MEAN, - value=ssd_constants.NORMALIZATION_MEAN) - mlperf.logger.log(key=mlperf.tags.DATA_NORMALIZATION_STD, - value=ssd_constants.NORMALIZATION_STD) - return images - - -class Encoder(object): - """Encoder for SSD boxes and labels.""" - - def __init__(self): - similarity_calc = region_similarity_calculator.IouSimilarity() - matcher = argmax_matcher.ArgMaxMatcher( - matched_threshold=ssd_constants.MATCH_THRESHOLD, - unmatched_threshold=ssd_constants.MATCH_THRESHOLD, - negatives_lower_than_unmatched=True, - force_match_for_each_row=True) - - box_coder = faster_rcnn_box_coder.FasterRcnnBoxCoder( - scale_factors=ssd_constants.BOX_CODER_SCALES) - - self.default_boxes = DefaultBoxes()('ltrb') - self.default_boxes = box_list.BoxList( - tf.convert_to_tensor(self.default_boxes)) - self.assigner = target_assigner.TargetAssigner( - similarity_calc, matcher, box_coder) - - def encode_labels(self, gt_boxes, gt_labels): - target_boxes = box_list.BoxList(gt_boxes) - encoded_classes, _, encoded_boxes, _, matches = self.assigner.assign( - self.default_boxes, target_boxes, gt_labels) - num_matched_boxes = tf.reduce_sum( - tf.cast(tf.not_equal(matches, -1), tf.float32)) - return encoded_classes, encoded_boxes, num_matched_boxes diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00000-of-00008 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00000-of-00008 deleted file mode 100644 index 4e65b92a9a5f252f7b1a9d9048e834217f468971..0000000000000000000000000000000000000000 Binary files a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00000-of-00008 and /dev/null differ diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00001-of-00008 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00001-of-00008 deleted file mode 100644 index 1cf1fec734f3d6bfd74a6e38ac7b0f43d24eaaab..0000000000000000000000000000000000000000 Binary files a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00001-of-00008 and /dev/null differ diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00002-of-00008 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00002-of-00008 deleted file mode 100644 index 631ca95c9e17362c498b71979466661ec7ce4be5..0000000000000000000000000000000000000000 Binary files a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00002-of-00008 and /dev/null differ diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00003-of-00008 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00003-of-00008 deleted file mode 100644 index addfd3424c00e4596de3bfa77751c1fd891164ba..0000000000000000000000000000000000000000 Binary files a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00003-of-00008 and /dev/null differ diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00004-of-00008 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00004-of-00008 deleted file mode 100644 index 32818ec981b1b648ea605e351012c4e58a075454..0000000000000000000000000000000000000000 Binary files a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00004-of-00008 and /dev/null differ diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00005-of-00008 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00005-of-00008 deleted file mode 100644 index 5f4e651519673b3b61726b5a3b0d21a8c962deb5..0000000000000000000000000000000000000000 Binary files a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00005-of-00008 and /dev/null differ diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00006-of-00008 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00006-of-00008 deleted file mode 100644 index 042a71fd169745357111f2f0de84f42e52849b2a..0000000000000000000000000000000000000000 Binary files a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00006-of-00008 and /dev/null differ diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00007-of-00008 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00007-of-00008 deleted file mode 100644 index a6d9ce627d88ec39344fbd6aae7badd629c5e54c..0000000000000000000000000000000000000000 Binary files a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/train-00007-of-00008 and /dev/null differ diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/validation-00000-of-00002 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/validation-00000-of-00002 deleted file mode 100644 index 1c7757759bad5f59007b429adb520fdb5eed4068..0000000000000000000000000000000000000000 Binary files a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/validation-00000-of-00002 and /dev/null differ diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/validation-00001-of-00002 b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/validation-00001-of-00002 deleted file mode 100644 index e0f379cab43b5fa46f6f232e93c1deba2548f7a1..0000000000000000000000000000000000000000 Binary files a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/fake_tf_record_data/validation-00001-of-00002 and /dev/null differ diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/images/black_image.jpg b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/images/black_image.jpg deleted file mode 100644 index 312873be3bd305bfb5962896ea8ae507ca44b572..0000000000000000000000000000000000000000 Binary files a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/images/black_image.jpg and /dev/null differ diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/images/white_image.jpg b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/images/white_image.jpg deleted file mode 100644 index ad96f25af79ca0d683642c3dbef1049cc7061f84..0000000000000000000000000000000000000000 Binary files a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/images/white_image.jpg and /dev/null differ diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/tfrecord_image_generator.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/tfrecord_image_generator.py deleted file mode 100644 index 8f0b9102134456fefd7b712c9e1d734c13a0b9e2..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_data/tfrecord_image_generator.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Generate black and white test TFRecords with Example protos. - -Each record within the TFRecord file is a -serialized Example proto. The Example proto contains the following fields: - - image/encoded: string containing JPEG encoded image in RGB colorspace - image/height: integer, image height in pixels - image/width: integer, image width in pixels - image/colorspace: string, specifying the colorspace, always 'RGB' - image/channels: integer, specifying the number of channels, always 3 - image/format: string, specifying the format, always'JPEG' - - image/filename: string containing the basename of the image file - e.g. 'n01440764_10026.JPEG' or 'ILSVRC2012_val_00000293.JPEG' - image/class/label: integer specifying the index in a classification layer. - The label ranges from [1, 1000] where 0 is not used. - image/class/synset: string specifying the unique ID of the label, - e.g. 'n01440764' - image/class/text: string specifying the human-readable version of the label - e.g. 'red fox, Vulpes vulpes' - - image/object/bbox/xmin: list of integers specifying the 0+ human annotated - bounding boxes - image/object/bbox/xmax: list of integers specifying the 0+ human annotated - bounding boxes - image/object/bbox/ymin: list of integers specifying the 0+ human annotated - bounding boxes - image/object/bbox/ymax: list of integers specifying the 0+ human annotated - bounding boxes - image/object/bbox/label: integer specifying the index in a classification - layer. The label ranges from [1, 1000] where 0 is not used. Note this is - always identical to the image label. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import random - -import numpy as np -import six -import tensorflow.compat.v1 as tf - - -def _int64_feature(value): - """Wrapper for inserting int64 features into Example proto.""" - if not isinstance(value, list): - value = [value] - return tf.train.Feature(int64_list=tf.train.Int64List(value=value)) - - -def _float_feature(value): - """Wrapper for inserting float features into Example proto.""" - if not isinstance(value, list): - value = [value] - return tf.train.Feature(float_list=tf.train.FloatList(value=value)) - - -def _bytes_feature(value): - """Wrapper for inserting bytes features into Example proto.""" - return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) - - -def _convert_to_example(filename, image_buffer, label, synset, human, bbox, - height, width): - """Build an Example proto for an example. - - Args: - filename: string, path to an image file, e.g., '/path/to/example.JPG' - image_buffer: bytes, JPEG encoding of RGB image - label: integer, identifier for the ground truth for the network - synset: string, unique WordNet ID specifying the label, e.g., 'n02323233' - human: string, human-readable label, e.g., 'red fox, Vulpes vulpes' - bbox: list of bounding boxes; each box is a list of integers - specifying [xmin, ymin, xmax, ymax]. All boxes are assumed to belong to - the same label as the image label. - height: integer, image height in pixels - width: integer, image width in pixels - Returns: - Example proto - """ - xmin = [] - ymin = [] - xmax = [] - ymax = [] - for b in bbox: - assert len(b) == 4 - # pylint: disable=expression-not-assigned - [l.append(point) for l, point in zip([xmin, ymin, xmax, ymax], b)] - # pylint: enable=expression-not-assigned - - colorspace = b'RGB' - channels = 3 - image_format = b'JPEG' - - example = tf.train.Example(features=tf.train.Features(feature={ - 'image/height': _int64_feature(height), - 'image/width': _int64_feature(width), - 'image/colorspace': _bytes_feature(colorspace), - 'image/channels': _int64_feature(channels), - 'image/class/label': _int64_feature(label), - 'image/class/synset': _bytes_feature(six.ensure_binary(synset)), - 'image/class/text': _bytes_feature(six.ensure_binary(human)), - 'image/object/bbox/xmin': _float_feature(xmin), - 'image/object/bbox/xmax': _float_feature(xmax), - 'image/object/bbox/ymin': _float_feature(ymin), - 'image/object/bbox/ymax': _float_feature(ymax), - 'image/object/bbox/label': _int64_feature([label] * len(xmin)), - 'image/format': _bytes_feature(image_format), - 'image/filename': _bytes_feature(os.path.basename(six.ensure_binary( - filename))), - 'image/encoded': _bytes_feature(image_buffer)})) - return example - - -class ImageCoder(object): - """Helper class that provides TensorFlow image coding utilities.""" - - def __init__(self): - # Create a single Session to run all image coding calls. - self._sess = tf.Session() - - # Initializes function that converts PNG to JPEG data. - self._image = tf.placeholder(dtype=tf.uint8) - self._encode_jpeg = tf.image.encode_jpeg( - self._image, format='rgb', quality=100) - - def encode_jpeg(self, image): - jpeg_image = self._sess.run(self._encode_jpeg, - feed_dict={self._image: image}) - return jpeg_image - - -def _process_image(coder, name): - """Process a single image file. - - If name is "train", a black image is returned. Otherwise, a white image is - returned. - - Args: - coder: instance of ImageCoder to provide TensorFlow image coding utils. - name: string, unique identifier specifying the data set. - Returns: - image_buffer: bytes, JPEG encoding of RGB image. - height: integer, image height in pixels. - width: integer, image width in pixels. - """ - # Read the image file. - value = 0 if name == 'train' else 255 - height = random.randint(30, 299) - width = random.randint(30, 299) - image = np.full((height, width, 3), value, np.uint8) - - jpeg_data = coder.encode_jpeg(image) - - return jpeg_data, height, width - - -def _process_dataset(output_directory, num_classes, coder, name, num_images, - num_shards): - """Process a complete data set and save it as a TFRecord. - - Args: - output_directory: Where to put outputs. - num_classes: number of classes. - coder: Instance of an ImageCoder. - name: string, unique identifier specifying the data set. - num_images: number of images to generate. - num_shards: integer number of shards to create. - """ - files_per_shard = num_images // num_shards - for shard in range(num_shards): - output_filename = '%s-%.5d-of-%.5d' % (name, shard, num_shards) - output_file = os.path.join(output_directory, output_filename) - with tf.python_io.TFRecordWriter(output_file) as writer: - for i in range(files_per_shard): - index = shard * files_per_shard + i - image_buffer, height, width = _process_image(coder, name) - - filename = '{}_{}_{}'.format(name, shard, i) - label = index % num_classes - synset = str(index) - human = name - bbox = [[0.1, 0.1, 0.9, 0.9]] - example = _convert_to_example(filename, image_buffer, label, - synset, human, bbox, - height, width) - writer.write(example.SerializeToString()) - - -def write_black_and_white_tfrecord_data( - output_directory, num_classes, num_train_images=512, - num_validation_images=128, train_shards=8, validation_shards=2): - """Writes black and white images in tfrecord format. - - Training images are black and validation images are white. - - Args: - output_directory: Where to put outputs. - num_classes: number of classes. - num_train_images: number of training images to generate. - num_validation_images: number of validation images to generate. - train_shards: integer number of training shards to create. - validation_shards: integer number of validation shards to create. - """ - - coder = ImageCoder() - _process_dataset(output_directory, num_classes, coder, 'validation', - num_validation_images, validation_shards) - _process_dataset(output_directory, num_classes, coder, 'train', - num_train_images, train_shards) diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_util.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_util.py deleted file mode 100644 index ccb930a6b1e2fba3285dc2e14cfd0a3fba85ce4b..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/test_util.py +++ /dev/null @@ -1,532 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Shared functionality across multiple test files.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from collections import namedtuple -from contextlib import contextmanager -import os - -import numpy as np -import tensorflow.compat.v1 as tf -import benchmark_cnn -import cnn_util -import datasets -import preprocessing -from models import model -from platforms import util as platforms_util -from test_data import tfrecord_image_generator -from tensorflow.core.protobuf import rewriter_config_pb2 # pylint: disable=g-direct-tensorflow-import -from tensorflow.python.platform import test - - -@contextmanager -def monkey_patch(obj, **kwargs): - """Context mgr to monkey patch attributes on an object (such as a module). - - The attributes are patched back to their original value when the context - manager exits. - - For example, to replace benchmark_cnn.get_data_type with an identity function, - do: - - ``` - with monkey_patch(benchmark_cnn, get_data_type=lambda x: x) - loss1 = benchmark_cnn.loss_function(1) # loss1 will be 1 - loss2 = benchmark_cnn.loss_function(params) # Call the original function - ``` - - Args: - obj: The object (which can be a module) to monkey patch attributes on. - **kwargs: Dictionary mapping from attribute name to value that the attribute - will be patched with. - Yields: - Nothing. - """ - old_values = {key: getattr(obj, key) for key in kwargs} - try: - for key, value in kwargs.items(): - setattr(obj, key, value) - yield - finally: - for key, value in old_values.items(): - setattr(obj, key, value) - - -def monkey_patch_base_cluster_manager(): - """Monkey patches get_cluster_manager to return a BaseClusterManager. - - This function replaces platforms_util.get_cluster_manager with a function that - always return a BaseClusterManager. - - This is useful for testing creating a graph in distributed mode, with only a - single process. GrpcClusterManager's constructor blocks until a cluster is set - up, which requires multiple processes to be created. - """ - def get_test_cluster_manager(params, config_proto): - del config_proto - return cnn_util.BaseClusterManager(params) - platforms_util.get_cluster_manager = get_test_cluster_manager - - -def print_and_add_to_list(print_list): - """Returns a function which prints the input, then adds it to print_list.""" - def f(string): - print(string) - print_list.append(string) - return f - - -TrainingOutput = namedtuple('TrainingOutput', - ['loss', 'top_1_accuracy', 'top_5_accuracy']) - - -EvalOutput = namedtuple('EvalOutput', ['top_1_accuracy', 'top_5_accuracy']) - - -def get_training_outputs_from_logs(logs, print_training_accuracy): - """Returns a list of TrainingOutputs by parsing the logs of a training run. - - Args: - logs: A list of strings, each which is a line from the standard output of - tf_cnn_benchmarks from training. Only lines in the form: - 10 images/sec: 14.2 +/- 0.0 (jitter = 0.0) 7.020 - are parsed (the line may also contain the training accuracies). - print_training_accuracy: The value of the param print_training_accuracy. - Returns: - A list of TrainingOutputs. The list has one element per element of logs - that is in the format above. top_1_accuracy and top_5_accuracy are set to -1 - if the line does not contain accuracies. - """ - outputs = [] - for log in logs: - if 'images/sec' in log and '+/-' in log: - parts = log.split() - if print_training_accuracy: - # Example log with training accuracy: - # 10 images/sec: 0.2 +/- 0.0 (jitter = 0.0) 6.908 0.500 1.000 - assert len(parts) == 11 - top_1_acc = float(parts[9]) - top_5_acc = float(parts[10]) - else: - # Example log without training accuracy: - # 10 images/sec: 0.2 +/- 0.0 (jitter = 0.0) 6.908 - assert len(parts) == 9 - top_1_acc = -1 - top_5_acc = -1 - loss = float(parts[8]) - outputs.append(TrainingOutput(loss=loss, top_1_accuracy=top_1_acc, - top_5_accuracy=top_5_acc)) - assert len(outputs) >= 1 - return outputs - - -def get_evaluation_outputs_from_logs(logs): - """Returns the top 1 and 5 accuracies by parsing the logs of an eval run. - - Args: - logs: A list of strings, each which is a line from the standard output of - tf_cnn_benchmarks from evaluation. Only lines in the form: - Accuracy @ 1 = 0.5000 Accuracy @ 5 = 1.0000 [80 examples] - is parsed. - Returns: - A list of EvalOutputs. Normally this list only has one EvalOutput, but can - contain multiple if training is done and - --eval_during_training_every_n_steps is specified. - """ - eval_outputs = [] - for log in logs: - if 'Accuracy @ ' in log: - # Example log: - # Accuracy @ 1 = 0.5000 Accuracy @ 5 = 1.0000 [80 examples] - parts = log.split() - assert len(parts) == 12 - top_1_accuracy = float(parts[4]) - top_5_accuracy = float(parts[9]) - eval_outputs.append(EvalOutput(top_1_accuracy, top_5_accuracy)) - assert eval_outputs - return eval_outputs - - -def check_training_outputs_are_reasonable(testcase, training_outputs, - print_training_accuracy, - max_final_loss=10., - previous_final_loss=None): - """Checks the outputs from training a model are reasonable. - - An assert is failed if the outputs are not reasonable. The final top-1 and - top-5 accuracies are asserted to be 1, and so the dataset used to train should - be trivial to learn. For example, the dataset could consist of a black image - with label 0 and a white image with label 1. - - Args: - testcase: A tf.test.TestCase used for assertions. - training_outputs: A list of TrainingOutputs, as returned from - get_training_outputs_from_logs(). - print_training_accuracy: Whether training accuracies were printed and stored - in training_outputs. - max_final_loss: The loss of the final training output is asserted to be at - most this value. - previous_final_loss: If training was resumed from a checkpoint, the loss of - the final step from the previous training run that saved the checkpoint. - """ - if previous_final_loss is not None: - # Ensure the loss hasn't raised significantly from the final loss of the - # previous training run. - testcase.assertLessEqual(training_outputs[0].loss, - previous_final_loss * 1.01) - for output in training_outputs: - testcase.assertLessEqual(output.loss, 100.) - last_output = training_outputs[-1] - if print_training_accuracy: - testcase.assertEqual(last_output.top_1_accuracy, 1.0) - testcase.assertEqual(last_output.top_5_accuracy, 1.0) - if max_final_loss is not None: - testcase.assertLessEqual(last_output.loss, max_final_loss) - - -def train_and_eval(testcase, - run_fn, - params, - check_output_values, - max_final_loss=10., - skip=None): - """Trains a model then evaluates it. - - This function should be used to verify training and evaluating - BenchmarkCNN works without crashing and that it outputs reasonable - values. BenchmarkCNN will be run three times. First, it will train a - model from scratch, saving a checkpoint. Second, it will load the checkpoint - to continue training. Finally, it evaluates based on the loaded checkpoint. - - Args: - testcase: A tf.test.TestCase used for assertions. - run_fn: Must run `BenchmarkCNN` exactly once. BenchmarkCNN is - never used directly, but instead is only run through `run_fn`. `run_fn` - has the signature (run_type, inner_params) -> output_list, where: - * run_type is a string indicating how BenchmarkCNN will be run. - Either 'InitialTraining', 'TrainingFromCheckpoint' or 'Evaluation'. - * inner_params is the params BenchmarkCNN should be run with. - * output_list[i] is a list of lines from the ith worker's stdout. - params: The params BenchmarkCNN will be run with. - Will be passed to `run_fn` slightly modified in order to run with both - training and evaluation. - check_output_values: Whether the outputs of the workers, such as training - accuracy, should be checked to make sure their values are reasonable. - Fails an assert on `testcase` if a check fails. - max_final_loss: The loss of the final training output is asserted to be at - most this value for both training runs. - skip: If 'eval', evaluation is not done. if - 'eval_and_train_from_checkpoint', evaluation and training from a - checkpoint are both not done. - """ - - assert not skip or skip in {'eval', 'eval_and_train_from_checkpoint'} - - # Part 1: Train from scratch. - tf.logging.info('Training model from scratch') - print_training_accuracy = (params.print_training_accuracy or - params.forward_only) - initial_train_logs = run_fn('InitialTraining', params) - testcase.assertGreaterEqual(len(initial_train_logs), 1) - for lines in initial_train_logs: - initial_train_outputs = get_training_outputs_from_logs( - lines, print_training_accuracy) - if params.cross_replica_sync and params.batch_group_size == 1: - testcase.assertEqual(len(initial_train_outputs), params.num_batches) - if check_output_values: - check_training_outputs_are_reasonable(testcase, initial_train_outputs, - print_training_accuracy, - max_final_loss=max_final_loss) - if params.train_dir is not None: - train_dir_entries = set(os.listdir(params.train_dir)) - testcase.assertGreater(len(train_dir_entries), 0) - else: - train_dir_entries = None - - if skip == 'eval_and_train_from_checkpoint': - return - - # Part 2: Train from the loaded checkpoint. - testcase.assertIsNotNone(train_dir_entries) - tf.logging.info('Training model from loaded checkpoint') - # Run for same number of batches as before. - params = params._replace(num_batches=params.num_batches * 2) - train_logs_from_ckpt = run_fn('TrainingFromCheckpoint', params) - testcase.assertGreaterEqual(len(train_logs_from_ckpt), 1) - for lines in train_logs_from_ckpt: - train_outputs_from_ckpt = get_training_outputs_from_logs( - lines, print_training_accuracy) - if params.cross_replica_sync and params.batch_group_size == 1: - testcase.assertEqual(len(train_outputs_from_ckpt), - params.num_batches // 2 - params.num_warmup_batches) - if check_output_values: - check_training_outputs_are_reasonable( - testcase, train_outputs_from_ckpt, print_training_accuracy, - max_final_loss=max_final_loss, - previous_final_loss=initial_train_outputs[-1].loss) - # Ensure a new checkpoint was written out. - testcase.assertNotEqual(train_dir_entries, set(os.listdir(params.train_dir))) - - if skip == 'eval': - return - - # Part 3: Evaluate from the loaded checkpoint. - tf.logging.info('Evaluating model from checkpoint') - params = params._replace(num_batches=params.num_batches // 2, eval=True) - eval_logs = run_fn('Evaluation', params) - testcase.assertGreaterEqual(len(eval_logs), 1) - for lines in eval_logs: - eval_outputs = get_evaluation_outputs_from_logs(lines) - assert len(eval_outputs) == 1 - top_1_accuracy, top_5_accuracy = eval_outputs[0] - if check_output_values: - testcase.assertEqual(top_1_accuracy, 1.0) - testcase.assertEqual(top_5_accuracy, 1.0) - - -def get_temp_dir(dir_name): - dir_path = os.path.join(test.get_temp_dir(), dir_name) - os.mkdir(dir_path) - return dir_path - - -def create_black_and_white_images(): - dir_path = get_temp_dir('black_and_white_images') - tfrecord_image_generator.write_black_and_white_tfrecord_data(dir_path, - num_classes=1) - return dir_path - - -def get_params(train_dir_name): - """Returns params that can be used to train.""" - params = benchmark_cnn.make_params( - batch_size=2, - display_every=1, - init_learning_rate=0.005, - model='trivial', - num_batches=20, - num_gpus=2, - num_warmup_batches=5, - optimizer='sgd', - print_training_accuracy=True, - train_dir=get_temp_dir(train_dir_name), - variable_update='parameter_server', - weight_decay=0, - distortions=True, - distort_color_in_yiq=False) - return benchmark_cnn.set_default_param_values_and_env_vars(params) - - -def get_var_update_params(): - """Returns params that are used when testing variable updates.""" - params = benchmark_cnn.make_params( - batch_size=2, - model='test_model', - num_gpus=2, - display_every=1, - num_warmup_batches=0, - num_batches=4, - weight_decay=2 ** -4, - init_learning_rate=2 ** -4, - optimizer='sgd') - return benchmark_cnn.set_default_param_values_and_env_vars(params) - - -def get_fake_var_update_inputs(): - """Returns fake input 1x1 images to use in variable update tests.""" - # BenchmarkCNN divides by 127.5 then subtracts 1.0 from the images, so after - # that, the images will be -1., 0., 1., ..., 14. - return np.resize(127.5 * np.array(range(16)), (16, 1, 1, 1)) - - -def _worker_batches_in_numpy_array(numpy_inputs, batch_size, shift_ratio): - """Yields batches from a numpy array, for a single worker.""" - numpy_inputs = cnn_util.roll_numpy_batches(numpy_inputs, batch_size, - shift_ratio) - i = 0 - total_batches = numpy_inputs.shape[0] - assert total_batches % batch_size == 0 - while True: - yield numpy_inputs[i:i + batch_size, ...] - i = (i + batch_size) % total_batches - - -def manually_compute_losses(numpy_inputs, inputs_placeholder, loss, num_workers, - params): - """Manually compute the losses each worker should report in tf_cnn_benchmarks. - - This function essentially simulates tf_cnn_benchmarks, computing what the loss - of each worker should be. The caller should create a model, that takes in - images from `inputs_placeholder`, a tf.placeholder, and computes `loss`. - - This function, and all ops passed to this function, must be run under a - tf.device('cpu:0') context manager. - - Non-SGD optimizers are not supported with multiple workers. - - Args: - numpy_inputs: A Numpy array to use as the input images. - inputs_placeholder: A tf.placeholder tensor, where input images can be fed - into. - loss: A scalar tensor representing the loss of the model, which is obtained - from the input images in inputs_placeholder. - num_workers: How many workers should be simulated. - params: Params tuple. This doesn't have to have information about the - distributed cluster, such as --num_workers, as num_workers is passed in - separately. - - Returns: - A list of list of losses. return_value[i][j] is the loss of the ith worker - after the jth step. - """ - batch_size = params.batch_size * params.num_gpus - assert numpy_inputs.shape[0] % (num_workers * batch_size) == 0 - l2_loss = tf.add_n([tf.nn.l2_loss(x) for x in tf.trainable_variables()]) - total_loss = loss + params.weight_decay * l2_loss - reported_loss = (loss if params.loss_type_to_report == 'base_loss' - else total_loss) - gradient_multiplier = 1 - if params.variable_update in ('replicated', 'distributed_all_reduce'): - # In certain variable updates, tf_cnn_benchmarks add the gradients of the - # GPUs instead of taking their mean, making the gradients effectively - # params.num_gpu times higher. - # TODO(b/62722498): Make all variable updates consistent. - gradient_multiplier = params.num_gpus - - opt = benchmark_cnn.get_optimizer(params, params.init_learning_rate) - grad_vars = opt.compute_gradients( - total_loss, grad_loss=tf.constant(gradient_multiplier, dtype=tf.float32)) - grads = [g for g, _ in grad_vars] - # We apply gradients from a placeholder. That way, we can first compute the - # gradients from each worker, then afterwards apply them one by one by feeding - # them into the placeholder. - placeholder_grad_vars = [(tf.placeholder(g.dtype, g.shape), v) - for g, v in grad_vars] - placeholder_grads = [g for g, _ in placeholder_grad_vars] - apply_grads_op = opt.apply_gradients(placeholder_grad_vars) - - batch_iterators = [_worker_batches_in_numpy_array(numpy_inputs, batch_size, - shift_ratio=i / num_workers) - for i in range(num_workers)] - # Set the GPU count to 0, to avoid taking all the GPU memory. Unfortunately, - # doing so still takes up about ~1GB for some reason. - config = tf.ConfigProto(device_count={'GPU': 0}) - config.graph_options.rewrite_options.pin_to_host_optimization = ( - rewriter_config_pb2.RewriterConfig.OFF) - with tf.Session(config=config) as sess: - sess.run(tf.global_variables_initializer()) - losses = [[] for _ in range(num_workers)] - for i in range(params.num_batches): - computed_grads = [] - for j in range(num_workers): - batch_feed = next(batch_iterators[j]) - batch_feed = batch_feed / 127.5 - 1 - worker_loss, worker_grads = sess.run((reported_loss, grads), - {inputs_placeholder: batch_feed}) - losses[j].append(worker_loss) - computed_grads.append(worker_grads) - for worker_grads in computed_grads: - # TODO(reedwm): With multiple workers, applying the gradients - # sequentially per worker is not equivalent to what tf_cnn_benchmarks - # does when the optmizer is not SGD. Therefore, this currently does not - # work currently when num_workers > 1 and params.optimizer != 'sgd'. - feed_dict = dict(zip(placeholder_grads, worker_grads)) - sess.run(apply_grads_op, feed_dict) - return losses - - -class TestCNNModel(model.CNNModel): - """A simple model used for testing. - - The input is a 1-channel 1x1 image, consisting of a single number. The model - has two scalar variables: A and B, initialized to 1 and 2 respectively. Given - an image x, the loss is defined as: - - loss = x * A * B - """ - - def __init__(self): - super(TestCNNModel, self).__init__( - 'test_cnn_model', image_size=1, batch_size=1, learning_rate=1) - self.depth = 1 - - VAR_A_INITIAL_VALUE = 1. - VAR_B_INITIAL_VALUE = 2. - - def add_inference(self, cnn): - # This model only supports 1x1 images with 1 channel - assert cnn.top_layer.shape[1:] == (1, 1, 1) - # Multiply by variable A. - with tf.name_scope('mult_by_var_A'): - cnn.conv(1, 1, 1, 1, 1, use_batch_norm=None, activation=None, bias=None, - kernel_initializer=tf.constant_initializer( - self.VAR_A_INITIAL_VALUE)) - # Multiply by variable B. - with tf.name_scope('mult_by_var_B'): - cnn.conv(1, 1, 1, 1, 1, use_batch_norm=None, activation=None, bias=None, - kernel_initializer=tf.constant_initializer( - self.VAR_B_INITIAL_VALUE)) - with tf.name_scope('reshape_to_scalar'): - cnn.reshape([-1, 1]) - - def skip_final_affine_layer(self): - return True - - def loss_function(self, inputs, build_network_result): - del inputs - return tf.reduce_mean(build_network_result.logits) - - def manually_compute_losses(self, inputs, num_workers, params): - with tf.Graph().as_default(), tf.device('/cpu:0'): - a = tf.Variable(self.VAR_A_INITIAL_VALUE, name='A') - b = tf.Variable(self.VAR_B_INITIAL_VALUE, name='B') - inputs_placeholder = tf.placeholder(tf.float32, - (None, 1, 1, 1), - name='inputs_placeholder') - inputs_reshaped = tf.reshape(inputs_placeholder, (-1, 1)) - loss = self.loss_function( - None, - model.BuildNetworkResult(logits=inputs_reshaped * a * b, - extra_info=None)) - return manually_compute_losses(inputs, inputs_placeholder, loss, - num_workers, params) - - def accuracy_function(self, inputs, logits): - del inputs - # Let the accuracy be the same as the loss function. - return {'top_1_accuracy': logits, 'top_5_accuracy': logits} - - -class TestDataSet(datasets.ImageDataset): - """A Dataset consisting of 1x1 images with a depth of 1.""" - - def __init__(self, height=1, width=1, depth=1): - super(TestDataSet, self).__init__('test_dataset', height=height, - width=width, depth=depth, data_dir=None, - queue_runner_required=True, num_classes=1) - - def num_examples_per_epoch(self, subset='train'): - del subset - return 1 - - def get_input_preprocessor(self, input_preprocessor='default'): - return preprocessing.TestImagePreprocessor - - def use_synthetic_gpu_inputs(self): - return False diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/tf_cnn_benchmarks.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/tf_cnn_benchmarks.py deleted file mode 100644 index 29702aa16808121d2027b70295c9a3ecb4f5d76e..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/tf_cnn_benchmarks.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -"""Benchmark script for TensorFlow. - -See the README for more information. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from absl import app -from absl import flags as absl_flags -import tensorflow.compat.v1 as tf - -import benchmark_cnn -import cnn_util -import flags -import mlperf -from cnn_util import log_fn - - -flags.define_flags() -for name in flags.param_specs.keys(): - absl_flags.declare_key_flag(name) - -absl_flags.DEFINE_boolean( - 'ml_perf_compliance_logging', False, - 'Print logs required to be compliant with MLPerf. If set, must clone the ' - 'MLPerf training repo https://github.com/mlperf/training and add ' - 'https://github.com/mlperf/training/tree/master/compliance to the ' - 'PYTHONPATH') - - -def main(positional_arguments): - # Command-line arguments like '--distortions False' are equivalent to - # '--distortions=True False', where False is a positional argument. To prevent - # this from silently running with distortions, we do not allow positional - # arguments. - assert len(positional_arguments) >= 1 - if len(positional_arguments) > 1: - raise ValueError('Received unknown positional arguments: %s' - % positional_arguments[1:]) - - params = benchmark_cnn.make_params_from_flags() - with mlperf.mlperf_logger(absl_flags.FLAGS.ml_perf_compliance_logging, - params.model): - params = benchmark_cnn.setup(params) - bench = benchmark_cnn.BenchmarkCNN(params) - - tfversion = cnn_util.tensorflow_version_tuple() - log_fn('TensorFlow: %i.%i' % (tfversion[0], tfversion[1])) - - bench.print_info() - bench.run() - - -if __name__ == '__main__': - tf.disable_v2_behavior() - app.run(main) # Raises error on invalid flags, unlike tf.app.run() diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/variable_mgr.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/variable_mgr.py deleted file mode 100644 index 119b0278c0c0a8ac0f49811267554b3db216ef98..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/variable_mgr.py +++ /dev/null @@ -1,839 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Defines VariableMgr and subclasses used to manage variables. - -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import contextlib -import re - -import tensorflow.compat.v1 as tf - -import allreduce -import batch_allreduce -import variable_mgr_util - - -class VariableMgr(object): - """Abstract superclass for class used by BenchmarkCNN to control variables. - - Functions on this class are used to control how variables are created and - managed, and how gradients are computed and applied. - """ - - def __init__(self, benchmark_cnn): - self.benchmark_cnn = benchmark_cnn - self.staging_delta_ops = [] - self.use_resource_vars = benchmark_cnn.params.use_resource_vars - - # A variable for automatic loss scaling. - self.grad_has_inf_nan = None - - self._reuse_vars = False - - def each_tower_has_variables(self): - """Returns True if each GPU tower of the model has separate variables.""" - assert False, 'Must be implemented in subclass' - - def supports_staged_vars(self): - """Whether staged variable management is supported.""" - return False - - def create_outer_variable_scope(self, device_num): - """Create the tf.variable_scope around all model graph operations.""" - del device_num # unused by this implementation - assert False, 'Must be implemented in subclass' - - def preprocess_device_grads(self, device_grads): - """Preprocess the device gradients prior to applying them. - - Args: - device_grads: List of lists of (gradient, variable) tuples. - device_grads[t][g] = (gradient, variable), where t is the index of the - tower and g is the index of the gradient-variable pair. - - Returns: a tuple of (apply_gradients_devices, gradient_state). - gradient_state is an opaque structure that should be passed to - get_gradients_to_apply() and append_apply_gradients_ops() (in that order). - apply_gradients_devices is a list of devices where the gradients will be - applied with get_gradients_to_apply() and append_apply_gradients_ops(). - """ - del device_grads # unused by this implementation - assert False, 'Must be implemented in subclass' - - def get_gradients_to_apply(self, device_num, gradient_state): - """Returns the [(gradient, variable)] list to apply for device_num. - - Args: - device_num: indexes into apply_gradients_devices, which was returned by an - earlier call to preprocess_device_grads. - gradient_state: from previous call to apply_gradients_devices. - """ - del device_num, gradient_state # unused by this implementation - assert False, 'Must be implemented in subclass' - - def append_apply_gradients_ops(self, gradient_state, opt, grads, training_ops, - loss_scale_params): - """Adds training ops for grads to 'training_ops'. - - - - Args: - gradient_state: from previous call to apply_gradients_devices. - opt: the underlying optimizer - grads: [(grad, var)] to apply - training_ops: list to which to add ops - loss_scale_params: parameters for loss scaling. - """ - del gradient_state # unused by this implementation - - def get_apply_gradients_ops_func(): - """Returns the apply_gradients op.""" - return [opt.apply_gradients(grads)] - - variable_mgr_util.append_gradients_with_loss_scale( - training_ops, get_apply_gradients_ops_func, loss_scale_params, - self.grad_has_inf_nan) - - def get_post_init_ops(self): - """Returns ops that should run post-initialization.""" - return [] - - def get_devices(self): - """Returns devices to use for computation; includes replica selection.""" - assert False, 'Must be implemented in subclass' - - def savable_variables(self): - """Returns a list/dict of savable variables to pass to tf.train.Saver.""" - return tf.global_variables() - - def trainable_variables_on_device(self, - rel_device_num, - abs_device_num, - writable=False): - """Return the set of trainable variables on device. - - Args: - rel_device_num: local worker device index. - abs_device_num: global graph device index. - writable: whether to get a reference to the underlying variable. - - Returns: - The set of trainable variables on the specified device. - """ - del rel_device_num, writable - if self.each_tower_has_variables(): - params = [ - v for v in tf.trainable_variables() - if v.name.startswith('v%s/' % abs_device_num) - ] - else: - params = tf.trainable_variables() - return params - - @contextlib.contextmanager - def reuse_variables(self): - """Context manager that causes variables requested to be reused. - - Variables requested under this context manager must already exist, and will - be reused instead of being created again. This should be used if the - evaluation model is being built after the training model has already been - built. This is because the evaluation model should reuse variables from the - training model. - - Yields: - Nothing. - """ - old_reuse_vars = self._reuse_vars - try: - self._reuse_vars = True - yield - finally: - self._reuse_vars = old_reuse_vars - - -class VariableMgrIndependent(VariableMgr): - """VariableMgr that implements the --independent mode for local jobs. - - Each GPU has its own copy of the variables, and gradients are - not shared between towers. This can be used to check - performance when no data is moved between GPUs. - """ - - def each_tower_has_variables(self): - return True - - def create_outer_variable_scope(self, device_num): - return tf.variable_scope('v%s' % device_num, reuse=self._reuse_vars, - use_resource=self.use_resource_vars) - - def preprocess_device_grads(self, device_grads): - return (self.benchmark_cnn.devices, device_grads) - - def get_gradients_to_apply(self, device_num, gradient_state): - device_grads = gradient_state - tower_grad = device_grads[device_num] - - if self.benchmark_cnn.enable_auto_loss_scale and device_num == 0: - # Since we don't aggregate variables in --independent mode, we cannot tell - # if there are NaNs on all GPUs. So we arbitrarily choose to only check - # NaNs on the first GPU. - has_inf_nan_list = [] - for grad, _ in tower_grad: - has_inf_nan_list.append(tf.reduce_all(tf.is_finite(grad))) - self.grad_has_inf_nan = tf.logical_not(tf.reduce_all(has_inf_nan_list)) - - return tower_grad - - def get_devices(self): - return self.benchmark_cnn.raw_devices - - -class VariableMgrLocalFetchFromPS(VariableMgr): - """VariableMgr that implements the --parameter_server mode for local jobs. - - Variables are stored on a parameter server. For each step, each tower gets - a copy of the variables from the parameter server, and sends its gradients - to the param server. - """ - - def each_tower_has_variables(self): - return False - - def create_outer_variable_scope(self, device_num): - return tf.variable_scope('v', reuse=bool(device_num) or self._reuse_vars, - use_resource=self.use_resource_vars) - - def preprocess_device_grads(self, device_grads): - return ([self.benchmark_cnn.param_server_device], device_grads) - - def get_gradients_to_apply(self, device_num, gradient_state): - assert device_num == 0 - device_grads = gradient_state - agg_grads, self.grad_has_inf_nan = ( - variable_mgr_util. - aggregate_gradients_using_copy_with_variable_colocation( - device_grads, - use_mean=True, - check_inf_nan=self.benchmark_cnn.enable_auto_loss_scale)) - return agg_grads - - def get_devices(self): - raw_devices = self.benchmark_cnn.raw_devices - if self.benchmark_cnn.local_parameter_device_flag == 'gpu': - return [ - variable_mgr_util.ParamServerDeviceSetter(d, raw_devices) - for d in raw_devices - ] - else: - return [ - tf.train.replica_device_setter( - worker_device=d, - ps_device=self.benchmark_cnn.param_server_device, - ps_tasks=1) for d in raw_devices - ] - - -class VariableMgrLocalFetchFromStagedPS(VariableMgrLocalFetchFromPS): - """Implements fetching a local variable through staging buffers. - """ - - def __init__(self, benchmark_cnn): - super(VariableMgrLocalFetchFromStagedPS, self).__init__(benchmark_cnn) - # A data structure to track where the variables are used on each device. - # Indexed by device_num and var_name, each entry stores the "put" and "get" - # ops used for that variable on that device: - # staging_vars_on_devices[device_num][var_name] == (put_op, get_op) - self.staging_vars_on_devices = [ - dict() for _ in self.benchmark_cnn.raw_devices - ] - - def supports_staged_vars(self): - return True - - def create_outer_variable_scope(self, device_num): - self._custom_getter = variable_mgr_util.StagedVariableGetter( - device_num, self.benchmark_cnn.raw_devices, None, self) - return tf.variable_scope( - 'v', reuse=bool(device_num) or self._reuse_vars, - custom_getter=self._custom_getter, use_resource=self.use_resource_vars) - - def trainable_variables_on_device(self, - rel_device_num, - abs_device_num, - writable=False): - return self._custom_getter.trainable_variables_on_device( - rel_device_num, abs_device_num, writable=writable) - - -class VariableMgrLocalReplicated(VariableMgr): - """VariableMgr that implements the --replicated mode for local jobs. - - Each GPU has its own copy of the variables. To apply gradients, - either a local all-reduce algorithm is applied or a regular - cross-device aggregation is used to replicate the combined - gradients to all towers. - """ - - def __init__(self, benchmark_cnn, all_reduce_spec, - agg_small_grads_max_bytes, agg_small_grads_max_group, - allreduce_merge_scope): - super(VariableMgrLocalReplicated, self).__init__(benchmark_cnn) - if all_reduce_spec: - spec = allreduce.parse_all_reduce_spec(all_reduce_spec) - if len(spec) != 1: - raise ValueError( - 'replicated mode does not support hybrid all-reduce strategies') - self._all_reduce_spec = spec[0] - else: - self._all_reduce_spec = None - self._agg_small_grads_max_bytes = agg_small_grads_max_bytes - self._agg_small_grads_max_group = agg_small_grads_max_group - self._warmup_ops = [] - self._allreduce_merge_scope = allreduce_merge_scope - self._gradient_put_ops = None - - def each_tower_has_variables(self): - return True - - def create_outer_variable_scope(self, device_num): - return tf.variable_scope('v%s' % device_num, reuse=self._reuse_vars, - use_resource=self.use_resource_vars) - - def preprocess_device_grads(self, device_grads): - compact_grads = (self.benchmark_cnn.params.use_fp16 and - self.benchmark_cnn.params.compact_gradient_transfer) - defer_grads = (self.benchmark_cnn.params.variable_consistency == 'relaxed') - - grads_to_reduce = [[g for g, _ in grad_vars] for grad_vars in device_grads] - algorithm = batch_allreduce.algorithm_from_params(self.benchmark_cnn.params) - reduced_grads, self._warmup_ops = algorithm.batch_all_reduce( - grads_to_reduce, self.benchmark_cnn.params.gradient_repacking, - compact_grads, defer_grads, self.benchmark_cnn.params.xla_compile) - if self.benchmark_cnn.enable_auto_loss_scale: - # Check for infs or nans - is_finite_list = [] - with tf.name_scope('check_for_inf_and_nan'): - for tower_grads in reduced_grads: - with tf.colocate_with(tower_grads[0]): - # TODO(tanmingxing): Create fused op that takes in a list of tensors - # as input and returns scalar boolean True if there are any - # infs/nans. - is_finite_list.append(tf.reduce_all( - [tf.reduce_all(tf.is_finite(g)) for g in tower_grads])) - self.grad_has_inf_nan = tf.logical_not(tf.reduce_all(is_finite_list)) - reduced_device_grads = [[ - (g, v) for g, (_, v) in zip(grads, grad_vars) - ] for grads, grad_vars in zip(reduced_grads, device_grads)] - return self.benchmark_cnn.devices, reduced_device_grads - - def get_gradients_to_apply(self, device_num, gradient_state): - device_grads = gradient_state - return device_grads[device_num] - - def get_post_init_ops(self): - # Copy initialized values for variables on GPU 0 to other GPUs. - global_vars = tf.global_variables() - var_by_name = dict([(v.name, v) for v in global_vars]) - post_init_ops = [] - for v in global_vars: - split_name = v.name.split('/') - # TODO(b/62630508): use more specific prefix than v or v0. - if split_name[0] == 'v0' or not v.name.startswith('v'): - continue - split_name[0] = 'v0' - copy_from = var_by_name['/'.join(split_name)] - post_init_ops.append(v.assign(copy_from.read_value())) - post_init_ops += self._warmup_ops - return post_init_ops - - def savable_variables(self): - """Return the set of variables used for saving/loading the model.""" - params = [] - for v in tf.global_variables(): - split_name = v.name.split('/') - if split_name[0] == 'v0' or not v.name.startswith('v'): - params.append(v) - return params - - def get_devices(self): - return self.benchmark_cnn.raw_devices - - -class VariableMgrDistributedAllReduce(VariableMgr): - """VariableMgr that implements the --distributed_all_reduce mode. - - Each GPU has its own copy of the variables. To apply gradients, - the specified all-reduce algorithm is used to reduce the gradients - and replicate the final value to all GPUs. - """ - - def __init__(self, benchmark_cnn, all_reduce_spec, job_name, - num_workers, agg_small_grads_max_bytes, - agg_small_grads_max_group, allreduce_merge_scope): - super(VariableMgrDistributedAllReduce, self).__init__(benchmark_cnn) - if not all_reduce_spec: - raise ValueError( - 'distributed_all_reduce requires a non-empty all_reduce_spec') - self._all_reduce_spec = allreduce.parse_all_reduce_spec(all_reduce_spec) - self._all_reduce_device_prefixes = ( - allreduce.build_all_reduce_device_prefixes(job_name, num_workers)) - self._num_workers = num_workers - self._agg_small_grads_max_bytes = agg_small_grads_max_bytes - self._agg_small_grads_max_group = agg_small_grads_max_group - self._allreduce_merge_scope = allreduce_merge_scope - if not self._all_reduce_spec: - raise ValueError('all_reduce_spec must be specified') - self._single_session = True - - def each_tower_has_variables(self): - return True - - def create_outer_variable_scope(self, device_num): - """Create a scope for the named device. - - Args: - device_num: index of device for variable scope. (Note that - device_num spans all processes in cluster since a single global - graph is used.) - - Returns: - the requested variable_scope - """ - return tf.variable_scope('v%s' % device_num, reuse=self._reuse_vars, - use_resource=self.use_resource_vars) - - def preprocess_device_grads(self, device_grads): - remaining_grads = device_grads - aggregated_grads = [] - for spec_tuple in self._all_reduce_spec: - if spec_tuple.limit < 0: - this_grads = remaining_grads - remaining_grads = [] - else: - (this_grads, remaining_grads) = allreduce.split_grads_by_size( - spec_tuple.limit, remaining_grads) - if this_grads: - range_agg_grads = allreduce.sum_gradients_all_reduce( - self._single_session, - self._all_reduce_device_prefixes, - this_grads, - self._num_workers, - spec_tuple.alg, - spec_tuple.shards, - self.benchmark_cnn.gpu_indices, - agg_small_grads_max_bytes=self._agg_small_grads_max_bytes, - agg_small_grads_max_group=self._agg_small_grads_max_group, - allreduce_merge_scope=self._allreduce_merge_scope) - if not aggregated_grads: - aggregated_grads = range_agg_grads - else: - assert len(aggregated_grads) == len(range_agg_grads) - for i in range(len(aggregated_grads)): - aggregated_grads[i] += range_agg_grads[i] - assert not remaining_grads - full_device_set = [] - for grads in device_grads: - g, v = grads[0] - del v - full_device_set.append(g.device) - return (full_device_set, aggregated_grads) - - def get_gradients_to_apply(self, device_num, gradient_state): - device_grads = gradient_state - if device_num >= len(device_grads): - raise ValueError('device_num %d exceeds length of device_grads (%d)' % - (device_num, len(device_grads))) - return device_grads[device_num] - - def get_post_init_ops(self): - """Copy initialized values for variables to other devices.""" - global_vars = tf.global_variables() - var_by_name = dict([(v.name, v) for v in global_vars]) - post_init_ops = [] - for v in global_vars: - split_name = v.name.split('/') - # TODO(b/62630508): use more specific prefix than v or v0. - if split_name[0] == 'v0' or not v.name.startswith('v'): - continue - split_name[0] = 'v0' - copy_from = var_by_name['/'.join(split_name)] - post_init_ops.append(v.assign(copy_from.read_value())) - return post_init_ops - - def savable_variables(self): - """Return the set of variables used for saving/loading the model.""" - params = [] - for v in tf.global_variables(): - split_name = v.name.split('/') - if split_name[0] == 'v0' or not v.name.startswith('v'): - params.append(v) - return params - - def get_devices(self): - return self.benchmark_cnn.raw_devices - - -# TODO(tucker): Merge this mode with DistributedAllReduce. -class VariableMgrCollectiveAllReduce(VariableMgr): - """VariableMgr that implements the --collective_all_reduce mode. - - Each GPU has its own copy of the variables. To apply gradients - the TF native collective all-reduce op is used to reduce the gradients - and replicate the final value to all GPUs. - """ - - def __init__(self, benchmark_cnn, all_reduce_spec, - num_workers, num_gpus, task_id, allreduce_merge_scope): - super(VariableMgrCollectiveAllReduce, self).__init__(benchmark_cnn) - if not all_reduce_spec: - raise ValueError( - 'collective_all_reduce requires a non-empty all_reduce_spec: %s' - % all_reduce_spec) - parsed_spec = allreduce.parse_all_reduce_spec(all_reduce_spec) - # So far we only support a length-1 all_reduce_spec - if len(parsed_spec) > 1 or parsed_spec[0].limit > 0: - raise ValueError( - 'collective_all_reduce requires one single-range all_reduce_spec %s' - % parsed_spec) - self._all_reduce_spec = parsed_spec[0] - if self._all_reduce_spec.alg != 'collective': - raise ValueError( - 'VariableMgrCollectiveAllReduce initialized with non-collective ' - 'all_reduce_spec %s' % self.all_reduce_spec) - self._num_workers = num_workers - self._num_gpus = num_gpus - self._task_id = task_id - self._allreduce_merge_scope = allreduce_merge_scope - self._instance_key_counter = 10000 - self._instance_key_table = dict() - self._single_session = False - # List of prefixes for generating PS devices, unused here. - self._all_reduce_device_prefixes = None - - def each_tower_has_variables(self): - return True - - def create_outer_variable_scope(self, device_num): - """Create a scope for the named device. - - Args: - device_num: index of device for variable scope. - - Returns: - the requested variable_scope - """ - return tf.variable_scope('v%s' % device_num, reuse=self._reuse_vars) - - def preprocess_device_grads(self, device_grads): - reduced_grads = allreduce.sum_gradients_all_reduce( - self._single_session, - self._all_reduce_device_prefixes, - device_grads, - self._num_workers, - 'collective', - self._all_reduce_spec.shards, - self.benchmark_cnn.gpu_indices, - allreduce_merge_scope=self._allreduce_merge_scope) - assert len(reduced_grads) == len(device_grads) - full_device_set = [] - for grads in device_grads: - g, _ = grads[0] - full_device_set.append(g.device) - return (full_device_set, reduced_grads) - - def get_gradients_to_apply(self, device_num, gradient_state): - device_grads = gradient_state - if device_num >= len(device_grads): - raise ValueError('device_num %d exceeds length of device_grads (%d)' % - (device_num, len(device_grads))) - return device_grads[device_num] - - def _get_instance_key(self, name): - if name not in self._instance_key_table.keys(): - self._instance_key_counter += 1 - self._instance_key_table[name] = self._instance_key_counter - return self._instance_key_table[name] - - def get_post_init_ops(self): - """Broadcast initialized values of variables to other devices. - - Returns: - At task 0 device 0, broadcast_send. - At all other devices and tasks, broadcast_recv. - """ - global_vars = tf.global_variables() - group_size = self._num_workers * self._num_gpus - post_init_ops = [] - # Gather variables into same-var-different-device groups. - vars_by_suffix = dict() - for v in global_vars: - split_name = v.name.split('/') - mo = re.match(r'v(\d+)$', split_name[0]) - if mo: - device_id = int(mo.group(1)) - suffix = '/'.join(split_name[1:]) - if suffix in vars_by_suffix.keys(): - vars_by_suffix[suffix].append(v) - else: - vars_by_suffix[suffix] = [v] - # Generate broadcast ops for each such group. - for suffix in sorted(vars_by_suffix): - vlist = vars_by_suffix[suffix] - assert self._num_gpus == len(vlist) - devices = [v.device for v in vlist] - # NOTE: this key should generate the same value for all tasks - group_key = allreduce.collective_group_key(devices) - group_size = self._num_workers * len(devices) - instance_key = self._get_instance_key(suffix) - for v in vlist: - split_name = v.name.split('/') - mo = re.match(r'v(\d+)$', split_name[0]) - if mo: - device_id = int(mo.group(1)) - if (self._task_id == 0 and device_id == 0): - with tf.device(v.device): - bcast_send = allreduce.broadcast_send( - v, v.shape, v.dtype, group_size, group_key, instance_key) - post_init_ops.append(v.assign(bcast_send)) - else: - with tf.device(v.device): - bcast_recv = allreduce.broadcast_recv( - v.shape, v.dtype, group_size, group_key, instance_key) - post_init_ops.append(v.assign(bcast_recv)) - return post_init_ops - - def savable_variables(self): - """Return the set of variables used for saving/loading the model.""" - params = [] - if self._task_id == 0: - for v in tf.global_variables(): - split_name = v.name.split('/') - if split_name[0] == 'v0' or not v.name.startswith('v'): - params.append(v) - return params - - def get_devices(self): - return self.benchmark_cnn.raw_devices - - -class VariableMgrDistributedFetchFromPS(VariableMgr): - """Implements --variable_update=parameter_server mode for distributed jobs. - - Variables are stored on a parameter server. For each step, each tower gets - a copy of the variables from the parameter server, and sends its gradients - to the param server. - """ - - def each_tower_has_variables(self): - return False - - def create_outer_variable_scope(self, device_num): - if self.benchmark_cnn.local_parameter_device_flag == 'gpu': - caching_devices = self.benchmark_cnn.raw_devices - else: - caching_devices = [self.benchmark_cnn.cpu_device] - custom_getter = variable_mgr_util.OverrideCachingDevice( - caching_devices, self.benchmark_cnn.cpu_device, 1024 * 64) - return tf.variable_scope( - 'v', reuse=bool(device_num) or self._reuse_vars, - custom_getter=custom_getter, use_resource=self.use_resource_vars) - - def preprocess_device_grads(self, device_grads): - # Returns (gradient_devices, gradient_state) - return ([self.benchmark_cnn.param_server_device], device_grads) - - def get_gradients_to_apply(self, device_num, gradient_state): - assert device_num == 0 - agg_grads, self.grad_has_inf_nan = ( - variable_mgr_util.aggregate_gradients_using_copy( - gradient_state, - use_mean=True, - check_inf_nan=self.benchmark_cnn.enable_auto_loss_scale)) - return agg_grads - - def get_devices(self): - ps_strategy = variable_mgr_util.GreedyLoadBalancingStrategy( - self.benchmark_cnn.num_ps, variable_mgr_util.byte_size_load_fn) - return [ - tf.train.replica_device_setter( - worker_device=d, - cluster=self.benchmark_cnn.cluster_manager.get_cluster_spec(), - ps_strategy=ps_strategy) for d in self.benchmark_cnn.raw_devices - ] - - -class VariableMgrDistributedFetchFromStagedPS( - VariableMgrDistributedFetchFromPS): - """Extends VariableMgrDistributedFetchFromPS for --staged_vars.""" - - def __init__(self, benchmark_cnn): - super(VariableMgrDistributedFetchFromStagedPS, self).__init__(benchmark_cnn) - self.staging_vars_on_devices = [ - dict() for _ in self.benchmark_cnn.raw_devices - ] - self.staged_vars_on_cpu = {} - - def create_outer_variable_scope(self, device_num): - self._custom_getter = variable_mgr_util.StagedVariableGetter( - device_num, self.benchmark_cnn.raw_devices, - self.benchmark_cnn.cpu_device, self) - return tf.variable_scope( - 'v', reuse=bool(device_num) or self._reuse_vars, - custom_getter=self._custom_getter, use_resource=self.use_resource_vars) - - def supports_staged_vars(self): - return True - - def trainable_variables_on_device(self, - rel_device_num, - abs_device_num, - writable=False): - return self._custom_getter.trainable_variables_on_device( - rel_device_num, abs_device_num, writable=writable) - - -class VariableMgrDistributedReplicated(VariableMgr): - """VariableMgr that implements the --distributed_replicated mode. - - Each GPU has a copy of the variables, and updates its copy after the - parameter servers are all updated with the gradients from all servers. Only - works with cross_replica_sync=true. Unlike 'replicated', does not use nccl - all-reduce for replicating within a server. - """ - - def each_tower_has_variables(self): - return True - - def create_outer_variable_scope(self, device_num): - return tf.variable_scope( - 'v%s' % device_num, reuse=self._reuse_vars, - custom_getter=variable_mgr_util.OverrideToLocalVariableIfNotPsVar(), - use_resource=self.use_resource_vars) - - def preprocess_device_grads(self, device_grads): - return ([self.benchmark_cnn.param_server_device], device_grads) - - def get_gradients_to_apply(self, device_num, gradient_state): - device_grads = gradient_state # From 2nd result of preprocess_device_grads. - - avg_grads, self.grad_has_inf_nan = ( - variable_mgr_util.aggregate_gradients_using_copy_with_device_selection( - self.benchmark_cnn, - device_grads, - use_mean=True, - check_inf_nan=self.benchmark_cnn.enable_auto_loss_scale)) - - # Make shadow variable on a parameter server for each original trainable - # variable. - for i, (g, v) in enumerate(avg_grads): - my_name = variable_mgr_util.PS_SHADOW_VAR_PREFIX + '/' + v.name - if my_name.endswith(':0'): - my_name = my_name[:-2] - new_v = tf.get_variable( - my_name, - dtype=v.dtype.base_dtype, - initializer=v.initial_value, - trainable=True) - avg_grads[i] = (g, new_v) - return avg_grads - - def append_apply_gradients_ops(self, gradient_state, opt, grads, training_ops, - loss_scale_params): - device_grads = gradient_state # From 2nd result of preprocess_device_grads. - - def get_apply_gradients_ops_func(): - """Returns a list of ops for updating gradients.""" - apply_gradients_ops = [] - # For each variable, apply the combined gradients for this server on - # the parameter server, and then wait for all other servers to do this. - for i, (g, v) in enumerate(grads): - apply_gradient_op = opt.apply_gradients([(g, v)]) - barrier = self.benchmark_cnn.add_sync_queues_and_barrier( - 'replicate_variable_%s' % i, [apply_gradient_op]) - with tf.control_dependencies([barrier]): - with tf.device(self.benchmark_cnn.cpu_device): - updated_value = v.read_value() - for my_d in range(len(self.benchmark_cnn.devices)): - apply_gradients_ops.append( - device_grads[my_d][i][1].assign(updated_value)) - return apply_gradients_ops - - variable_mgr_util.append_gradients_with_loss_scale( - training_ops, get_apply_gradients_ops_func, loss_scale_params, - self.grad_has_inf_nan) - - def _strip_port(self, s): - if s.endswith(':0'): - return s[:-2] - return s - - def get_post_init_ops(self): - # Copy initialized variables for variables on the parameter server - # to the local copy of the variable. - - local_vars = tf.local_variables() - local_var_by_name = dict( - [(self._strip_port(v.name), v) for v in local_vars]) - post_init_ops = [] - for v in tf.global_variables(): - if v.name.startswith(variable_mgr_util.PS_SHADOW_VAR_PREFIX + '/v0/'): - prefix = self._strip_port( - v.name[len(variable_mgr_util.PS_SHADOW_VAR_PREFIX + '/v0'):]) - for i in range(self.benchmark_cnn.num_gpus): - name = 'v%s%s' % (i, prefix) - if name in local_var_by_name: - copy_to = local_var_by_name[name] - post_init_ops.append(copy_to.assign(v.read_value())) - return post_init_ops - - def _remove_shadow_var_prefix_if_present(self, var_name): - if var_name.startswith(variable_mgr_util.PS_SHADOW_VAR_PREFIX + '/'): - return var_name[len(variable_mgr_util.PS_SHADOW_VAR_PREFIX + '/'):] - else: - return var_name - - def var_dict_name(self, v): - return self._strip_port(self._remove_shadow_var_prefix_if_present(v.name)) - - def savable_variables(self): - """Returns a list/dict of savable variables to pass to tf.train.Saver.""" - params = {} - for v in tf.global_variables(): - assert (v.name.startswith(variable_mgr_util.PS_SHADOW_VAR_PREFIX + '/v0/') - or v.name in ('global_step:0', 'loss_scale:0', - 'loss_scale_normal_steps:0')), ( - 'Invalid global variable: %s' % v) - # We store variables in the checkpoint with the shadow variable prefix - # removed so we can evaluate checkpoints in non-distributed replicated - # mode. The checkpoints can also be loaded for training in - # distributed_replicated mode. - name = self._strip_port(self._remove_shadow_var_prefix_if_present(v.name)) - params[name] = v - for v in tf.local_variables(): - # Non-trainable variables, such as batch norm moving averages, do not have - # corresponding global shadow variables, so we add them here. Trainable - # local variables have corresponding global shadow variables, which were - # added in the global variable loop above. - if v.name.startswith('v0/') and v not in tf.trainable_variables(): - params[self._strip_port(v.name)] = v - return params - - def get_devices(self): - return self.benchmark_cnn.raw_devices diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/variable_mgr_util.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/variable_mgr_util.py deleted file mode 100644 index 94ce3e4b7c48d49797802f3dfadbaf0d4108d902..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/variable_mgr_util.py +++ /dev/null @@ -1,676 +0,0 @@ -# Copyright 2017 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utilities for VariableMgr.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections as pycoll -import operator - -import numpy as np -import tensorflow.compat.v1 as tf - -# pylint: disable=g-direct-tensorflow-import -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_shape -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import data_flow_ops -from tensorflow.python.ops import math_ops - - -PS_SHADOW_VAR_PREFIX = 'ps_var' - -AutoLossScaleParams = pycoll.namedtuple( - 'AutoLossScaleParams', - [ - # If true, enable automatic loss scaling. - 'enable_auto_loss_scale', - # The value to scale the loss before computing gradients. - 'loss_scale', - # Number of normal steps with the current `loss_scale`. - 'loss_scale_normal_steps', - # Increase loss scale every n steps. - 'inc_loss_scale_every_n', - # If true, the current worker is chief. The current implementation - # relies on the chief to update loss_scale value, but in future, we - # might change this to ask the parameter server to update loss_scales - # for better performance. - # TODO(tanmingxing): remove this if loss_scale is updated in ps. - 'is_chief', - ]) - - -def get_loss_scale_update_op(loss_scale, loss_scale_normal_steps, - inc_loss_scale_every_n): - """Returns the update op for loss scaling variables. - - We maintain the counter `loss_scale_normal_steps` to count the number of steps - we have been using the current `loss_scale`. In most cases, this function - increments `loss_scale_normal_steps`. However, if `loss_scale_normal_steps` is - greater than the threshold `inc_loss_scale_every_n`, we double `loss_scale` - and reset `loss_scale_normal_steps` to zero. - - This op is only called if the gradients don't have any infs or nans. Instead, - if infs or nans occur in the gradients, we immeditately halve `loss_scale` and - reset `loss_scale_normal_steps` to zero. - - Args: - loss_scale: a tf.Variable represneting the loss_scale value. - loss_scale_normal_steps: a tf.Variable representing the number of training - steps that have run since the loss_scale last changed. - inc_loss_scale_every_n: a Python integer threshold. `loss_scale` is - increased every `inc_loss_scale_every_n` steps, unless the gradients have - infs or nans. - - Returns: - An op for updating `loss_scale` and `loss_scale_normal_steps`. - """ - - def increment_loss_scale_normal_steps_func(): - return tf.group(loss_scale_normal_steps.assign_add(1)) - - def increase_loss_scale_func(): - return tf.group( - tf.assign(loss_scale_normal_steps, 0), - tf.assign(loss_scale, loss_scale * 2)) - - # true_fn and false_fn must have the same type. - return tf.cond(loss_scale_normal_steps < inc_loss_scale_every_n, - increment_loss_scale_normal_steps_func, - increase_loss_scale_func) - - -def append_gradients_with_loss_scale(training_ops, get_apply_gradients_ops_func, - loss_scale_params, grad_has_inf_nan): - """Selectively appends gradients update ops with loss scaling. - - Args: - training_ops: a list of training ops to be executed. - get_apply_gradients_ops_func: a function that returns a list of ops for - applying gradients. Here, we must pass a function instead of the actual - list of ops; otherwise, those ops would be executed unconditionally due to - the semantics of tf.cond. - loss_scale_params: An AutoLossScaleParams tuple. - grad_has_inf_nan: Boolean tensor indicating whether the gradients have infs - or nans. - """ - is_chief = loss_scale_params.is_chief - loss_scale = loss_scale_params.loss_scale - loss_scale_normal_steps = loss_scale_params.loss_scale_normal_steps - inc_loss_scale_every_n = loss_scale_params.inc_loss_scale_every_n - enable_auto_loss_scale = loss_scale_params.enable_auto_loss_scale - - if loss_scale is None or not enable_auto_loss_scale or not is_chief: - training_ops.extend(get_apply_gradients_ops_func()) - else: - # If nans/infs occurred, skip applying gradients and instead update - # loss_scale (halve loss_scale and reset loss_scale_normal_steps to zero). - def update_op_if_nan_or_inf(): - """Update loss_scale and discard gradients if nans/infs occurred.""" - return tf.group( - tf.assign(loss_scale, loss_scale / 2.), - tf.assign(loss_scale_normal_steps, 0)) - - # Otherwise, apply gradients, and update loss_scale and - # loss_scale_normal_steps. - def update_op_if_no_nan_or_inf(): - """Apply gradients, and update loss scaling.""" - return tf.group( - get_loss_scale_update_op(loss_scale, loss_scale_normal_steps, - inc_loss_scale_every_n), - *get_apply_gradients_ops_func()) - - # TODO(tanmingxing): Add support for independent and distributed all_reduce. - assert grad_has_inf_nan is not None - update_op = tf.cond( - grad_has_inf_nan, - update_op_if_nan_or_inf, - update_op_if_no_nan_or_inf, - name='cond_if_grad_has_inf_nan' - ) - training_ops.append(update_op) - - -# To be used with custom_getter on tf.get_variable. -class OverrideCachingDevice(object): - """Variable getter which caches variables on the least loaded device. - - Variables smaller than a certain threshold are cached on a single specific - device, as specified in the constructor. All other variables are load balanced - across a pool of devices, by caching each variable on the least loaded device. - - Note that variable creation only happen when building the model graph on the - first device (see how it sets the 'reuse' parameter in - VariableMgr.*.create_outer_variable_scope()). That means, for all other - devices, the variable scope will reuse the variables created before, which - requires that we set the caching_device correctly as otherwise it may not be - able to find the previously created variable and will create a new one. This - requires when building the model graph on different devices, variables with - the same name should have same size. - - TODO(laigd): consider adding tests or verification logic to enforce this, or - refactor it. - """ - - def __init__(self, devices, device_for_small_variables, - small_variable_size_threshold): - self.devices = devices - self.sizes = [0] * len(self.devices) - self.device_for_small_variables = device_for_small_variables - self.small_variable_size_threshold = small_variable_size_threshold - - def __call__(self, getter, *args, **kwargs): - size = tf.TensorShape(kwargs['shape']).num_elements() - if size < self.small_variable_size_threshold: - device_name = self.device_for_small_variables - else: - device_index, _ = min(enumerate(self.sizes), key=operator.itemgetter(1)) - device_name = self.devices[device_index] - self.sizes[device_index] += size - - kwargs['caching_device'] = device_name - var = getter(*args, **kwargs) - return var - - -# To be used with custom_getter on tf.get_variable. Ensures the created variable -# is in LOCAL_VARIABLES and not GLOBAL_VARIBLES collection. -class OverrideToLocalVariableIfNotPsVar(object): - - # args and kwargs come from the custom_getter interface for Tensorflow - # variables, and matches tf.get_variable's signature, with the addition of - # 'getter' at the beginning. - def __call__(self, getter, name, *args, **kwargs): - if name.startswith(PS_SHADOW_VAR_PREFIX): - return getter(*args, **kwargs) - - if 'collections' in kwargs: - collections = kwargs['collections'] - if not collections: - collections = [tf.GraphKeys.GLOBAL_VARIABLES] - else: - collections = collections[:] - collections.remove(tf.GraphKeys.GLOBAL_VARIABLES) - collections.append(tf.GraphKeys.LOCAL_VARIABLES) - kwargs['collections'] = list(collections) - return getter(name, *args, **kwargs) - - -class ParamServerDeviceSetter(object): - """Helper class to assign variables on the least loaded ps-device.""" - - def __init__(self, worker_device, ps_devices): - """Initializer for ParamServerDevicSetter. - - Args: - worker_device: the device to use for computer ops. - ps_devices: a list of device to use for Variable ops. Each variable is - assigned to the least loaded device. - """ - self.ps_devices = ps_devices - self.worker_device = worker_device - self.ps_sizes = [0] * len(self.ps_devices) - - def __call__(self, op): - if op.device: - return op.device - if op.type not in ['Variable', 'VariableV2']: - return self.worker_device - - device_index, _ = min(enumerate(self.ps_sizes), key=operator.itemgetter(1)) - device_name = self.ps_devices[device_index] - var_size = op.outputs[0].get_shape().num_elements() - self.ps_sizes[device_index] += var_size - - return device_name - - -class StagedModelVariable(object): - """Staging variable wrapper that decouples reads and updates. - - This class represents a variable through a staging buffer. Reads from this - variable directly gets from the staging buffer. Updates are stacked into - another staging buffer, and will be processed later. - """ - - def __init__(self, real_var, var_stage_get, variable_mgr): - """Initializer for the model variables through a staging buffer. - - Args: - real_var: the underlying real variable. - var_stage_get: the read op from the staging buffer. - variable_mgr: the parent variable-manager. - """ - self.real_var = real_var - self.var_stage_get = var_stage_get - self.variable_mgr = variable_mgr - - def _value(self): - """The read access of this variable. The content from the staging buffer.""" - return self.var_stage_get - - def _ref(self): - """Return the underlying variable ref, required by tf.colocate_with.""" - return self.real_var._ref() # pylint: disable=protected-access - - def read_value(self): - """Mimics tf.Variable.read_value().""" - return tf.identity(self.var_stage_get, name='read') - - @property - def dtype(self): - """Return the non-reference dtype.""" - return self.var_stage_get.dtype - - def assign_sub(self, delta, name=None, read_value=True): - """Mimic the updates to the variable. - - Args: - delta: is pushed into a staging buffer and will be pumped later. - name: currently ignored; names of ops and the StagingArea are - computed without using this pass name. - read_value: if True, will return something which evaluates to the new - value of the variable; if False will return the assign op. - Returns: - The actual updates. The colocation constraint will be reapplied. - """ - # This parameter is ignored: the StagingArea only supports setting - # the shared name, not the names of individual ops it uses. - del name - - # colocate_with(None, True) clears the colocation constraints. - # Push the delta into a staging buffer. - with ops.colocate_with(None, True), tf.device(self.var_stage_get.device): - delta_staging_area = data_flow_ops.StagingArea( - [self.var_stage_get.dtype], shapes=[self.var_stage_get.shape]) - delta_put_op = delta_staging_area.put([delta]) - self.variable_mgr.staging_delta_ops.append(delta_put_op) - delta_get_op = delta_staging_area.get()[0] - # Return the actual updates. The colocation constraint will be reapplied. - return self.real_var.assign_sub(delta_get_op, read_value=read_value) - - @staticmethod - # pylint: disable=bad-staticmethod-argument,invalid-name - def _TensorConversionFunction(self, dtype=None, name=None, as_ref=False): - """Utility function for converting a StagedModelVariable to a Tensor.""" - del dtype, name # unused: this function returns the cached ref or value. - if as_ref: - return self._ref() - else: - return self._value() - - -ops.register_tensor_conversion_function( - StagedModelVariable, StagedModelVariable._TensorConversionFunction) # pylint: disable=protected-access - - -class StagedVariableGetter(object): - """A variable getter through staging buffers on devices. - - Instead of a caching device, this getter tracks where the variable is used. - And on each device, it goes through a staging buffer. - """ - - def __init__(self, device_num, devices, cpu_device, variable_mgr): - """Initializer for StagedVariableGetter. - - Args: - device_num: the current device index. - devices: a list of all the devices to build towers. - cpu_device: a cpu_device for this replica. If None, no cpu-caching is - done. - variable_mgr: the parent variable manager. - """ - self.device_num = device_num - self.devices = devices - self.cpu_device = cpu_device - self.variable_mgr = variable_mgr - - def __call__(self, getter, name, *args, **kwargs): - staging_ops = self.variable_mgr.staging_vars_on_devices[self.device_num] - if name in staging_ops: - put_op, get_op = staging_ops[name] - return get_op - real_var = getter(name, *args, **kwargs) - shape = kwargs['shape'] - dtype = kwargs['dtype'] - trainable = kwargs['trainable'] - if self.cpu_device: - with tf.device(self.cpu_device): - # This helps copying the weights from the parameter to this server only - # once. - if name in self.variable_mgr.staged_vars_on_cpu: - cpu_var = self.variable_mgr.staged_vars_on_cpu[name] - else: - cpu_var = tf.identity(real_var) - self.variable_mgr.staged_vars_on_cpu[name] = cpu_var - var_to_stage = cpu_var - else: - var_to_stage = tf.identity(real_var) # de-reference the variable. - - with tf.device(self.devices[self.device_num]): - staging_area = data_flow_ops.StagingArea([dtype], shapes=[shape]) - put_op = staging_area.put([var_to_stage]) - get_op = staging_area.get()[0] - staging_ops[name] = (put_op, get_op) - if trainable: - # For trainable variables, they are managed separatedly through - # apply_gradients. - return get_op - else: - # For other shadow variables, the access is decoupled through a wrapper - # class. - return StagedModelVariable(real_var, get_op, self.variable_mgr) - - def trainable_variables_on_device(self, rel_device_num, abs_device_num, - writable): - """Return the set of trainable variables on the specified device. - - Args: - rel_device_num: local worker device index. - abs_device_num: global graph device index. - writable: whether the returned variables is writable or read-only. - - Returns: - Return the set of trainable variables on the specified device. - """ - del abs_device_num - params_refs = tf.trainable_variables() - if writable: - return params_refs - params = [] - for param in params_refs: - var_name = param.name.split(':')[0] - _, var_get_op = self.variable_mgr.staging_vars_on_devices[rel_device_num][ - var_name] - params.append(var_get_op) - return params - - -def aggregate_gradients_using_copy_with_device_selection( - benchmark_cnn, tower_grads, use_mean, check_inf_nan): - """Aggregate gradients, controlling device for the aggregation. - - Args: - benchmark_cnn: benchmark_cnn class. - tower_grads: List of lists of (gradient, variable) tuples. The outer list - is over towers. The inner list is over individual gradients. - use_mean: if True, mean is taken, else sum of gradients is taken. - check_inf_nan: If true, check grads for nans and infs. - - Returns: - The tuple ([(average_gradient, variable),], has_nan_or_inf) where the - gradient has been averaged across all towers. The variable is chosen from - the first tower. The has_nan_or_inf indicates the grads has nan or inf. - """ - if benchmark_cnn.local_parameter_device_flag == 'gpu': - avail_devices = benchmark_cnn.raw_devices - else: - avail_devices = [benchmark_cnn.param_server_device] - agg_grads = [] - has_nan_or_inf_list = [] - for i, single_grads in enumerate(zip(*tower_grads)): - with tf.device(avail_devices[i % len(avail_devices)]): - grad_and_var, has_nan_or_inf = aggregate_single_gradient_using_copy( - single_grads, use_mean, check_inf_nan) - agg_grads.append(grad_and_var) - has_nan_or_inf_list.append(has_nan_or_inf) - if check_inf_nan: - return agg_grads, tf.reduce_any(has_nan_or_inf_list) - else: - return agg_grads, None - - -def aggregate_gradients_using_copy_with_variable_colocation( - tower_grads, use_mean, check_inf_nan): - """Aggregate gradients, colocating computation with the gradient's variable. - - Args: - tower_grads: List of lists of (gradient, variable) tuples. The outer list - is over towers. The inner list is over individual gradients. All variables - of the same gradient across towers must be the same (that is, - tower_grads[x][a][1] == tower_grads[y][a][1] for all indices x, y, and a) - use_mean: if True, mean is taken, else sum of gradients is taken. - check_inf_nan: If true, check grads for nans and infs. - - Returns: - The tuple ([(average_gradient, variable),], has_nan_or_inf) where the - gradient has been averaged across all towers. The variable is chosen from - the first tower. The has_nan_or_inf indicates the grads has nan or inf. - """ - agg_grads = [] - has_nan_or_inf_list = [] - for single_grads in zip(*tower_grads): - # Note that each single_grads looks like the following: - # ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN)) - var = single_grads[0][1] - - for _, v in single_grads: - assert v == var - - with tf.device(var.device): - grad_and_var, has_nan_or_inf = aggregate_single_gradient_using_copy( - single_grads, use_mean, check_inf_nan) - agg_grads.append(grad_and_var) - has_nan_or_inf_list.append(has_nan_or_inf) - - if check_inf_nan: - return agg_grads, tf.reduce_any(has_nan_or_inf_list) - else: - return agg_grads, None - - -def aggregate_gradients_using_copy(tower_grads, use_mean, check_inf_nan): - """Calculate the average gradient for each shared variable across all towers. - - Note that this function provides a synchronization point across all towers. - - Args: - tower_grads: List of lists of (gradient, variable) tuples. The outer list - is over towers. The inner list is over individual gradients. - use_mean: if True, mean is taken, else sum of gradients is taken. - check_inf_nan: check grads for nans and infs. - - Returns: - The tuple ([(average_gradient, variable),], has_nan_or_inf) where the - gradient has been averaged across all towers. The variable is chosen from - the first tower. The has_nan_or_inf indicates the grads has nan or inf. - """ - agg_grads = [] - has_nan_or_inf_list = [] - - for single_grads in zip(*tower_grads): - grad_and_var, has_nan_or_inf = aggregate_single_gradient_using_copy( - single_grads, use_mean, check_inf_nan) - agg_grads.append(grad_and_var) - has_nan_or_inf_list.append(has_nan_or_inf) - - if check_inf_nan: - return agg_grads, tf.reduce_any(has_nan_or_inf_list) - else: - return agg_grads, None - - -# The following two functions are copied from -# tensorflow/python/eager/backprop.py. We do not directly use them as they are -# not exported and subject to change at any time. -def flatten_nested_indexed_slices(grad): - assert isinstance(grad, ops.IndexedSlices) - if isinstance(grad.values, ops.Tensor): - return grad - else: - assert isinstance(grad.values, ops.IndexedSlices) - g = flatten_nested_indexed_slices(grad.values) - return ops.IndexedSlices(g.values, array_ops.gather(grad.indices, - g.indices), - g.dense_shape) - - -def aggregate_indexed_slices_gradients(grads): - """Aggregates gradients containing `IndexedSlices`s.""" - if len(grads) < 1: - return None - elif len(grads) == 1: - return grads[0] - else: - grads = [g for g in grads if g is not None] - # If any gradient is a `Tensor`, sum them up and return a dense tensor - # object. - if any(isinstance(g, ops.Tensor) for g in grads): - return math_ops.add_n(grads) - - # The following `_as_indexed_slices_list` casts ids of IndexedSlices into - # int64. It is to make sure the inputs of `concat` all have same the data - # type. - grads = math_ops._as_indexed_slices_list(grads) # pylint: disable=protected-access - - grads = [flatten_nested_indexed_slices(x) for x in grads] - # Form IndexedSlices out of the concatenated values and indices. - concat_grad = ops.IndexedSlices( - array_ops.concat([x.values for x in grads], axis=0), - array_ops.concat([x.indices for x in grads], axis=0), - grads[0].dense_shape) - - return concat_grad - - -def aggregate_single_gradient_using_copy(grad_and_vars, use_mean, - check_inf_nan): - """Calculate the average gradient for a shared variable across all towers. - - Note that this function provides a synchronization point across all towers. - - Args: - grad_and_vars: A list or tuple of (gradient, variable) tuples. Each - (gradient, variable) pair within the outer list represents the gradient - of the variable calculated for a single tower, and the number of pairs - equals the number of towers. - use_mean: if True, mean is taken, else sum of gradients is taken. - check_inf_nan: check grads for nans and infs. - - Returns: - The tuple ([(average_gradient, variable),], has_nan_or_inf) where the - gradient has been averaged across all towers. The variable is chosen from - the first tower. The has_nan_or_inf indicates the grads has nan or inf. - """ - grads = [g for g, _ in grad_and_vars] - if any(isinstance(g, tf.IndexedSlices) for g in grads): - # TODO(reedwm): All-reduce IndexedSlices more effectively. - grad = aggregate_indexed_slices_gradients(grads) - else: - grad = tf.add_n(grads) - - if use_mean and len(grads) > 1: - grad = tf.scalar_mul(1.0 / len(grads), grad) - - v = grad_and_vars[0][1] - if check_inf_nan: - with tf.name_scope('check_for_inf_and_nan'): - has_nan_or_inf = tf.logical_not(tf.reduce_all(tf.is_finite(grads))) - return (grad, v), has_nan_or_inf - else: - return (grad, v), None - - -# This class is copied from -# https://github.com/tensorflow/tensorflow/blob/590d6eef7e91a6a7392c8ffffb7b58f2e0c8bc6b/tensorflow/contrib/training/python/training/device_setter.py#L56. -# We copy it since contrib has been removed from TensorFlow. -class GreedyLoadBalancingStrategy(object): - """Returns the least-loaded ps task for op placement. - - The load is calculated by a user-specified load function passed in at - construction. There are no units for load, and the load function is - responsible for providing an internally consistent measure. - - Note that this strategy is very sensitive to the exact order in which - ps ops (typically variables) are created, as it greedily places ops - on the least-loaded ps at the point each op is processed. - - One reasonable heuristic is the `byte_size_load_fn`, which - estimates load as the number of bytes that would be used to store and - transmit the entire variable. More advanced load functions - could consider the difference in access patterns across ops, or trade - off CPU-intensive ops with RAM-intensive ops with network bandwidth. - - This class is intended to be used as a `ps_strategy` in - `tf.compat.v1.train.replica_device_setter`. - """ - - def __init__(self, num_tasks, load_fn): - """Create a new `LoadBalancingStrategy`. - - Args: - num_tasks: Number of ps tasks to cycle among. - load_fn: A callable that takes an `Operation` and returns a - numeric load value for that op. - """ - self._num_tasks = num_tasks - self._load_fn = load_fn - self._ps_loads = np.zeros(num_tasks) - - def __call__(self, op): - """Choose a ps task index for the given `Operation`. - - Args: - op: A `Operation` to be placed on ps. - - Returns: - The next ps task index to use for the `Operation`. Greedily - places the op on the least-loaded ps task so far, as determined - by the load function. - """ - task = np.argmin(self._ps_loads) - self._ps_loads[task] += self._load_fn(op) - return task - - -# This function is copied from -# https://github.com/tensorflow/tensorflow/blob/590d6eef7e91a6a7392c8ffffb7b58f2e0c8bc6b/tensorflow/contrib/training/python/training/device_setter.py#L105. -# We copy it since contrib has been removed from TensorFlow. -def byte_size_load_fn(op): - """Load function that computes the byte size of a single-output `Operation`. - - This is intended to be used with `"Variable"` ops, which have a single - `Tensor` output with the contents of the variable. However, it can also be - used for calculating the size of any op that has a single output. - - Intended to be used with `GreedyLoadBalancingStrategy`. - - Args: - op: An `Operation` with a single output, typically a "Variable" op. - - Returns: - The number of bytes in the output `Tensor`. - - Raises: - ValueError: if `op` does not have a single output, or if the shape of the - single output is not fully-defined. - """ - if len(op.outputs) != 1: - raise ValueError('Op %s must have a single output' % op) - output = op.outputs[0] - elem_size = output.dtype.size - shape = output.get_shape() - if not shape.is_fully_defined(): - # Due to legacy behavior, scalar "Variable" ops have output Tensors that - # have unknown shape when the op is created (and hence passed to this - # load function for placement), even though the scalar shape is set - # explicitly immediately afterward. - shape = tensor_shape.TensorShape(op.get_attr('shape')) - shape.assert_is_fully_defined() - return shape.num_elements() * elem_size - diff --git a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/variable_mgr_util_test.py b/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/variable_mgr_util_test.py deleted file mode 100644 index 0915224f9681ab34daee03e01d12852b15d95298..0000000000000000000000000000000000000000 --- a/TensorFlow2x/ComputeVision/Classification/benchmarks-master/scripts/tf_cnn_benchmarks/variable_mgr_util_test.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright 2018 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for variable_mgr_util.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow.compat.v1 as tf -import variable_mgr_util - - -class VariableMgrUtilTest(tf.test.TestCase): - - def testGetLossScaleUpdateOpTruePath(self): - loss_scale = tf.Variable(4) - # loss_scale_normal_steps >= inc_loss_scale_every_n - loss_scale_normal_steps = tf.Variable(10) - inc_loss_scale_every_n = 10 - update_op = variable_mgr_util.get_loss_scale_update_op( - loss_scale, loss_scale_normal_steps, inc_loss_scale_every_n) - - with self.test_session() as sess: - sess.run(tf.global_variables_initializer()) - sess.run(update_op) - - self.assertEqual(sess.run(loss_scale), 8) - self.assertEqual(sess.run(loss_scale_normal_steps), 0) - - def testGetLossScaleUpdateOpFalsePath(self): - loss_scale = tf.Variable(4) - # loss_scale_normal_steps < inc_loss_scale_every_n - loss_scale_normal_steps = tf.Variable(9) - inc_loss_scale_every_n = 10 - update_op = variable_mgr_util.get_loss_scale_update_op( - loss_scale, loss_scale_normal_steps, inc_loss_scale_every_n) - - with self.test_session() as sess: - sess.run(tf.global_variables_initializer()) - sess.run(update_op) - - self.assertEqual(sess.run(loss_scale), 4) - self.assertEqual(sess.run(loss_scale_normal_steps), 10) - - def testAppendGradientsWithLossScaleWithAutoScaleDisabled(self): - v = tf.Variable(0) - training_ops = [] - get_apply_gradients_ops_func = lambda: [tf.assign(v, v + 1)] - loss_scale_params = variable_mgr_util.AutoLossScaleParams( - enable_auto_loss_scale=False, # no auto loss scale. - loss_scale=tf.Variable(4), - loss_scale_normal_steps=tf.Variable(10), - inc_loss_scale_every_n=10, - is_chief=True) - variable_mgr_util.append_gradients_with_loss_scale( - training_ops, - get_apply_gradients_ops_func, - loss_scale_params, - grad_has_inf_nan=True) - - with self.test_session() as sess: - sess.run(tf.global_variables_initializer()) - sess.run(training_ops) - self.assertEqual(sess.run(v), 1) - self.assertEqual(sess.run(loss_scale_params.loss_scale), 4) - self.assertEqual(sess.run(loss_scale_params.loss_scale_normal_steps), 10) - - def testAppendGradientsWithLossScaleForNonChiefWorker(self): - v = tf.Variable(0) - training_ops = [] - get_apply_gradients_ops_func = lambda: [tf.assign(v, v + 1)] - loss_scale_params = variable_mgr_util.AutoLossScaleParams( - enable_auto_loss_scale=True, - loss_scale=tf.Variable(4), - loss_scale_normal_steps=tf.Variable(10), - inc_loss_scale_every_n=10, - is_chief=False) # Non-chief - variable_mgr_util.append_gradients_with_loss_scale( - training_ops, - get_apply_gradients_ops_func, - loss_scale_params, - grad_has_inf_nan=False) - - with self.test_session() as sess: - sess.run(tf.global_variables_initializer()) - sess.run(training_ops) - self.assertEqual(sess.run(v), 1) - self.assertEqual(sess.run(loss_scale_params.loss_scale), 4) - self.assertEqual(sess.run(loss_scale_params.loss_scale_normal_steps), 10) - - def testAppendGradientsWithLossScaleWithoutNan(self): - v = tf.Variable(0) - training_ops = [] - get_apply_gradients_ops_func = lambda: [tf.assign(v, v + 1)] - loss_scale_params = variable_mgr_util.AutoLossScaleParams( - enable_auto_loss_scale=True, - loss_scale=tf.Variable(4, dtype=tf.float32), - loss_scale_normal_steps=tf.Variable(10), - inc_loss_scale_every_n=10, - is_chief=True) - variable_mgr_util.append_gradients_with_loss_scale( - training_ops, - get_apply_gradients_ops_func, - loss_scale_params, - grad_has_inf_nan=tf.constant(False)) - - with self.test_session() as sess: - sess.run(tf.global_variables_initializer()) - sess.run(training_ops) - self.assertEqual(sess.run(v), 1) - self.assertEqual(sess.run(loss_scale_params.loss_scale), 8) - self.assertEqual(sess.run(loss_scale_params.loss_scale_normal_steps), 0) - - def testAppendGradientsWithLossScaleWithtNan(self): - v = tf.Variable(0) - training_ops = [] - get_apply_gradients_ops_func = lambda: [tf.assign(v, v + 1)] - loss_scale_params = variable_mgr_util.AutoLossScaleParams( - enable_auto_loss_scale=True, - loss_scale=tf.Variable(4, dtype=tf.float32), - loss_scale_normal_steps=tf.Variable(10), - inc_loss_scale_every_n=10, - is_chief=True) - variable_mgr_util.append_gradients_with_loss_scale( - training_ops, - get_apply_gradients_ops_func, - loss_scale_params, - grad_has_inf_nan=tf.constant(True)) - - with self.test_session() as sess: - sess.run(tf.global_variables_initializer()) - sess.run(training_ops) - self.assertEqual(sess.run(v), 0) # Skip updating for v. - # halve loss_scale and reset local_scale_normal_steps. - self.assertEqual(sess.run(loss_scale_params.loss_scale), 2) - self.assertEqual(sess.run(loss_scale_params.loss_scale_normal_steps), 0) - - -if __name__ == '__main__': - tf.disable_v2_behavior() - tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/configfile b/TensorFlow2x/ComputeVision/Classification/configfile new file mode 100644 index 0000000000000000000000000000000000000000..03766eb1902e1fd98c922d5a5fd3ff2a1c2e33ea --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/configfile @@ -0,0 +1,22 @@ + + worker = [] + port_number =40000 + + filePath='./nodefile' + with open(filePath, 'r') as f: + nodename = f.read().splitlines() + + worker_nodes = nodename + num_index=flags_obj.task_index + + for node in worker_nodes: + for index in range(4): + worker_sockets = ":".join([node, str(port_number + index )]) + worker.append(worker_sockets) + os.environ['TF_CONFIG'] = json.dumps({ + 'cluster': { + 'worker': worker + }, + 'task': {'type': 'worker', 'index': num_index} + }) + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/00-official-bug-report-issue.md b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/00-official-bug-report-issue.md new file mode 100644 index 0000000000000000000000000000000000000000..51e08c26db66114de0b604bf0cc5c461311a0b4f --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/00-official-bug-report-issue.md @@ -0,0 +1,59 @@ +--- +name: "[Official Model] Bug Report" +about: Use this template for reporting a bug for the “official” directory +labels: type:bug,models:official + +--- + +# Prerequisites + +Please answer the following questions for yourself before submitting an issue. + +- [ ] I am using the latest TensorFlow Model Garden release and TensorFlow 2. +- [ ] I am reporting the issue to the correct repository. (Model Garden official or research directory) +- [ ] I checked to make sure that this issue has not been filed already. + +## 1. The entire URL of the file you are using + +https://github.com/tensorflow/models/tree/master/official/... + +## 2. Describe the bug + +A clear and concise description of what the bug is. + +## 3. Steps to reproduce + +Steps to reproduce the behavior. + +## 4. Expected behavior + +A clear and concise description of what you expected to happen. + +## 5. Additional context + +Include any logs that would be helpful to diagnose the problem. + +## 6. System information + +- OS Platform and Distribution (e.g., Linux Ubuntu 16.04): +- Mobile device name if the issue happens on a mobile device: +- TensorFlow installed from (source or binary): +- TensorFlow version (use command below): +- Python version: +- Bazel version (if compiling from source): +- GCC/Compiler version (if compiling from source): +- CUDA/cuDNN version: +- GPU model and memory: + + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/10-official-documentation-issue.md b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/10-official-documentation-issue.md new file mode 100644 index 0000000000000000000000000000000000000000..00d79a16916c327d2d8a729791db7d7d3d96b735 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/10-official-documentation-issue.md @@ -0,0 +1,20 @@ +--- +name: "[Official Model] Documentation Issue" +about: Use this template for reporting a documentation issue for the “official” directory +labels: type:docs,models:official + +--- + +# Prerequisites + +Please answer the following question for yourself before submitting an issue. + +- [ ] I checked to make sure that this issue has not been filed already. + +## 1. The entire URL of the documentation with the issue + +https://github.com/tensorflow/models/tree/master/official/... + +## 2. Describe the issue + +A clear and concise description of what needs to be changed. diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/20-official-feature-request-issue.md b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/20-official-feature-request-issue.md new file mode 100644 index 0000000000000000000000000000000000000000..02d8cab52218202707646345a4ab2570519660dd --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/20-official-feature-request-issue.md @@ -0,0 +1,26 @@ +--- +name: "[Official Model] Feature request" +about: Use this template for raising a feature request for the “official” directory +labels: type:feature,models:official + +--- + +# Prerequisites + +Please answer the following question for yourself before submitting an issue. + +- [ ] I checked to make sure that this feature has not been requested already. + +## 1. The entire URL of the file you are using + +https://github.com/tensorflow/models/tree/master/official/... + +## 2. Describe the feature you request + +A clear and concise description of what you want to happen. + +## 3. Additional context + +Add any other context about the feature request here. + +## 4. Are you willing to contribute it? (Yes or No) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/30-research-bug-report-issue.md b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/30-research-bug-report-issue.md new file mode 100644 index 0000000000000000000000000000000000000000..4448ed9e40d6a089b84881635c2ee0f53524ae61 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/30-research-bug-report-issue.md @@ -0,0 +1,58 @@ +--- +name: "[Research Model] Bug Report" +about: Use this template for reporting a bug for the “research” directory +labels: type:bug,models:research + +--- +# Prerequisites + +Please answer the following questions for yourself before submitting an issue. + +- [ ] I am using the latest TensorFlow Model Garden release and TensorFlow 2. +- [ ] I am reporting the issue to the correct repository. (Model Garden official or research directory) +- [ ] I checked to make sure that this issue has not already been filed. + +## 1. The entire URL of the file you are using + +https://github.com/tensorflow/models/tree/master/research/... + +## 2. Describe the bug + +A clear and concise description of what the bug is. + +## 3. Steps to reproduce + +Steps to reproduce the behavior. + +## 4. Expected behavior + +A clear and concise description of what you expected to happen. + +## 5. Additional context + +Include any logs that would be helpful to diagnose the problem. + +## 6. System information + +- OS Platform and Distribution (e.g., Linux Ubuntu 16.04): +- Mobile device name if the issue happens on a mobile device: +- TensorFlow installed from (source or binary): +- TensorFlow version (use command below): +- Python version: +- Bazel version (if compiling from source): +- GCC/Compiler version (if compiling from source): +- CUDA/cuDNN version: +- GPU model and memory: + + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/40-research-documentation-issue.md b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/40-research-documentation-issue.md new file mode 100644 index 0000000000000000000000000000000000000000..26adfd83e1fbe27d045ecd8dfccef91bbd27fcf1 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/40-research-documentation-issue.md @@ -0,0 +1,20 @@ +--- +name: "[Research Model] Documentation Issue" +about: Use this template for reporting a documentation issue for the “research” directory +labels: type:docs,models:research + +--- + +# Prerequisites + +Please answer the following question for yourself before submitting an issue. + +- [ ] I checked to make sure that this issue has not been filed already. + +## 1. The entire URL of the documentation with the issue + +https://github.com/tensorflow/models/tree/master/research/... + +## 2. Describe the issue + +A clear and concise description of what needs to be changed. diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/50-research-feature-request-issue.md b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/50-research-feature-request-issue.md new file mode 100644 index 0000000000000000000000000000000000000000..412942a31be9cc4c2935dcd38ecb059a8a4ec18c --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/50-research-feature-request-issue.md @@ -0,0 +1,26 @@ +--- +name: "[Research Model] Feature Request" +about: Use this template for raising a feature request for the “research” directory +labels: type:feature,models:research + +--- + +# Prerequisites + +Please answer the following question for yourself before submitting an issue. + +- [ ] I checked to make sure that this feature has not been requested already. + +## 1. The entire URL of the file you are using + +https://github.com/tensorflow/models/tree/master/research/... + +## 2. Describe the feature you request + +A clear and concise description of what you want to happen. + +## 3. Additional context + +Add any other context about the feature request here. + +## 4. Are you willing to contribute it? (Yes or No) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/60-questions-help-issue.md b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/60-questions-help-issue.md new file mode 100644 index 0000000000000000000000000000000000000000..bc85e0bb019fd2d5960b822c18358f906d5264b7 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/60-questions-help-issue.md @@ -0,0 +1,14 @@ +--- +name: Questions and Help +about: Use this template for Questions and Help. +labels: type:support + +--- + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/config.yml b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..3ba13e0cec6cbbfd462e9ebf529dd2093148cd69 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/.github/PULL_REQUEST_TEMPLATE.md b/TensorFlow2x/ComputeVision/Classification/models-master/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000000000000000000000000000000000..379b31c57c118a174d4e787e03099288957f9fe2 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,41 @@ +# Description + +> :memo: Please include a summary of the change. +> +> * Please also include relevant motivation and context. +> * List any dependencies that are required for this change. + +## Type of change + +For a new feature or function, please create an issue first to discuss it +with us before submitting a pull request. + +Note: Please delete options that are not relevant. + +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] Documentation update +- [ ] TensorFlow 2 migration +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] A new research paper code implementation +- [ ] Other (Specify) + +## Tests + +> :memo: Please describe the tests that you ran to verify your changes. +> +> * Provide instructions so we can reproduce. +> * Please also list any relevant details for your test configuration. + +**Test Configuration**: + +## Checklist + +- [ ] I have signed the [Contributor License Agreement](https://github.com/tensorflow/models/wiki/Contributor-License-Agreements). +- [ ] I have read [guidelines for pull request](https://github.com/tensorflow/models/wiki/Submitting-a-pull-request). +- [ ] My code follows the [coding guidelines](https://github.com/tensorflow/models/wiki/Coding-guidelines). +- [ ] I have performed a self [code review](https://github.com/tensorflow/models/wiki/Code-review) of my own code. +- [ ] I have commented my code, particularly in hard-to-understand areas. +- [ ] I have made corresponding changes to the documentation. +- [ ] My changes generate no new warnings. +- [ ] I have added tests that prove my fix is effective or that my feature works. diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/.github/README_TEMPLATE.md b/TensorFlow2x/ComputeVision/Classification/models-master/.github/README_TEMPLATE.md new file mode 100644 index 0000000000000000000000000000000000000000..43dba40f59684df0f79faa341c8de67916313210 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/.github/README_TEMPLATE.md @@ -0,0 +1,124 @@ +> :memo: A README.md template for releasing a paper code implementation to a GitHub repository. +> +> * Template version: 1.0.2020.170 +> * Please modify sections depending on needs. + +# Model name, Paper title, or Project Name + +> :memo: Add a badge for the ArXiv identifier of your paper (arXiv:YYMM.NNNNN) + +[![Paper](http://img.shields.io/badge/Paper-arXiv.YYMM.NNNNN-B3181B?logo=arXiv)](https://arxiv.org/abs/...) + +This repository is the official or unofficial implementation of the following paper. + +* Paper title: [Paper Title](https://arxiv.org/abs/YYMM.NNNNN) + +## Description + +> :memo: Provide description of the model. +> +> * Provide brief information of the algorithms used. +> * Provide links for demos, blog posts, etc. + +## History + +> :memo: Provide a changelog. + +## Authors or Maintainers + +> :memo: Provide maintainer information. + +* Full name ([@GitHub username](https://github.com/username)) +* Full name ([@GitHub username](https://github.com/username)) + +## Table of Contents + +> :memo: Provide a table of contents to help readers navigate a lengthy README document. + +## Requirements + +[![TensorFlow 2.1](https://img.shields.io/badge/TensorFlow-2.1-FF6F00?logo=tensorflow)](https://github.com/tensorflow/tensorflow/releases/tag/v2.1.0) +[![Python 3.6](https://img.shields.io/badge/Python-3.6-3776AB)](https://www.python.org/downloads/release/python-360/) + +> :memo: Provide details of the software required. +> +> * Add a `requirements.txt` file to the root directory for installing the necessary dependencies. +> * Describe how to install requirements using pip. +> * Alternatively, create INSTALL.md. + +To install requirements: + +```setup +pip install -r requirements.txt +``` + +## Results + +[![TensorFlow Hub](https://img.shields.io/badge/TF%20Hub-Models-FF6F00?logo=tensorflow)](https://tfhub.dev/...) + +> :memo: Provide a table with results. (e.g., accuracy, latency) +> +> * Provide links to the pre-trained models (checkpoint, SavedModel files). +> * Publish TensorFlow SavedModel files on TensorFlow Hub (tfhub.dev) if possible. +> * Add links to [TensorBoard.dev](https://tensorboard.dev/) for visualizing metrics. +> +> An example table for image classification results +> +> ### Image Classification +> +> | Model name | Download | Top 1 Accuracy | Top 5 Accuracy | +> |------------|----------|----------------|----------------| +> | Model name | [Checkpoint](https://drive.google.com/...), [SavedModel](https://tfhub.dev/...) | xx% | xx% | + +## Dataset + +> :memo: Provide information of the dataset used. + +## Training + +> :memo: Provide training information. +> +> * Provide details for preprocessing, hyperparameters, random seeds, and environment. +> * Provide a command line example for training. + +Please run this command line for training. + +```shell +python3 ... +``` + +## Evaluation + +> :memo: Provide an evaluation script with details of how to reproduce results. +> +> * Describe data preprocessing / postprocessing steps. +> * Provide a command line example for evaluation. + +Please run this command line for evaluation. + +```shell +python3 ... +``` + +## References + +> :memo: Provide links to references. + +## License + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +> :memo: Place your license text in a file named LICENSE in the root of the repository. +> +> * Include information about your license. +> * Reference: [Adding a license to a repository](https://help.github.com/en/github/building-a-strong-community/adding-a-license-to-a-repository) + +This project is licensed under the terms of the **Apache License 2.0**. + +## Citation + +> :memo: Make your repository citable. +> +> * Reference: [Making Your Code Citable](https://guides.github.com/activities/citable-code/) + +If you want to cite this repository in your research paper, please use the following information. diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/.github/bot_config.yml b/TensorFlow2x/ComputeVision/Classification/models-master/.github/bot_config.yml new file mode 100644 index 0000000000000000000000000000000000000000..952afc316e78d823f865ef651981fda1dde32097 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/.github/bot_config.yml @@ -0,0 +1,24 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +# +# THIS IS A GENERATED DOCKERFILE. +# +# This file was assembled from multiple pieces, whose use is documented +# throughout. Please refer to the TensorFlow dockerfiles documentation +# for more information. + +# A list of assignees +assignees: + - saikumarchalla diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/.github/scripts/pylint.sh b/TensorFlow2x/ComputeVision/Classification/models-master/.github/scripts/pylint.sh new file mode 100644 index 0000000000000000000000000000000000000000..bb2ebebd8a87199a2138ef513cfd930af5b822bf --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/.github/scripts/pylint.sh @@ -0,0 +1,178 @@ +#!/bin/bash +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# +# Pylint wrapper extracted from main TensorFlow, sharing same exceptions. +# Specify --incremental to only check files touched since last commit on master, +# otherwise will recursively check current directory (full repo takes long!). + +set -euo pipefail + +# Download latest configs from main TensorFlow repo. +wget -q -O /tmp/pylintrc https://raw.githubusercontent.com/tensorflow/tensorflow/master/tensorflow/tools/ci_build/pylintrc + +SCRIPT_DIR=/tmp + +num_cpus() { + # Get the number of CPUs + if [[ -f /proc/cpuinfo ]]; then + N_CPUS=$(grep -c ^processor /proc/cpuinfo) + else + # Fallback method + N_CPUS=`getconf _NPROCESSORS_ONLN` + fi + if [[ -z ${N_CPUS} ]]; then + die "ERROR: Unable to determine the number of CPUs" + fi + + echo ${N_CPUS} +} + +get_changed_files_in_last_non_merge_git_commit() { + git diff --name-only $(git merge-base master $(git branch --show-current)) +} + +# List Python files changed in the last non-merge git commit that still exist, +# i.e., not removed. +# Usage: get_py_files_to_check [--incremental] +get_py_files_to_check() { + if [[ "$1" == "--incremental" ]]; then + CHANGED_PY_FILES=$(get_changed_files_in_last_non_merge_git_commit | \ + grep '.*\.py$') + + # Do not include files removed in the last non-merge commit. + PY_FILES="" + for PY_FILE in ${CHANGED_PY_FILES}; do + if [[ -f "${PY_FILE}" ]]; then + PY_FILES="${PY_FILES} ${PY_FILE}" + fi + done + + echo "${PY_FILES}" + else + find . -name '*.py' + fi +} + +do_pylint() { + if [[ $# == 1 ]] && [[ "$1" == "--incremental" ]]; then + PYTHON_SRC_FILES=$(get_py_files_to_check --incremental) + + if [[ -z "${PYTHON_SRC_FILES}" ]]; then + echo "do_pylint will NOT run due to --incremental flag and due to the "\ +"absence of Python code changes in the last commit." + return 0 + fi + elif [[ $# != 0 ]]; then + echo "Invalid syntax for invoking do_pylint" + echo "Usage: do_pylint [--incremental]" + return 1 + else + PYTHON_SRC_FILES=$(get_py_files_to_check) + fi + + # Something happened. TF no longer has Python code if this branch is taken + if [[ -z ${PYTHON_SRC_FILES} ]]; then + echo "do_pylint found no Python files to check. Returning." + return 0 + fi + + # Now that we know we have to do work, check if `pylint` is installed + PYLINT_BIN="python3.8 -m pylint" + + echo "" + echo "check whether pylint is available or not." + echo "" + ${PYLINT_BIN} --version + if [[ $? -eq 0 ]] + then + echo "" + echo "pylint available, proceeding with pylint sanity check." + echo "" + else + echo "" + echo "pylint not available." + echo "" + return 1 + fi + + # Configure pylint using the following file + PYLINTRC_FILE="${SCRIPT_DIR}/pylintrc" + + if [[ ! -f "${PYLINTRC_FILE}" ]]; then + die "ERROR: Cannot find pylint rc file at ${PYLINTRC_FILE}" + fi + + # Run pylint in parallel, after some disk setup + NUM_SRC_FILES=$(echo ${PYTHON_SRC_FILES} | wc -w) + NUM_CPUS=$(num_cpus) + + echo "Running pylint on ${NUM_SRC_FILES} files with ${NUM_CPUS} "\ +"parallel jobs..." + echo "" + + PYLINT_START_TIME=$(date +'%s') + OUTPUT_FILE="$(mktemp)_pylint_output.log" + ERRORS_FILE="$(mktemp)_pylint_errors.log" + + rm -rf ${OUTPUT_FILE} + rm -rf ${ERRORS_FILE} + + set +e + # When running, filter to only contain the error code lines. Removes module + # header, removes lines of context that show up from some lines. + # Also, don't redirect stderr as this would hide pylint fatal errors. + ${PYLINT_BIN} --rcfile="${PYLINTRC_FILE}" --output-format=parseable \ + --jobs=${NUM_CPUS} ${PYTHON_SRC_FILES} | grep '\[[CEFW]' > ${OUTPUT_FILE} + PYLINT_END_TIME=$(date +'%s') + + echo "" + echo "pylint took $((PYLINT_END_TIME - PYLINT_START_TIME)) s" + echo "" + + # Report only what we care about + # Ref https://pylint.readthedocs.io/en/latest/technical_reference/features.html + # E: all errors + # W0311 bad-indentation + # W0312 mixed-indentation + # C0330 bad-continuation + # C0301 line-too-long + # C0326 bad-whitespace + # W0611 unused-import + # W0622 redefined-builtin + grep -E '(\[E|\[W0311|\[W0312|\[C0330|\[C0301|\[C0326|\[W0611|\[W0622)' ${OUTPUT_FILE} > ${ERRORS_FILE} + + # Determine counts of errors + N_FORBID_ERRORS=$(wc -l ${ERRORS_FILE} | cut -d' ' -f1) + set -e + + # Now, print the errors we should fix + echo "" + if [[ ${N_FORBID_ERRORS} != 0 ]]; then + echo "Found ${N_FORBID_ERRORS} pylint errors:" + cat ${ERRORS_FILE} + fi + + echo "" + if [[ ${N_FORBID_ERRORS} != 0 ]]; then + echo "FAIL: Found ${N_FORBID_ERRORS} errors" + return 1 + else + echo "PASS: Found no errors" + fi +} + +do_pylint "$@" + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/.github/stale.yml b/TensorFlow2x/ComputeVision/Classification/models-master/.github/stale.yml new file mode 100644 index 0000000000000000000000000000000000000000..7eef5309ecdf53125eb976f90c3b62f1a31a55d4 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/.github/stale.yml @@ -0,0 +1,39 @@ + # Copyright 2019 The TensorFlow Authors. All Rights Reserved. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + # ============================================================================ + # + # THIS IS A GENERATED DOCKERFILE. + # + # This file was assembled from multiple pieces, whose use is documented + # throughout. Please refer to the TensorFlow dockerfiles documentation + # for more information. + +# Number of days of inactivity before an Issue or Pull Request becomes stale +daysUntilStale: 7 +# Number of days of inactivity before a stale Issue or Pull Request is closed +daysUntilClose: 7 +# Only issues or pull requests with all of these labels are checked if stale. Defaults to `[]` (disabled) +onlyLabels: + - stat:awaiting response +# Comment to post when marking as stale. Set to `false` to disable +markComment: > + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed if no further activity occurs. Thank you. +# Comment to post when removing the stale label. Set to `false` to disable +unmarkComment: false +closeComment: > + Closing as stale. Please reopen if you'd like to work on this further. +limitPerRun: 30 +# Limit to only `issues` or `pulls` +only: issues diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/.github/workflows/ci.yml b/TensorFlow2x/ComputeVision/Classification/models-master/.github/workflows/ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..744f440b053ddb5391a827b5406ddb9ad94eccef --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/.github/workflows/ci.yml @@ -0,0 +1,29 @@ +name: CI +on: pull_request + +jobs: + pylint: + runs-on: ubuntu-latest + + steps: + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install pylint 2.4.4 + run: | + python -m pip install --upgrade pip + pip install pylint==2.4.4 + + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Fetch master for diff + run: git fetch origin master:master + + - name: Run pylint script + run: bash ./.github/scripts/pylint.sh --incremental diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/.gitignore b/TensorFlow2x/ComputeVision/Classification/models-master/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..cbc8846d64152b8a933f4bd2727877a94f98f92a --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/.gitignore @@ -0,0 +1,98 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# mypy +.mypy_cache + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject + +# PyCharm +.idea/ + +# For mac +.DS_Store diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/AUTHORS b/TensorFlow2x/ComputeVision/Classification/models-master/AUTHORS new file mode 100644 index 0000000000000000000000000000000000000000..0fa85c98ffeb38c6d6d0ef2bddb790b75b90f3dc --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/AUTHORS @@ -0,0 +1,10 @@ +# This is the official list of authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. + +# Names should be added to this file as: +# Name or Organization +# The email address is not required for organizations. + +Google Inc. +David Dao diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/CODEOWNERS b/TensorFlow2x/ComputeVision/Classification/models-master/CODEOWNERS new file mode 100644 index 0000000000000000000000000000000000000000..a46d472ca3545921ad4f76283c79ea4ca2e8a1e9 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/CODEOWNERS @@ -0,0 +1,29 @@ +* @tensorflow/tf-model-garden-team +/official/ @rachellj218 @saberkun @jaeyounkim +/official/nlp/ @saberkun @lehougoogle @rachellj218 @jaeyounkim +/official/recommendation/ranking/ @gagika +/official/vision/ @xianzhidu @yeqingli @arashwan @saberkun @rachellj218 @jaeyounkim +/official/vision/beta/projects/assemblenet/ @mryoo @yeqingli +/official/vision/beta/projects/deepmac_maskrcnn/ @vighneshbirodkar +/official/vision/beta/projects/movinet/ @hyperparticle @yuanliangzhe @yeqingli +/official/vision/beta/projects/simclr/ @luotigerlsx @chentingpc @saxenasaurabh +/official/vision/beta/projects/video_ssl/ @richardaecn @yeqingli +/research/adversarial_text/ @rsepassi @a-dai +/research/attention_ocr/ @xavigibert +/research/audioset/ @plakal @dpwe +/research/autoaugment/ @barretzoph +/research/cognitive_planning/ @s-gupta +/research/cvt_text/ @clarkkev @lmthang +/research/deep_speech/ @yhliang2018 +/research/deeplab/ @aquariusjay @yknzhu +/research/delf/ @andrefaraujo +/research/efficient-hrl/ @ofirnachum +/research/lfads/ @jazcollins @sussillo +/research/lstm_object_detection/ @yinxiaoli @yongzhe2160 +/research/marco/ @vincentvanhoucke +/research/object_detection/ @jch1 @tombstone @pkulzc +/research/pcl_rl/ @ofirnachum +/research/rebar/ @gjtucker +/research/seq_flow_lite/ @thunderfyc +/research/slim/ @sguada @marksandler2 +/research/vid2depth/ @rezama diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/CONTRIBUTING.md b/TensorFlow2x/ComputeVision/Classification/models-master/CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..f909461ae7b9c75264e0915ecb37228314933e4a --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/CONTRIBUTING.md @@ -0,0 +1,10 @@ +# How to contribute + +![Contributors](https://img.shields.io/github/contributors/tensorflow/models) + +We encourage you to contribute to the TensorFlow Model Garden. + +Please read our [guidelines](../../wiki/How-to-contribute) for details. + +**NOTE**: Only [code owners](./CODEOWNERS) are allowed to merge a pull request. +Please contact the code owners of each model to merge your pull request. diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/ISSUES.md b/TensorFlow2x/ComputeVision/Classification/models-master/ISSUES.md new file mode 100644 index 0000000000000000000000000000000000000000..b23d6daa1654188d640beb67e6614bd0743f919f --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/ISSUES.md @@ -0,0 +1,24 @@ +# If you open a GitHub issue, here is our policy. + +* It must be a **bug**, a **feature request**, or a significant problem +with **documentation**. + * Please send a pull request instead for small documentation fixes. +* The required form must be filled out. +* The issue should be related to the repository it is created in. + +General help and support should be sought on [Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow-model-garden) or other non-GitHub channels. + +[![](https://img.shields.io/stackexchange/stackoverflow/t/tensorflow-model-garden)](https://stackoverflow.com/questions/tagged/tensorflow-model-garden) + +TensorFlow developers respond to issues. +We want to focus on work that benefits the whole community such as fixing bugs +and adding new features. +It helps us to address bugs and feature requests in a timely manner. + +--- + +Please understand that research models in the [research directory](https://github.com/tensorflow/models/tree/master/research) +included in this repository are experimental and research-style code. +They are not officially supported by the TensorFlow team. + + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/LICENSE b/TensorFlow2x/ComputeVision/Classification/models-master/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..43fcf7bf1f1f9f824a1debf05d6ced45bf5810aa --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/LICENSE @@ -0,0 +1,203 @@ +Copyright 2016 The TensorFlow Authors. All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016, The Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/README.md b/TensorFlow2x/ComputeVision/Classification/models-master/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a27d3e7bedb48be508c920c5177e22647c4cd616 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/README.md @@ -0,0 +1,47 @@ +![Logo](https://storage.googleapis.com/tf_model_garden/tf_model_garden_logo.png) + +# Welcome to the Model Garden for TensorFlow + +The TensorFlow Model Garden is a repository with a number of different +implementations of state-of-the-art (SOTA) models and modeling solutions for +TensorFlow users. We aim to demonstrate the best practices for modeling so that +TensorFlow users can take full advantage of TensorFlow for their research and +product development. + +To improve the transparency and reproducibility of our models, training logs on +[TensorBoard.dev](https://tensorboard.dev) are also provided for models to the +extent possible though not all models are suitable. + +| Directory | Description | +|-----------|-------------| +| [official](official) | • A collection of example implementations for SOTA models using the latest TensorFlow 2's high-level APIs
• Officially maintained, supported, and kept up to date with the latest TensorFlow 2 APIs by TensorFlow
• Reasonably optimized for fast performance while still being easy to read | +| [research](research) | • A collection of research model implementations in TensorFlow 1 or 2 by researchers
• Maintained and supported by researchers | +| [community](community) | • A curated list of the GitHub repositories with machine learning models and implementations powered by TensorFlow 2 | +| [orbit](orbit) | • A flexible and lightweight library that users can easily use or fork when writing customized training loop code in TensorFlow 2.x. It seamlessly integrates with `tf.distribute` and supports running on different device types (CPU, GPU, and TPU). | + +## [Announcements](https://github.com/tensorflow/models/wiki/Announcements) + +## Contributions + +[![help wanted:paper implementation](https://img.shields.io/github/issues/tensorflow/models/help%20wanted%3Apaper%20implementation)](https://github.com/tensorflow/models/labels/help%20wanted%3Apaper%20implementation) + +If you want to contribute, please review the [contribution guidelines](https://github.com/tensorflow/models/wiki/How-to-contribute). + +## License + +[Apache License 2.0](LICENSE) + +## Citing TensorFlow Model Garden + +If you use TensorFlow Model Garden in your research, please cite this repository. + +``` +@misc{tensorflowmodelgarden2020, + author = {Hongkun Yu and Chen Chen and Xianzhi Du and Yeqing Li and + Abdullah Rashwan and Le Hou and Pengchong Jin and Fan Yang and + Frederick Liu and Jaeyoun Kim and Jing Li}, + title = {{TensorFlow Model Garden}}, + howpublished = {\url{https://github.com/tensorflow/models}}, + year = {2020} +} +``` diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/community/README.md b/TensorFlow2x/ComputeVision/Classification/models-master/community/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b3f2bac74897d440d7e16efadcae45b8b5e46249 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/community/README.md @@ -0,0 +1,60 @@ +![Logo](https://storage.googleapis.com/tf_model_garden/tf_model_garden_logo.png) + +# TensorFlow Community Models + +This repository provides a curated list of the GitHub repositories with machine learning models and implementations powered by TensorFlow 2. + +**Note**: Contributing companies or individuals are responsible for maintaining their repositories. + +## Computer Vision + +### Image Recognition + +| Model | Paper | Features | Maintainer | +|-------|-------|----------|------------| +| [DenseNet 169](https://github.com/IntelAI/models/tree/master/benchmarks/image_recognition/tensorflow/densenet169) | [Densely Connected Convolutional Networks](https://arxiv.org/pdf/1608.06993) | • FP32 Inference | [Intel](https://github.com/IntelAI) | +| [Inception V3](https://github.com/IntelAI/models/tree/master/benchmarks/image_recognition/tensorflow/inceptionv3) | [Rethinking the Inception Architecture
for Computer Vision](https://arxiv.org/pdf/1512.00567.pdf) | • Int8 Inference
• FP32 Inference | [Intel](https://github.com/IntelAI) | +| [Inception V4](https://github.com/IntelAI/models/tree/master/benchmarks/image_recognition/tensorflow/inceptionv4) | [Inception-v4, Inception-ResNet and the Impact
of Residual Connections on Learning](https://arxiv.org/pdf/1602.07261) | • Int8 Inference
• FP32 Inference | [Intel](https://github.com/IntelAI) | +| [MobileNet V1](https://github.com/IntelAI/models/tree/master/benchmarks/image_recognition/tensorflow/mobilenet_v1) | [MobileNets: Efficient Convolutional Neural Networks
for Mobile Vision Applications](https://arxiv.org/pdf/1704.04861) | • Int8 Inference
• FP32 Inference | [Intel](https://github.com/IntelAI) | +| [ResNet 101](https://github.com/IntelAI/models/tree/master/benchmarks/image_recognition/tensorflow/resnet101) | [Deep Residual Learning for Image Recognition](https://arxiv.org/pdf/1512.03385) | • Int8 Inference
• FP32 Inference | [Intel](https://github.com/IntelAI) | +| [ResNet 50](https://github.com/IntelAI/models/tree/master/benchmarks/image_recognition/tensorflow/resnet50) | [Deep Residual Learning for Image Recognition](https://arxiv.org/pdf/1512.03385) | • Int8 Inference
• FP32 Inference | [Intel](https://github.com/IntelAI) | +| [ResNet 50v1.5](https://github.com/IntelAI/models/tree/master/benchmarks/image_recognition/tensorflow/resnet50v1_5) | [Deep Residual Learning for Image Recognition](https://arxiv.org/pdf/1512.03385) | • Int8 Inference
• FP32 Inference
• FP32 Training | [Intel](https://github.com/IntelAI) | +| [EfficientNet](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow2/Classification/ConvNets/efficientnet) | [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/pdf/1905.11946.pdf) | • Automatic mixed precision
• Horovod Multi-GPU training (NCCL)
• Multi-node training on a Pyxis/Enroot Slurm cluster
• XLA | [NVIDIA](https://github.com/NVIDIA) | + +### Object Detection + +| Model | Paper | Features | Maintainer | +|-------|-------|----------|------------| +| [R-FCN](https://github.com/IntelAI/models/tree/master/benchmarks/object_detection/tensorflow/rfcn) | [R-FCN: Object Detection
via Region-based Fully Convolutional Networks](https://arxiv.org/pdf/1605.06409) | • Int8 Inference
• FP32 Inference | [Intel](https://github.com/IntelAI) | +| [SSD-MobileNet](https://github.com/IntelAI/models/tree/master/benchmarks/object_detection/tensorflow/ssd-mobilenet) | [MobileNets: Efficient Convolutional Neural Networks
for Mobile Vision Applications](https://arxiv.org/pdf/1704.04861) | • Int8 Inference
• FP32 Inference | [Intel](https://github.com/IntelAI) | +| [SSD-ResNet34](https://github.com/IntelAI/models/tree/master/benchmarks/object_detection/tensorflow/ssd-resnet34) | [SSD: Single Shot MultiBox Detector](https://arxiv.org/pdf/1512.02325) | • Int8 Inference
• FP32 Inference
• FP32 Training | [Intel](https://github.com/IntelAI) | + +### Segmentation + +| Model | Paper | Features | Maintainer | +|-------|-------|----------|------------| +| [Mask R-CNN](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow2/Segmentation/MaskRCNN) | [Mask R-CNN](https://arxiv.org/abs/1703.06870) | • Automatic Mixed Precision
• Multi-GPU training support with Horovod
• TensorRT | [NVIDIA](https://github.com/NVIDIA) | +| [U-Net Medical Image Segmentation](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow2/Segmentation/UNet_Medical) | [U-Net: Convolutional Networks for Biomedical Image Segmentation](https://arxiv.org/abs/1505.04597) | • Automatic Mixed Precision
• Multi-GPU training support with Horovod
• TensorRT | [NVIDIA](https://github.com/NVIDIA) | + +## Natural Language Processing + +| Model | Paper | Features | Maintainer | +|-------|-------|----------|------------| +| [BERT](https://github.com/IntelAI/models/tree/master/benchmarks/language_modeling/tensorflow/bert_large) | [BERT: Pre-training of Deep Bidirectional Transformers
for Language Understanding](https://arxiv.org/pdf/1810.04805) | • FP32 Inference
• FP32 Training | [Intel](https://github.com/IntelAI) | +| [BERT](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow2/LanguageModeling/BERT) | [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/pdf/1810.04805) | • Horovod Multi-GPU
• Multi-node with Horovod and Pyxis/Enroot Slurm cluster
• XLA
• Automatic mixed precision
• LAMB | [NVIDIA](https://github.com/NVIDIA) | +| [ELECTRA](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow2/LanguageModeling/ELECTRA) | [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://openreview.net/forum?id=r1xMH1BtvB) | • Automatic Mixed Precision
• Multi-GPU training support with Horovod
• Multi-node training on a Pyxis/Enroot Slurm cluster | [NVIDIA](https://github.com/NVIDIA) | +| [GNMT](https://github.com/IntelAI/models/tree/master/benchmarks/language_translation/tensorflow/mlperf_gnmt) | [Google’s Neural Machine Translation System:
Bridging the Gap between Human and Machine Translation](https://arxiv.org/pdf/1609.08144) | • FP32 Inference | [Intel](https://github.com/IntelAI) | +| [Transformer-LT (Official)](https://github.com/IntelAI/models/tree/master/benchmarks/language_translation/tensorflow/transformer_lt_official) | [Attention Is All You Need](https://arxiv.org/pdf/1706.03762) | • FP32 Inference | [Intel](https://github.com/IntelAI) | +| [Transformer-LT (MLPerf)](https://github.com/IntelAI/models/tree/master/benchmarks/language_translation/tensorflow/transformer_mlperf) | [Attention Is All You Need](https://arxiv.org/pdf/1706.03762) | • FP32 Training | [Intel](https://github.com/IntelAI) | + +## Recommendation Systems + +| Model | Paper | Features | Maintainer | +|-------|-------|----------|------------| +| [Wide & Deep](https://github.com/IntelAI/models/tree/master/benchmarks/recommendation/tensorflow/wide_deep_large_ds) | [Wide & Deep Learning for Recommender Systems](https://arxiv.org/pdf/1606.07792) | • FP32 Inference
• FP32 Training | [Intel](https://github.com/IntelAI) | +| [Wide & Deep](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow2/Recommendation/WideAndDeep) | [Wide & Deep Learning for Recommender Systems](https://arxiv.org/pdf/1606.07792) | • Automatic mixed precision
• Multi-GPU training support with Horovod
• XLA | [NVIDIA](https://github.com/NVIDIA) | +| [DLRM](https://github.com/NVIDIA/DeepLearningExamples/tree/master/TensorFlow2/Recommendation/DLRM) | [Deep Learning Recommendation Model for Personalization and Recommendation Systems](https://arxiv.org/pdf/1906.00091.pdf) | • Automatic Mixed Precision
• Hybrid-parallel multiGPU training using Horovod all2all
• Multinode training for Pyxis/Enroot Slurm clusters
• XLA
• Criteo dataset preprocessing with Spark on GPU | [NVIDIA](https://github.com/NVIDIA) | + +## Contributions + +If you want to contribute, please review the [contribution guidelines](https://github.com/tensorflow/models/wiki/How-to-contribute). diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/LICENSE b/TensorFlow2x/ComputeVision/Classification/models-master/official/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d3da228420e973edaf4123d5eeb42210f4450b0c --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/LICENSE @@ -0,0 +1,203 @@ +Copyright 2015 The TensorFlow Authors. All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2015, The TensorFlow Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/README-TPU.md b/TensorFlow2x/ComputeVision/Classification/models-master/official/README-TPU.md new file mode 100644 index 0000000000000000000000000000000000000000..a6031c44f0338e18762d8d7183299931aa3a285f --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/README-TPU.md @@ -0,0 +1,32 @@ +# Offically Supported TensorFlow 2.1+ Models on Cloud TPU + +## Natural Language Processing + +* [bert](nlp/bert): A powerful pre-trained language representation model: + BERT, which stands for Bidirectional Encoder Representations from + Transformers. + [BERT FineTuning with Cloud TPU](https://cloud.google.com/tpu/docs/tutorials/bert-2.x) provides step by step instructions on Cloud TPU training. You can look [Bert MNLI Tensorboard.dev metrics](https://tensorboard.dev/experiment/LijZ1IrERxKALQfr76gndA) for MNLI fine tuning task. +* [transformer](nlp/transformer): A transformer model to translate the WMT + English to German dataset. + [Training transformer on Cloud TPU](https://cloud.google.com/tpu/docs/tutorials/transformer-2.x) for step by step instructions on Cloud TPU training. + +## Computer Vision + +* [efficientnet](vision/image_classification): A family of convolutional + neural networks that scale by balancing network depth, width, and + resolution and can be used to classify ImageNet's dataset of 1000 classes. + See [Tensorboard.dev training metrics](https://tensorboard.dev/experiment/KnaWjrq5TXGfv0NW5m7rpg/#scalars). +* [mnist](vision/image_classification): A basic model to classify digits + from the MNIST dataset. See [Running MNIST on Cloud TPU](https://cloud.google.com/tpu/docs/tutorials/mnist-2.x) tutorial and [Tensorboard.dev metrics](https://tensorboard.dev/experiment/mIah5lppTASvrHqWrdr6NA). +* [mask-rcnn](vision/detection): An object detection and instance segmentation model. See [Tensorboard.dev training metrics](https://tensorboard.dev/experiment/LH7k0fMsRwqUAcE09o9kPA). +* [resnet](vision/image_classification): A deep residual network that can + be used to classify ImageNet's dataset of 1000 classes. + See [Training ResNet on Cloud TPU](https://cloud.google.com/tpu/docs/tutorials/resnet-2.x) tutorial and [Tensorboard.dev metrics](https://tensorboard.dev/experiment/CxlDK8YMRrSpYEGtBRpOhg). +* [retinanet](vision/detection): A fast and powerful object detector. See [Tensorboard.dev training metrics](https://tensorboard.dev/experiment/b8NRnWU3TqG6Rw0UxueU6Q). +* [shapemask](vision/detection): An object detection and instance segmentation model using shape priors. See [Tensorboard.dev training metrics](https://tensorboard.dev/experiment/ZbXgVoc6Rf6mBRlPj0JpLA). + +## Recommendation +* [dlrm](recommendation/ranking): [Deep Learning Recommendation Model for +Personalization and Recommendation Systems](https://arxiv.org/abs/1906.00091). +* [dcn v2](recommendation/ranking): [Improved Deep & Cross Network and Practical Lessons for Web-scale Learning to Rank Systems](https://arxiv.org/abs/2008.13535). +* [ncf](recommendation): Neural Collaborative Filtering. See [Tensorboard.dev training metrics](https://tensorboard.dev/experiment/0k3gKjZlR1ewkVTRyLB6IQ). diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/README.md b/TensorFlow2x/ComputeVision/Classification/models-master/official/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9745b6c034fdc03f36578d676fba127830920866 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/README.md @@ -0,0 +1,171 @@ +![Logo](https://storage.googleapis.com/model_garden_artifacts/TF_Model_Garden.png) + +# TensorFlow Official Models + +The TensorFlow official models are a collection of models +that use TensorFlow’s high-level APIs. +They are intended to be well-maintained, tested, and kept up to date +with the latest TensorFlow API. + +They should also be reasonably optimized for fast performance while still +being easy to read. +These models are used as end-to-end tests, ensuring that the models run +with the same or improved speed and performance with each new TensorFlow build. + +## More models to come! + +The team is actively developing new models. +In the near future, we will add: + +* State-of-the-art language understanding models. +* State-of-the-art image classification models. +* State-of-the-art object detection and instance segmentation models. + +## Table of Contents + +- [Models and Implementations](#models-and-implementations) + * [Computer Vision](#computer-vision) + + [Image Classification](#image-classification) + + [Object Detection and Segmentation](#object-detection-and-segmentation) + * [Natural Language Processing](#natural-language-processing) + * [Recommendation](#recommendation) +- [How to get started with the official models](#how-to-get-started-with-the-official-models) + +## Models and Implementations + +### Computer Vision + +#### Image Classification + +| Model | Reference (Paper) | +|-------|-------------------| +| [MNIST](vision/image_classification) | A basic model to classify digits from the [MNIST dataset](http://yann.lecun.com/exdb/mnist/) | +| [ResNet](vision/beta/MODEL_GARDEN.md) | [Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385) | +| [ResNet-RS](vision/beta/MODEL_GARDEN.md) | [Revisiting ResNets: Improved Training and Scaling Strategies](https://arxiv.org/abs/2103.07579) | +| [EfficientNet](vision/image_classification) | [EfficientNet: Rethinking Model Scaling for Convolutional Neural Networks](https://arxiv.org/abs/1905.11946) | + +#### Object Detection and Segmentation + +| Model | Reference (Paper) | +|-------|-------------------| +| [RetinaNet](vision/beta/MODEL_GARDEN.md) | [Focal Loss for Dense Object Detection](https://arxiv.org/abs/1708.02002) | +| [Mask R-CNN](vision/beta/MODEL_GARDEN.md) | [Mask R-CNN](https://arxiv.org/abs/1703.06870) | +| [ShapeMask](vision/detection) | [ShapeMask: Learning to Segment Novel Objects by Refining Shape Priors](https://arxiv.org/abs/1904.03239) | +| [SpineNet](vision/beta/MODEL_GARDEN.md) | [SpineNet: Learning Scale-Permuted Backbone for Recognition and Localization](https://arxiv.org/abs/1912.05027) | +| [Cascade RCNN-RS and RetinaNet-RS](vision/beta/MODEL_GARDEN.md) | [Simple Training Strategies and Model Scaling for Object Detection](https://arxiv.org/abs/2107.00057)| + +### Natural Language Processing + +| Model | Reference (Paper) | +|-------|-------------------| +| [ALBERT (A Lite BERT)](nlp/albert) | [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942) | +| [BERT (Bidirectional Encoder Representations from Transformers)](nlp/bert) | [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805) | +| [NHNet (News Headline generation model)](projects/nhnet) | [Generating Representative Headlines for News Stories](https://arxiv.org/abs/2001.09386) | +| [Transformer](nlp/transformer) | [Attention Is All You Need](https://arxiv.org/abs/1706.03762) | +| [XLNet](nlp/xlnet) | [XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) | +| [MobileBERT](nlp/projects/mobilebert) | [MobileBERT: a Compact Task-Agnostic BERT for Resource-Limited Devices](https://arxiv.org/abs/2004.02984) | + +### Recommendation + +Model | Reference (Paper) +-------------------------------- | ----------------- +[DLRM](recommendation/ranking) | [Deep Learning Recommendation Model for Personalization and Recommendation Systems](https://arxiv.org/abs/1906.00091) +[DCN v2](recommendation/ranking) | [Improved Deep & Cross Network and Practical Lessons for Web-scale Learning to Rank Systems](https://arxiv.org/abs/2008.13535) +[NCF](recommendation) | [Neural Collaborative Filtering](https://arxiv.org/abs/1708.05031) + +## How to get started with the official models + +* The models in the master branch are developed using TensorFlow 2, +and they target the TensorFlow [nightly binaries](https://github.com/tensorflow/tensorflow#installation) +built from the +[master branch of TensorFlow](https://github.com/tensorflow/tensorflow/tree/master). +* The stable versions targeting releases of TensorFlow are available +as tagged branches or [downloadable releases](https://github.com/tensorflow/models/releases). +* Model repository version numbers match the target TensorFlow release, +such that +[release v2.5.0](https://github.com/tensorflow/models/releases/tag/v2.5.0) +are compatible with +[TensorFlow v2.5.0](https://github.com/tensorflow/tensorflow/releases/tag/v2.5.0). + +Please follow the below steps before running models in this repository. + +### Requirements + +* The latest TensorFlow Model Garden release and TensorFlow 2 + * If you are on a version of TensorFlow earlier than 2.2, please +upgrade your TensorFlow to [the latest TensorFlow 2](https://www.tensorflow.org/install/). + +```shell +pip3 install tf-nightly +``` + +* Python 3.7+ + +Our integration tests run with Python 3.7. Although Python 3.6 should work, we +don't recommend earlier versions. + +### Installation + +#### Method 1: Install the TensorFlow Model Garden pip package + +**tf-models-official** is the stable Model Garden package. +pip will install all models and dependencies automatically. + +```shell +pip install tf-models-official +``` + +If you are using nlp packages, please also install **tensorflow-text**: + +```shell +pip install tensorflow-text +``` + +Please check out our [example](colab/fine_tuning_bert.ipynb) +to learn how to use a PIP package. + +Note that **tf-models-official** may not include the latest changes in this +github repo. To include latest changes, you may install **tf-models-nightly**, +which is the nightly Model Garden package created daily automatically. + +```shell +pip install tf-models-nightly +``` + +#### Method 2: Clone the source + +1. Clone the GitHub repository: + +```shell +git clone https://github.com/tensorflow/models.git +``` + +2. Add the top-level ***/models*** folder to the Python path. + +```shell +export PYTHONPATH=$PYTHONPATH:/path/to/models +``` + +If you are using a Colab notebook, please set the Python path with os.environ. + +```python +import os +os.environ['PYTHONPATH'] += ":/path/to/models" +``` + +3. Install other dependencies + +```shell +pip3 install --user -r official/requirements.txt +``` + +Finally, if you are using nlp packages, please also install +**tensorflow-text-nightly**: + +```shell +pip3 install tensorflow-text-nightly +``` + +## Contributions + +If you want to contribute, please review the [contribution guidelines](https://github.com/tensorflow/models/wiki/How-to-contribute). diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/__init__.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e419af524b5f349fe04abfa820c3cb51b777d422 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/colab/decoding_api_in_tf_nlp.ipynb b/TensorFlow2x/ComputeVision/Classification/models-master/official/colab/decoding_api_in_tf_nlp.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..726b382e228265fa1e19c2af3150e7cc32a0ec56 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/colab/decoding_api_in_tf_nlp.ipynb @@ -0,0 +1,492 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "vXLA5InzXydn" + }, + "source": [ + "##### Copyright 2021 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "RuRlpLL-X0R_" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fsACVQpVSifi" + }, + "source": [ + "### Install the TensorFlow Model Garden pip package\n", + "\n", + "* `tf-models-official` is the stable Model Garden package. Note that it may not include the latest changes in the `tensorflow_models` github repo. To include latest changes, you may install `tf-models-nightly`,\n", + "which is the nightly Model Garden package created daily automatically.\n", + "* pip will install all models and dependencies automatically." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hYEwGTeCXnnX" + }, + "source": [ + "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", + " \u003ctd\u003e\n", + " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/official_models/tutorials/decoding_api_in_tf_nlp.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", + " \u003c/td\u003e\n", + " \u003ctd\u003e\n", + " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/models/blob/master/official/colab/decoding_api_in_tf_nlp.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", + " \u003c/td\u003e\n", + " \u003ctd\u003e\n", + " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/models/blob/master/official/colab/decoding_api_in_tf_nlp.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", + " \u003c/td\u003e\n", + " \u003ctd\u003e\n", + " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/models/official/colab/decoding_api_in_tf_nlp.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", + " \u003c/td\u003e\n", + "\u003c/table\u003e" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2j-xhrsVQOQT" + }, + "outputs": [], + "source": [ + "pip install tf-models-nightly" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BjP7zwxmskpY" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import tensorflow as tf\n", + "\n", + "from official import nlp\n", + "from official.nlp.modeling.ops import sampling_module\n", + "from official.nlp.modeling.ops import beam_search" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0AWgyo-IQ5sP" + }, + "source": [ + "# Decoding API\n", + "This API provides an interface to experiment with different decoding strategies used for auto-regressive models.\n", + "\n", + "1. The following sampling strategies are provided in sampling_module.py, which inherits from the base Decoding class:\n", + " * [top_p](https://arxiv.org/abs/1904.09751) : [github](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/ops/sampling_module.py#L65) \n", + "\n", + " This implementation chooses most probable logits with cumulative probabilities upto top_p.\n", + "\n", + " * [top_k](https://arxiv.org/pdf/1805.04833.pdf) : [github](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/ops/sampling_module.py#L48)\n", + "\n", + " At each timestep, this implementation samples from top-k logits based on their probability distribution\n", + "\n", + " * Greedy : [github](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/ops/sampling_module.py#L26)\n", + "\n", + " This implementation returns the top logits based on probabilities.\n", + "\n", + "2. Beam search is provided in beam_search.py. [github](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/ops/beam_search.py)\n", + "\n", + " This implementation reduces the risk of missing hidden high probability logits by keeping the most likely num_beams of logits at each time step and eventually choosing the logits that has the overall highest probability." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MfOj7oaBRQnS" + }, + "source": [ + "## Initialize Sampling Module in TF-NLP.\n", + "\n", + "\n", + "\u003e **symbols_to_logits_fn** : This is a closure implemented by the users of the API. The input to this closure will be \n", + "```\n", + "Args:\n", + " 1] ids [batch_size, .. (index + 1 or 1 if padded_decode is True)],\n", + " 2] index [scalar] : current decoded step,\n", + " 3] cache [nested dictionary of tensors].\n", + "Returns:\n", + " 1] tensor for next-step logits [batch_size, vocab]\n", + " 2] the updated_cache [nested dictionary of tensors].\n", + "```\n", + "This closure calls the model to predict the logits for the 'index+1' step. The cache is used for faster decoding.\n", + "Here is a [reference](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/ops/beam_search_test.py#L88) implementation for the above closure.\n", + "\n", + "\n", + "\u003e **length_normalization_fn** : Closure for returning length normalization parameter.\n", + "```\n", + "Args: \n", + " 1] length : scalar for decoded step index.\n", + " 2] dtype : data-type of output tensor\n", + "Returns:\n", + " 1] value of length normalization factor.\n", + "Example :\n", + " def _length_norm(length, dtype):\n", + " return tf.pow(((5. + tf.cast(length, dtype)) / 6.), 0.0)\n", + "```\n", + "\n", + "\u003e **vocab_size** : Output vocabulary size.\n", + "\n", + "\u003e **max_decode_length** : Scalar for total number of decoding steps.\n", + "\n", + "\u003e **eos_id** : Decoding will stop if all output decoded ids in the batch have this ID.\n", + "\n", + "\u003e **padded_decode** : Set this to True if running on TPU. Tensors are padded to max_decoding_length if this is True.\n", + "\n", + "\u003e **top_k** : top_k is enabled if this value is \u003e 1.\n", + "\n", + "\u003e **top_p** : top_p is enabled if this value is \u003e 0 and \u003c 1.0\n", + "\n", + "\u003e **sampling_temperature** : This is used to re-estimate the softmax output. Temperature skews the distribution towards high probability tokens and lowers the mass in tail distribution. Value has to be positive. Low temperature is equivalent to greedy and makes the distribution sharper, while high temperature makes it more flat.\n", + "\n", + "\u003e **enable_greedy** : By default, this is true and greedy decoding is enabled.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lV1RRp6ihnGX" + }, + "source": [ + "# Initialize the Model Hyper-parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "eTsGp2gaKLdE" + }, + "outputs": [], + "source": [ + "params = {}\n", + "params['num_heads'] = 2\n", + "params['num_layers'] = 2\n", + "params['batch_size'] = 2\n", + "params['n_dims'] = 256\n", + "params['max_decode_length'] = 4" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UGvmd0_dRFYI" + }, + "source": [ + "## What is a Cache?\n", + "In auto-regressive architectures like Transformer based [Encoder-Decoder](https://arxiv.org/abs/1706.03762) models, \n", + "Cache is used for fast sequential decoding.\n", + "It is a nested dictionary storing pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) for every layer.\n", + "\n", + "```\n", + "{\n", + " 'layer_%d' % layer: {\n", + " 'k': tf.zeros([params['batch_size'], params['max_decode_length'], params['num_heads'], params['n_dims']/params['num_heads']], dtype=tf.float32),\n", + " 'v': tf.zeros([params['batch_size'], params['max_decode_length'], params['num_heads'], params['n_dims']/params['num_heads']], dtype=tf.float32)\n", + " } for layer in range(params['num_layers']),\n", + " 'model_specific_item' : Model specific tensor shape,\n", + "}\n", + "\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CYXkoplAij01" + }, + "source": [ + "# Initialize cache. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "D6kfZOOKgkm1" + }, + "outputs": [], + "source": [ + "cache = {\n", + " 'layer_%d' % layer: {\n", + " 'k': tf.zeros([params['batch_size'], params['max_decode_length'], params['num_heads'], params['n_dims']/params['num_heads']], dtype=tf.float32),\n", + " 'v': tf.zeros([params['batch_size'], params['max_decode_length'], params['num_heads'], params['n_dims']/params['num_heads']], dtype=tf.float32)\n", + " } for layer in range(params['num_layers'])\n", + " }\n", + "print(\"cache key shape for layer 1 :\", cache['layer_1']['k'].shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nNY3Xn8SiblP" + }, + "source": [ + "# Define closure for length normalization. **optional.**\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "T92ccAzlnGqh" + }, + "outputs": [], + "source": [ + "def length_norm(length, dtype):\n", + " \"\"\"Return length normalization factor.\"\"\"\n", + " return tf.pow(((5. + tf.cast(length, dtype)) / 6.), 0.0)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "syl7I5nURPgW" + }, + "source": [ + "# Create model_fn\n", + " In practice, this will be replaced by an actual model implementation such as [here](https://github.com/tensorflow/models/blob/master/official/nlp/transformer/transformer.py#L236)\n", + "```\n", + "Args:\n", + "i : Step that is being decoded.\n", + "Returns:\n", + " logit probabilities of size [batch_size, 1, vocab_size]\n", + "```\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "AhzSkRisRdB6" + }, + "outputs": [], + "source": [ + "probabilities = tf.constant([[[0.3, 0.4, 0.3], [0.3, 0.3, 0.4],\n", + " [0.1, 0.1, 0.8], [0.1, 0.1, 0.8]],\n", + " [[0.2, 0.5, 0.3], [0.2, 0.7, 0.1],\n", + " [0.1, 0.1, 0.8], [0.1, 0.1, 0.8]]])\n", + "def model_fn(i):\n", + " return probabilities[:, i, :]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DBMUkaVmVZBg" + }, + "source": [ + "# Initialize symbols_to_logits_fn\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "FAJ4CpbfVdjr" + }, + "outputs": [], + "source": [ + "def _symbols_to_logits_fn():\n", + " \"\"\"Calculates logits of the next tokens.\"\"\"\n", + " def symbols_to_logits_fn(ids, i, temp_cache):\n", + " del ids\n", + " logits = tf.cast(tf.math.log(model_fn(i)), tf.float32)\n", + " return logits, temp_cache\n", + " return symbols_to_logits_fn" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "R_tV3jyWVL47" + }, + "source": [ + "# Greedy \n", + "Greedy decoding selects the token id with the highest probability as its next id: $id_t = argmax_{w}P(id | id_{1:t-1})$ at each timestep $t$. The following sketch shows greedy decoding. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "aGt9idSkVQEJ" + }, + "outputs": [], + "source": [ + "greedy_obj = sampling_module.SamplingModule(\n", + " length_normalization_fn=None,\n", + " dtype=tf.float32,\n", + " symbols_to_logits_fn=_symbols_to_logits_fn(),\n", + " vocab_size=3,\n", + " max_decode_length=params['max_decode_length'],\n", + " eos_id=10,\n", + " padded_decode=False)\n", + "ids, _ = greedy_obj.generate(\n", + " initial_ids=tf.constant([9, 1]), initial_cache=cache)\n", + "print(\"Greedy Decoded Ids:\", ids)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "s4pTTsQXVz5O" + }, + "source": [ + "# top_k sampling\n", + "In *Top-K* sampling, the *K* most likely next token ids are filtered and the probability mass is redistributed among only those *K* ids. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "pCLWIn6GV5_G" + }, + "outputs": [], + "source": [ + "top_k_obj = sampling_module.SamplingModule(\n", + " length_normalization_fn=length_norm,\n", + " dtype=tf.float32,\n", + " symbols_to_logits_fn=_symbols_to_logits_fn(),\n", + " vocab_size=3,\n", + " max_decode_length=params['max_decode_length'],\n", + " eos_id=10,\n", + " sample_temperature=tf.constant(1.0),\n", + " top_k=tf.constant(3),\n", + " padded_decode=False,\n", + " enable_greedy=False)\n", + "ids, _ = top_k_obj.generate(\n", + " initial_ids=tf.constant([9, 1]), initial_cache=cache)\n", + "print(\"top-k sampled Ids:\", ids)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Jp3G-eE_WI4Y" + }, + "source": [ + "# top_p sampling\n", + "Instead of sampling only from the most likely *K* token ids, in *Top-p* sampling chooses from the smallest possible set of ids whose cumulative probability exceeds the probability *p*." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "rEGdIWcuWILO" + }, + "outputs": [], + "source": [ + "top_p_obj = sampling_module.SamplingModule(\n", + " length_normalization_fn=length_norm,\n", + " dtype=tf.float32,\n", + " symbols_to_logits_fn=_symbols_to_logits_fn(),\n", + " vocab_size=3,\n", + " max_decode_length=params['max_decode_length'],\n", + " eos_id=10,\n", + " sample_temperature=tf.constant(1.0),\n", + " top_p=tf.constant(0.9),\n", + " padded_decode=False,\n", + " enable_greedy=False)\n", + "ids, _ = top_p_obj.generate(\n", + " initial_ids=tf.constant([9, 1]), initial_cache=cache)\n", + "print(\"top-p sampled Ids:\", ids)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2hcuyJ2VWjDz" + }, + "source": [ + "# Beam search decoding\n", + "Beam search reduces the risk of missing hidden high probability token ids by keeping the most likely num_beams of hypotheses at each time step and eventually choosing the hypothesis that has the overall highest probability. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cJ3WzvSrWmSA" + }, + "outputs": [], + "source": [ + "beam_size = 2\n", + "params['batch_size'] = 1\n", + "beam_cache = {\n", + " 'layer_%d' % layer: {\n", + " 'k': tf.zeros([params['batch_size'], params['max_decode_length'], params['num_heads'], params['n_dims']], dtype=tf.float32),\n", + " 'v': tf.zeros([params['batch_size'], params['max_decode_length'], params['num_heads'], params['n_dims']], dtype=tf.float32)\n", + " } for layer in range(params['num_layers'])\n", + " }\n", + "print(\"cache key shape for layer 1 :\", beam_cache['layer_1']['k'].shape)\n", + "ids, _ = beam_search.sequence_beam_search(\n", + " symbols_to_logits_fn=_symbols_to_logits_fn(),\n", + " initial_ids=tf.constant([9], tf.int32),\n", + " initial_cache=beam_cache,\n", + " vocab_size=3,\n", + " beam_size=beam_size,\n", + " alpha=0.6,\n", + " max_decode_length=params['max_decode_length'],\n", + " eos_id=10,\n", + " padded_decode=False,\n", + " dtype=tf.float32)\n", + "print(\"Beam search ids:\", ids)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "decoding_api_in_tf_nlp.ipynb", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/colab/nlp/customize_encoder.ipynb b/TensorFlow2x/ComputeVision/Classification/models-master/official/colab/nlp/customize_encoder.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..aeddb29f96352fbd4c8df3540e6bd4b8fe70bb8b --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/colab/nlp/customize_encoder.ipynb @@ -0,0 +1,575 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "Customizing a Transformer Encoder", + "private_outputs": true, + "provenance": [], + "collapsed_sections": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Bp8t2AI8i7uP" + }, + "source": [ + "##### Copyright 2020 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "metadata": { + "cellView": "form", + "id": "rxPj2Lsni9O4" + }, + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6xS-9i5DrRvO" + }, + "source": [ + "# Customizing a Transformer Encoder" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Mwb9uw1cDXsa" + }, + "source": [ + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iLrcV4IyrcGX" + }, + "source": [ + "## Learning objectives\n", + "\n", + "The [TensorFlow Models NLP library](https://github.com/tensorflow/models/tree/master/official/nlp/modeling) is a collection of tools for building and training modern high performance natural language models.\n", + "\n", + "The [TransformEncoder](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/networks/encoder_scaffold.py) is the core of this library, and lots of new network architectures are proposed to improve the encoder. In this Colab notebook, we will learn how to customize the encoder to employ new network architectures." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YYxdyoWgsl8t" + }, + "source": [ + "## Install and import" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fEJSFutUsn_h" + }, + "source": [ + "### Install the TensorFlow Model Garden pip package\n", + "\n", + "* `tf-models-official` is the stable Model Garden package. Note that it may not include the latest changes in the `tensorflow_models` github repo. To include latest changes, you may install `tf-models-nightly`,\n", + "which is the nightly Model Garden package created daily automatically.\n", + "* `pip` will install all models and dependencies automatically." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "thsKZDjhswhR" + }, + "source": [ + "!pip install -q tf-models-official==2.4.0" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hpf7JPCVsqtv" + }, + "source": [ + "### Import Tensorflow and other libraries" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "my4dp-RMssQe" + }, + "source": [ + "import numpy as np\n", + "import tensorflow as tf\n", + "\n", + "from official.modeling import activations\n", + "from official.nlp import modeling\n", + "from official.nlp.modeling import layers, losses, models, networks" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vjDmVsFfs85n" + }, + "source": [ + "## Canonical BERT encoder\n", + "\n", + "Before learning how to customize the encoder, let's firstly create a canonical BERT enoder and use it to instantiate a `BertClassifier` for classification task." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Oav8sbgstWc-" + }, + "source": [ + "cfg = {\n", + " \"vocab_size\": 100,\n", + " \"hidden_size\": 32,\n", + " \"num_layers\": 3,\n", + " \"num_attention_heads\": 4,\n", + " \"intermediate_size\": 64,\n", + " \"activation\": activations.gelu,\n", + " \"dropout_rate\": 0.1,\n", + " \"attention_dropout_rate\": 0.1,\n", + " \"max_sequence_length\": 16,\n", + " \"type_vocab_size\": 2,\n", + " \"initializer\": tf.keras.initializers.TruncatedNormal(stddev=0.02),\n", + "}\n", + "bert_encoder = modeling.networks.BertEncoder(**cfg)\n", + "\n", + "def build_classifier(bert_encoder):\n", + " return modeling.models.BertClassifier(bert_encoder, num_classes=2)\n", + "\n", + "canonical_classifier_model = build_classifier(bert_encoder)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Qe2UWI6_tsHo" + }, + "source": [ + "`canonical_classifier_model` can be trained using the training data. For details about how to train the model, please see the colab [fine_tuning_bert.ipynb](https://github.com/tensorflow/models/blob/master/official/colab/fine_tuning_bert.ipynb). We skip the code that trains the model here.\n", + "\n", + "After training, we can apply the model to do prediction.\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "csED2d-Yt5h6" + }, + "source": [ + "def predict(model):\n", + " batch_size = 3\n", + " np.random.seed(0)\n", + " word_ids = np.random.randint(\n", + " cfg[\"vocab_size\"], size=(batch_size, cfg[\"max_sequence_length\"]))\n", + " mask = np.random.randint(2, size=(batch_size, cfg[\"max_sequence_length\"]))\n", + " type_ids = np.random.randint(\n", + " cfg[\"type_vocab_size\"], size=(batch_size, cfg[\"max_sequence_length\"]))\n", + " print(model([word_ids, mask, type_ids], training=False))\n", + "\n", + "predict(canonical_classifier_model)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PzKStEK9t_Pb" + }, + "source": [ + "## Customize BERT encoder\n", + "\n", + "One BERT encoder consists of an embedding network and multiple transformer blocks, and each transformer block contains an attention layer and a feedforward layer." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rmwQfhj6fmKz" + }, + "source": [ + "We provide easy ways to customize each of those components via (1)\n", + "[EncoderScaffold](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/networks/encoder_scaffold.py) and (2) [TransformerScaffold](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/layers/transformer_scaffold.py)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xsMgEVHAui11" + }, + "source": [ + "### Use EncoderScaffold\n", + "\n", + "`EncoderScaffold` allows users to provide a custom embedding subnetwork\n", + " (which will replace the standard embedding logic) and/or a custom hidden layer class (which will replace the `Transformer` instantiation in the encoder)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-JBabpa2AOz8" + }, + "source": [ + "#### Without Customization\n", + "\n", + "Without any customization, `EncoderScaffold` behaves the same the canonical `BertEncoder`.\n", + "\n", + "As shown in the following example, `EncoderScaffold` can load `BertEncoder`'s weights and output the same values:" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "ktNzKuVByZQf" + }, + "source": [ + "default_hidden_cfg = dict(\n", + " num_attention_heads=cfg[\"num_attention_heads\"],\n", + " intermediate_size=cfg[\"intermediate_size\"],\n", + " intermediate_activation=activations.gelu,\n", + " dropout_rate=cfg[\"dropout_rate\"],\n", + " attention_dropout_rate=cfg[\"attention_dropout_rate\"],\n", + " kernel_initializer=tf.keras.initializers.TruncatedNormal(0.02),\n", + ")\n", + "default_embedding_cfg = dict(\n", + " vocab_size=cfg[\"vocab_size\"],\n", + " type_vocab_size=cfg[\"type_vocab_size\"],\n", + " hidden_size=cfg[\"hidden_size\"],\n", + " initializer=tf.keras.initializers.TruncatedNormal(0.02),\n", + " dropout_rate=cfg[\"dropout_rate\"],\n", + " max_seq_length=cfg[\"max_sequence_length\"]\n", + ")\n", + "default_kwargs = dict(\n", + " hidden_cfg=default_hidden_cfg,\n", + " embedding_cfg=default_embedding_cfg,\n", + " num_hidden_instances=cfg[\"num_layers\"],\n", + " pooled_output_dim=cfg[\"hidden_size\"],\n", + " return_all_layer_outputs=True,\n", + " pooler_layer_initializer=tf.keras.initializers.TruncatedNormal(0.02),\n", + ")\n", + "\n", + "encoder_scaffold = modeling.networks.EncoderScaffold(**default_kwargs)\n", + "classifier_model_from_encoder_scaffold = build_classifier(encoder_scaffold)\n", + "classifier_model_from_encoder_scaffold.set_weights(\n", + " canonical_classifier_model.get_weights())\n", + "predict(classifier_model_from_encoder_scaffold)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sMaUmLyIuwcs" + }, + "source": [ + "#### Customize Embedding\n", + "\n", + "Next, we show how to use a customized embedding network.\n", + "\n", + "We firstly build an embedding network that will replace the default network. This one will have 2 inputs (`mask` and `word_ids`) instead of 3, and won't use positional embeddings." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "LTinnaG6vcsw" + }, + "source": [ + "word_ids = tf.keras.layers.Input(\n", + " shape=(cfg['max_sequence_length'],), dtype=tf.int32, name=\"input_word_ids\")\n", + "mask = tf.keras.layers.Input(\n", + " shape=(cfg['max_sequence_length'],), dtype=tf.int32, name=\"input_mask\")\n", + "embedding_layer = modeling.layers.OnDeviceEmbedding(\n", + " vocab_size=cfg['vocab_size'],\n", + " embedding_width=cfg['hidden_size'],\n", + " initializer=tf.keras.initializers.TruncatedNormal(stddev=0.02),\n", + " name=\"word_embeddings\")\n", + "word_embeddings = embedding_layer(word_ids)\n", + "attention_mask = layers.SelfAttentionMask()([word_embeddings, mask])\n", + "new_embedding_network = tf.keras.Model([word_ids, mask],\n", + " [word_embeddings, attention_mask])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HN7_yu-6O3qI" + }, + "source": [ + "Inspecting `new_embedding_network`, we can see it takes two inputs:\n", + "`input_word_ids` and `input_mask`." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "fO9zKFE4OpHp" + }, + "source": [ + "tf.keras.utils.plot_model(new_embedding_network, show_shapes=True, dpi=48)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9cOaGQHLv12W" + }, + "source": [ + "We then can build a new encoder using the above `new_embedding_network`." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "mtFDMNf2vIl9" + }, + "source": [ + "kwargs = dict(default_kwargs)\n", + "\n", + "# Use new embedding network.\n", + "kwargs['embedding_cls'] = new_embedding_network\n", + "kwargs['embedding_data'] = embedding_layer.embeddings\n", + "\n", + "encoder_with_customized_embedding = modeling.networks.EncoderScaffold(**kwargs)\n", + "classifier_model = build_classifier(encoder_with_customized_embedding)\n", + "# ... Train the model ...\n", + "print(classifier_model.inputs)\n", + "\n", + "# Assert that there are only two inputs.\n", + "assert len(classifier_model.inputs) == 2" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Z73ZQDtmwg9K" + }, + "source": [ + "#### Customized Transformer\n", + "\n", + "User can also override the [hidden_cls](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/networks/encoder_scaffold.py#L103) argument in `EncoderScaffold`'s constructor to employ a customized Transformer layer.\n", + "\n", + "See [ReZeroTransformer](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/layers/rezero_transformer.py) for how to implement a customized Transformer layer.\n", + "\n", + "Following is an example of using `ReZeroTransformer`:\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "uAIarLZgw6pA" + }, + "source": [ + "kwargs = dict(default_kwargs)\n", + "\n", + "# Use ReZeroTransformer.\n", + "kwargs['hidden_cls'] = modeling.layers.ReZeroTransformer\n", + "\n", + "encoder_with_rezero_transformer = modeling.networks.EncoderScaffold(**kwargs)\n", + "classifier_model = build_classifier(encoder_with_rezero_transformer)\n", + "# ... Train the model ...\n", + "predict(classifier_model)\n", + "\n", + "# Assert that the variable `rezero_alpha` from ReZeroTransformer exists.\n", + "assert 'rezero_alpha' in ''.join([x.name for x in classifier_model.trainable_weights])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6PMHFdvnxvR0" + }, + "source": [ + "### Use [TransformerScaffold](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/layers/transformer_scaffold.py)\n", + "\n", + "The above method of customizing `Transformer` requires rewriting the whole `Transformer` layer, while sometimes you may only want to customize either attention layer or feedforward block. In this case, [TransformerScaffold](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/layers/transformer_scaffold.py) can be used.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "D6FejlgwyAy_" + }, + "source": [ + "#### Customize Attention Layer\n", + "\n", + "User can also override the [attention_cls](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/layers/transformer_scaffold.py#L45) argument in `TransformerScaffold`'s constructor to employ a customized Attention layer.\n", + "\n", + "See [TalkingHeadsAttention](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/layers/talking_heads_attention.py) for how to implement a customized `Attention` layer.\n", + "\n", + "Following is an example of using [TalkingHeadsAttention](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/layers/talking_heads_attention.py):" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nFrSMrZuyNeQ" + }, + "source": [ + "# Use TalkingHeadsAttention\n", + "hidden_cfg = dict(default_hidden_cfg)\n", + "hidden_cfg['attention_cls'] = modeling.layers.TalkingHeadsAttention\n", + "\n", + "kwargs = dict(default_kwargs)\n", + "kwargs['hidden_cls'] = modeling.layers.TransformerScaffold\n", + "kwargs['hidden_cfg'] = hidden_cfg\n", + "\n", + "encoder = modeling.networks.EncoderScaffold(**kwargs)\n", + "classifier_model = build_classifier(encoder)\n", + "# ... Train the model ...\n", + "predict(classifier_model)\n", + "\n", + "# Assert that the variable `pre_softmax_weight` from TalkingHeadsAttention exists.\n", + "assert 'pre_softmax_weight' in ''.join([x.name for x in classifier_model.trainable_weights])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kuEJcTyByVvI" + }, + "source": [ + "#### Customize Feedforward Layer\n", + "\n", + "Similiarly, one could also customize the feedforward layer.\n", + "\n", + "See [GatedFeedforward](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/layers/gated_feedforward.py) for how to implement a customized feedforward layer.\n", + "\n", + "Following is an example of using [GatedFeedforward](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/layers/gated_feedforward.py)." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "XAbKy_l4y_-i" + }, + "source": [ + "# Use TalkingHeadsAttention\n", + "hidden_cfg = dict(default_hidden_cfg)\n", + "hidden_cfg['feedforward_cls'] = modeling.layers.GatedFeedforward\n", + "\n", + "kwargs = dict(default_kwargs)\n", + "kwargs['hidden_cls'] = modeling.layers.TransformerScaffold\n", + "kwargs['hidden_cfg'] = hidden_cfg\n", + "\n", + "encoder_with_gated_feedforward = modeling.networks.EncoderScaffold(**kwargs)\n", + "classifier_model = build_classifier(encoder_with_gated_feedforward)\n", + "# ... Train the model ...\n", + "predict(classifier_model)\n", + "\n", + "# Assert that the variable `gate` from GatedFeedforward exists.\n", + "assert 'gate' in ''.join([x.name for x in classifier_model.trainable_weights])" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "a_8NWUhkzeAq" + }, + "source": [ + "### Build a new Encoder using building blocks from KerasBERT.\n", + "\n", + "Finally, you could also build a new encoder using building blocks in the modeling library.\n", + "\n", + "See [AlbertEncoder](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/networks/albert_encoder.py) as an example:\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "xsiA3RzUzmUM" + }, + "source": [ + "albert_encoder = modeling.networks.AlbertEncoder(**cfg)\n", + "classifier_model = build_classifier(albert_encoder)\n", + "# ... Train the model ...\n", + "predict(classifier_model)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MeidDfhlHKSO" + }, + "source": [ + "Inspecting the `albert_encoder`, we see it stacks the same `Transformer` layer multiple times." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Uv_juT22HERW" + }, + "source": [ + "tf.keras.utils.plot_model(albert_encoder, show_shapes=True, dpi=48)" + ], + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/colab/nlp/nlp_modeling_library_intro.ipynb b/TensorFlow2x/ComputeVision/Classification/models-master/official/colab/nlp/nlp_modeling_library_intro.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..e4ce780c96bfbf679c91891f38b08ac3b0bb983e --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/colab/nlp/nlp_modeling_library_intro.ipynb @@ -0,0 +1,544 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "80xnUmoI7fBX" + }, + "source": [ + "##### Copyright 2020 The TensorFlow Authors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "8nvTnfs6Q692" + }, + "outputs": [], + "source": [ + "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WmfcMK5P5C1G" + }, + "source": [ + "# Introduction to the TensorFlow Models NLP library" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cH-oJ8R6AHMK" + }, + "source": [ + "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", + " \u003ctd\u003e\n", + " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/official_models/nlp/nlp_modeling_library_intro\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", + " \u003c/td\u003e\n", + " \u003ctd\u003e\n", + " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/models/blob/master/official/colab/nlp/nlp_modeling_library_intro.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", + " \u003c/td\u003e\n", + " \u003ctd\u003e\n", + " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/models/blob/master/official/colab/nlp/nlp_modeling_library_intro.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", + " \u003c/td\u003e\n", + " \u003ctd\u003e\n", + " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/models/official/colab/nlp/nlp_modeling_library_intro.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", + " \u003c/td\u003e\n", + "\u003c/table\u003e" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0H_EFIhq4-MJ" + }, + "source": [ + "## Learning objectives\n", + "\n", + "In this Colab notebook, you will learn how to build transformer-based models for common NLP tasks including pretraining, span labelling and classification using the building blocks from [NLP modeling library](https://github.com/tensorflow/models/tree/master/official/nlp/modeling)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "2N97-dps_nUk" + }, + "source": [ + "## Install and import" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "459ygAVl_rg0" + }, + "source": [ + "### Install the TensorFlow Model Garden pip package\n", + "\n", + "* `tf-models-official` is the stable Model Garden package. Note that it may not include the latest changes in the `tensorflow_models` github repo. To include latest changes, you may install `tf-models-nightly`,\n", + "which is the nightly Model Garden package created daily automatically.\n", + "* `pip` will install all models and dependencies automatically." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Y-qGkdh6_sZc" + }, + "outputs": [], + "source": [ + "!pip install -q tf-models-official==2.4.0" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "e4huSSwyAG_5" + }, + "source": [ + "### Import Tensorflow and other libraries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jqYXqtjBAJd9" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import tensorflow as tf\n", + "\n", + "from official.nlp import modeling\n", + "from official.nlp.modeling import layers, losses, models, networks" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "djBQWjvy-60Y" + }, + "source": [ + "## BERT pretraining model\n", + "\n", + "BERT ([Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805)) introduced the method of pre-training language representations on a large text corpus and then using that model for downstream NLP tasks.\n", + "\n", + "In this section, we will learn how to build a model to pretrain BERT on the masked language modeling task and next sentence prediction task. For simplicity, we only show the minimum example and use dummy data." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MKuHVlsCHmiq" + }, + "source": [ + "### Build a `BertPretrainer` model wrapping `BertEncoder`\n", + "\n", + "The [BertEncoder](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/networks/bert_encoder.py) implements the Transformer-based encoder as described in [BERT paper](https://arxiv.org/abs/1810.04805). It includes the embedding lookups and transformer layers, but not the masked language model or classification task networks.\n", + "\n", + "The [BertPretrainer](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/models/bert_pretrainer.py) allows a user to pass in a transformer stack, and instantiates the masked language model and classification networks that are used to create the training objectives." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "EXkcXz-9BwB3" + }, + "outputs": [], + "source": [ + "# Build a small transformer network.\n", + "vocab_size = 100\n", + "sequence_length = 16\n", + "network = modeling.networks.BertEncoder(\n", + " vocab_size=vocab_size, num_layers=2, sequence_length=16)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0NH5irV5KTMS" + }, + "source": [ + "Inspecting the encoder, we see it contains few embedding layers, stacked `Transformer` layers and are connected to three input layers:\n", + "\n", + "`input_word_ids`, `input_type_ids` and `input_mask`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "lZNoZkBrIoff" + }, + "outputs": [], + "source": [ + "tf.keras.utils.plot_model(network, show_shapes=True, dpi=48)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "o7eFOZXiIl-b" + }, + "outputs": [], + "source": [ + "# Create a BERT pretrainer with the created network.\n", + "num_token_predictions = 8\n", + "bert_pretrainer = modeling.models.BertPretrainer(\n", + " network, num_classes=2, num_token_predictions=num_token_predictions, output='predictions')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "d5h5HT7gNHx_" + }, + "source": [ + "Inspecting the `bert_pretrainer`, we see it wraps the `encoder` with additional `MaskedLM` and `Classification` heads." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2tcNfm03IBF7" + }, + "outputs": [], + "source": [ + "tf.keras.utils.plot_model(bert_pretrainer, show_shapes=True, dpi=48)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "F2oHrXGUIS0M" + }, + "outputs": [], + "source": [ + "# We can feed some dummy data to get masked language model and sentence output.\n", + "batch_size = 2\n", + "word_id_data = np.random.randint(vocab_size, size=(batch_size, sequence_length))\n", + "mask_data = np.random.randint(2, size=(batch_size, sequence_length))\n", + "type_id_data = np.random.randint(2, size=(batch_size, sequence_length))\n", + "masked_lm_positions_data = np.random.randint(2, size=(batch_size, num_token_predictions))\n", + "\n", + "outputs = bert_pretrainer(\n", + " [word_id_data, mask_data, type_id_data, masked_lm_positions_data])\n", + "lm_output = outputs[\"masked_lm\"]\n", + "sentence_output = outputs[\"classification\"]\n", + "print(lm_output)\n", + "print(sentence_output)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bnx3UCHniCS5" + }, + "source": [ + "### Compute loss\n", + "Next, we can use `lm_output` and `sentence_output` to compute `loss`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "k30H4Q86f52x" + }, + "outputs": [], + "source": [ + "masked_lm_ids_data = np.random.randint(vocab_size, size=(batch_size, num_token_predictions))\n", + "masked_lm_weights_data = np.random.randint(2, size=(batch_size, num_token_predictions))\n", + "next_sentence_labels_data = np.random.randint(2, size=(batch_size))\n", + "\n", + "mlm_loss = modeling.losses.weighted_sparse_categorical_crossentropy_loss(\n", + " labels=masked_lm_ids_data,\n", + " predictions=lm_output,\n", + " weights=masked_lm_weights_data)\n", + "sentence_loss = modeling.losses.weighted_sparse_categorical_crossentropy_loss(\n", + " labels=next_sentence_labels_data,\n", + " predictions=sentence_output)\n", + "loss = mlm_loss + sentence_loss\n", + "print(loss)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wrmSs8GjHxVw" + }, + "source": [ + "With the loss, you can optimize the model.\n", + "After training, we can save the weights of TransformerEncoder for the downstream fine-tuning tasks. Please see [run_pretraining.py](https://github.com/tensorflow/models/blob/master/official/nlp/bert/run_pretraining.py) for the full example.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "k8cQVFvBCV4s" + }, + "source": [ + "## Span labeling model\n", + "\n", + "Span labeling is the task to assign labels to a span of the text, for example, label a span of text as the answer of a given question.\n", + "\n", + "In this section, we will learn how to build a span labeling model. Again, we use dummy data for simplicity." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "xrLLEWpfknUW" + }, + "source": [ + "### Build a BertSpanLabeler wrapping BertEncoder\n", + "\n", + "[BertSpanLabeler](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/models/bert_span_labeler.py) implements a simple single-span start-end predictor (that is, a model that predicts two values: a start token index and an end token index), suitable for SQuAD-style tasks.\n", + "\n", + "Note that `BertSpanLabeler` wraps a `BertEncoder`, the weights of which can be restored from the above pretraining model.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "B941M4iUCejO" + }, + "outputs": [], + "source": [ + "network = modeling.networks.BertEncoder(\n", + " vocab_size=vocab_size, num_layers=2, sequence_length=sequence_length)\n", + "\n", + "# Create a BERT trainer with the created network.\n", + "bert_span_labeler = modeling.models.BertSpanLabeler(network)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QpB9pgj4PpMg" + }, + "source": [ + "Inspecting the `bert_span_labeler`, we see it wraps the encoder with additional `SpanLabeling` that outputs `start_position` and `end_postion`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "RbqRNJCLJu4H" + }, + "outputs": [], + "source": [ + "tf.keras.utils.plot_model(bert_span_labeler, show_shapes=True, dpi=48)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fUf1vRxZJwio" + }, + "outputs": [], + "source": [ + "# Create a set of 2-dimensional data tensors to feed into the model.\n", + "word_id_data = np.random.randint(vocab_size, size=(batch_size, sequence_length))\n", + "mask_data = np.random.randint(2, size=(batch_size, sequence_length))\n", + "type_id_data = np.random.randint(2, size=(batch_size, sequence_length))\n", + "\n", + "# Feed the data to the model.\n", + "start_logits, end_logits = bert_span_labeler([word_id_data, mask_data, type_id_data])\n", + "print(start_logits)\n", + "print(end_logits)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WqhgQaN1lt-G" + }, + "source": [ + "### Compute loss\n", + "With `start_logits` and `end_logits`, we can compute loss:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "waqs6azNl3Nn" + }, + "outputs": [], + "source": [ + "start_positions = np.random.randint(sequence_length, size=(batch_size))\n", + "end_positions = np.random.randint(sequence_length, size=(batch_size))\n", + "\n", + "start_loss = tf.keras.losses.sparse_categorical_crossentropy(\n", + " start_positions, start_logits, from_logits=True)\n", + "end_loss = tf.keras.losses.sparse_categorical_crossentropy(\n", + " end_positions, end_logits, from_logits=True)\n", + "\n", + "total_loss = (tf.reduce_mean(start_loss) + tf.reduce_mean(end_loss)) / 2\n", + "print(total_loss)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Zdf03YtZmd_d" + }, + "source": [ + "With the `loss`, you can optimize the model. Please see [run_squad.py](https://github.com/tensorflow/models/blob/master/official/nlp/bert/run_squad.py) for the full example." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0A1XnGSTChg9" + }, + "source": [ + "## Classification model\n", + "\n", + "In the last section, we show how to build a text classification model.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MSK8OpZgnQa9" + }, + "source": [ + "### Build a BertClassifier model wrapping BertEncoder\n", + "\n", + "[BertClassifier](https://github.com/tensorflow/models/blob/master/official/nlp/modeling/models/bert_classifier.py) implements a [CLS] token classification model containing a single classification head." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cXXCsffkCphk" + }, + "outputs": [], + "source": [ + "network = modeling.networks.BertEncoder(\n", + " vocab_size=vocab_size, num_layers=2, sequence_length=sequence_length)\n", + "\n", + "# Create a BERT trainer with the created network.\n", + "num_classes = 2\n", + "bert_classifier = modeling.models.BertClassifier(\n", + " network, num_classes=num_classes)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8tZKueKYP4bB" + }, + "source": [ + "Inspecting the `bert_classifier`, we see it wraps the `encoder` with additional `Classification` head." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "snlutm9ZJgEZ" + }, + "outputs": [], + "source": [ + "tf.keras.utils.plot_model(bert_classifier, show_shapes=True, dpi=48)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yyHPHsqBJkCz" + }, + "outputs": [], + "source": [ + "# Create a set of 2-dimensional data tensors to feed into the model.\n", + "word_id_data = np.random.randint(vocab_size, size=(batch_size, sequence_length))\n", + "mask_data = np.random.randint(2, size=(batch_size, sequence_length))\n", + "type_id_data = np.random.randint(2, size=(batch_size, sequence_length))\n", + "\n", + "# Feed the data to the model.\n", + "logits = bert_classifier([word_id_data, mask_data, type_id_data])\n", + "print(logits)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "w--a2mg4nzKm" + }, + "source": [ + "### Compute loss\n", + "\n", + "With `logits`, we can compute `loss`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9X0S1DoFn_5Q" + }, + "outputs": [], + "source": [ + "labels = np.random.randint(num_classes, size=(batch_size))\n", + "\n", + "loss = tf.keras.losses.sparse_categorical_crossentropy(\n", + " labels, logits, from_logits=True)\n", + "print(loss)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mzBqOylZo3og" + }, + "source": [ + "With the `loss`, you can optimize the model. Please see [run_classifier.py](https://github.com/tensorflow/models/blob/master/official/nlp/bert/run_classifier.py) or the colab [fine_tuning_bert.ipynb](https://github.com/tensorflow/models/blob/master/official/colab/fine_tuning_bert.ipynb) for the full example." + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [], + "name": "Introduction to the TensorFlow Models NLP library", + "private_outputs": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/common/__init__.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/common/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a25710c222e3327cb20e000db5df5c5651c4a2cc --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/common/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/common/dataset_fn.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/common/dataset_fn.py new file mode 100644 index 0000000000000000000000000000000000000000..4ac16a31b555588368a6c0aba73adbe62a95c2eb --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/common/dataset_fn.py @@ -0,0 +1,42 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utility library for picking an appropriate dataset function.""" + +from typing import Any, Callable, Union, Type + +import tensorflow as tf + +PossibleDatasetType = Union[Type[tf.data.Dataset], Callable[[tf.Tensor], Any]] + + +def pick_dataset_fn(file_type: str) -> PossibleDatasetType: + if file_type == 'tfrecord': + return tf.data.TFRecordDataset + + raise ValueError('Unrecognized file_type: {}'.format(file_type)) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/common/distribute_utils.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/common/distribute_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c48d68d6d93111e0959c9bbfde1e767fc673a979 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/common/distribute_utils.py @@ -0,0 +1,233 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper functions for running models in a distributed setting.""" + +import json +import os +import tensorflow as tf + + +def _collective_communication(all_reduce_alg): + """Return a CollectiveCommunication based on all_reduce_alg. + + Args: + all_reduce_alg: a string specifying which collective communication to pick, + or None. + + Returns: + tf.distribute.experimental.CollectiveCommunication object + + Raises: + ValueError: if `all_reduce_alg` not in [None, "ring", "nccl"] + """ + collective_communication_options = { + None: tf.distribute.experimental.CollectiveCommunication.AUTO, + "ring": tf.distribute.experimental.CollectiveCommunication.RING, + "nccl": tf.distribute.experimental.CollectiveCommunication.NCCL + } + if all_reduce_alg not in collective_communication_options: + raise ValueError( + "When used with `multi_worker_mirrored`, valid values for " + "all_reduce_alg are [`ring`, `nccl`]. Supplied value: {}".format( + all_reduce_alg)) + return collective_communication_options[all_reduce_alg] + + +def _mirrored_cross_device_ops(all_reduce_alg, num_packs): + """Return a CrossDeviceOps based on all_reduce_alg and num_packs. + + Args: + all_reduce_alg: a string specifying which cross device op to pick, or None. + num_packs: an integer specifying number of packs for the cross device op. + + Returns: + tf.distribute.CrossDeviceOps object or None. + + Raises: + ValueError: if `all_reduce_alg` not in [None, "nccl", "hierarchical_copy"]. + """ + if all_reduce_alg is None: + return None + mirrored_all_reduce_options = { + "nccl": tf.distribute.NcclAllReduce, + "hierarchical_copy": tf.distribute.HierarchicalCopyAllReduce + } + if all_reduce_alg not in mirrored_all_reduce_options: + raise ValueError( + "When used with `mirrored`, valid values for all_reduce_alg are " + "[`nccl`, `hierarchical_copy`]. Supplied value: {}".format( + all_reduce_alg)) + cross_device_ops_class = mirrored_all_reduce_options[all_reduce_alg] + return cross_device_ops_class(num_packs=num_packs) + + +def tpu_initialize(tpu_address): + """Initializes TPU for TF 2.x training. + + Args: + tpu_address: string, bns address of master TPU worker. + + Returns: + A TPUClusterResolver. + """ + cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver( + tpu=tpu_address) + if tpu_address not in ("", "local"): + tf.config.experimental_connect_to_cluster(cluster_resolver) + tf.tpu.experimental.initialize_tpu_system(cluster_resolver) + return cluster_resolver + + +def get_distribution_strategy(distribution_strategy="mirrored", + num_gpus=0, + all_reduce_alg=None, + num_packs=1, + tpu_address=None, + **kwargs): + """Return a DistributionStrategy for running the model. + + Args: + distribution_strategy: a string specifying which distribution strategy to + use. Accepted values are "off", "one_device", "mirrored", + "parameter_server", "multi_worker_mirrored", and "tpu" -- case + insensitive. "tpu" means to use TPUStrategy using `tpu_address`. + "off" means to use the default strategy which is obtained from + tf.distribute.get_strategy (for details on the default strategy, see + https://www.tensorflow.org/guide/distributed_training#default_strategy). + num_gpus: Number of GPUs to run this model. + all_reduce_alg: Optional. Specifies which algorithm to use when performing + all-reduce. For `MirroredStrategy`, valid values are "nccl" and + "hierarchical_copy". For `MultiWorkerMirroredStrategy`, valid values are + "ring" and "nccl". If None, DistributionStrategy will choose based on + device topology. + num_packs: Optional. Sets the `num_packs` in `tf.distribute.NcclAllReduce` + or `tf.distribute.HierarchicalCopyAllReduce` for `MirroredStrategy`. + tpu_address: Optional. String that represents TPU to connect to. Must not be + None if `distribution_strategy` is set to `tpu`. + **kwargs: Additional kwargs for internal usages. + + Returns: + tf.distribute.DistibutionStrategy object. + Raises: + ValueError: if `distribution_strategy` is "off" or "one_device" and + `num_gpus` is larger than 1; or `num_gpus` is negative or if + `distribution_strategy` is `tpu` but `tpu_address` is not specified. + """ + del kwargs + if num_gpus < 0: + raise ValueError("`num_gpus` can not be negative.") + + if not isinstance(distribution_strategy, str): + msg = ("distribution_strategy must be a string but got: %s." % + (distribution_strategy,)) + if distribution_strategy == False: # pylint: disable=singleton-comparison,g-explicit-bool-comparison + msg += (" If you meant to pass the string 'off', make sure you add " + "quotes around 'off' so that yaml interprets it as a string " + "instead of a bool.") + raise ValueError(msg) + + distribution_strategy = distribution_strategy.lower() + if distribution_strategy == "off": + if num_gpus > 1: + raise ValueError(f"When {num_gpus} GPUs are specified, " + "distribution_strategy flag cannot be set to `off`.") + # Return the default distribution strategy. + return tf.distribute.get_strategy() + + if distribution_strategy == "tpu": + # When tpu_address is an empty string, we communicate with local TPUs. + cluster_resolver = tpu_initialize(tpu_address) + return tf.distribute.TPUStrategy(cluster_resolver) + + if distribution_strategy == "multi_worker_mirrored": + return tf.distribute.experimental.MultiWorkerMirroredStrategy( + communication=_collective_communication(all_reduce_alg)) + + if distribution_strategy == "one_device": + if num_gpus == 0: + return tf.distribute.OneDeviceStrategy("device:CPU:0") + if num_gpus > 1: + raise ValueError("`OneDeviceStrategy` can not be used for more than " + "one device.") + return tf.distribute.OneDeviceStrategy("device:GPU:0") + + if distribution_strategy == "mirrored": + if num_gpus == 0: + devices = ["device:CPU:0"] + else: + devices = ["device:GPU:%d" % i for i in range(num_gpus)] + return tf.distribute.MirroredStrategy( + devices=devices, + cross_device_ops=_mirrored_cross_device_ops(all_reduce_alg, num_packs)) + + if distribution_strategy == "parameter_server": + cluster_resolver = tf.distribute.cluster_resolver.TFConfigClusterResolver() + return tf.distribute.experimental.ParameterServerStrategy(cluster_resolver) + + raise ValueError("Unrecognized Distribution Strategy: %r" % + distribution_strategy) + + +def configure_cluster(worker_hosts=None, task_index=-1): + """Set multi-worker cluster spec in TF_CONFIG environment variable. + + Args: + worker_hosts: comma-separated list of worker ip:port pairs. + task_index: index of the worker. + + Returns: + Number of workers in the cluster. + """ + tf_config = json.loads(os.environ.get("TF_CONFIG", "{}")) + if tf_config: + num_workers = ( + len(tf_config["cluster"].get("chief", [])) + + len(tf_config["cluster"].get("worker", []))) + elif worker_hosts: + workers = worker_hosts.split(",") + num_workers = len(workers) + if num_workers > 1 and task_index < 0: + raise ValueError("Must specify task_index when number of workers > 1") + task_index = 0 if num_workers == 1 else task_index + os.environ["TF_CONFIG"] = json.dumps({ + "cluster": { + "worker": workers + }, + "task": { + "type": "worker", + "index": task_index + } + }) + else: + num_workers = 1 + return num_workers + + +def get_strategy_scope(strategy): + if strategy: + strategy_scope = strategy.scope() + else: + strategy_scope = DummyContextManager() + + return strategy_scope + + +class DummyContextManager(object): + + def __enter__(self): + pass + + def __exit__(self, *args): + pass diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/common/distribute_utils_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/common/distribute_utils_test.py new file mode 100644 index 0000000000000000000000000000000000000000..9de8fe82a0580173e0f7f934f7b4ba317556a3d4 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/common/distribute_utils_test.py @@ -0,0 +1,102 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for distribution util functions.""" + +import tensorflow as tf + +from official.common import distribute_utils + + +class DistributeUtilsTest(tf.test.TestCase): + """Tests for distribute util functions.""" + + def test_invalid_args(self): + with self.assertRaisesRegex(ValueError, '`num_gpus` can not be negative.'): + _ = distribute_utils.get_distribution_strategy(num_gpus=-1) + + with self.assertRaisesRegex(ValueError, + '.*If you meant to pass the string .*'): + _ = distribute_utils.get_distribution_strategy( + distribution_strategy=False, num_gpus=0) + with self.assertRaisesRegex(ValueError, 'When 2 GPUs are specified.*'): + _ = distribute_utils.get_distribution_strategy( + distribution_strategy='off', num_gpus=2) + with self.assertRaisesRegex(ValueError, + '`OneDeviceStrategy` can not be used.*'): + _ = distribute_utils.get_distribution_strategy( + distribution_strategy='one_device', num_gpus=2) + + def test_one_device_strategy_cpu(self): + ds = distribute_utils.get_distribution_strategy('one_device', num_gpus=0) + self.assertEquals(ds.num_replicas_in_sync, 1) + self.assertEquals(len(ds.extended.worker_devices), 1) + self.assertIn('CPU', ds.extended.worker_devices[0]) + + def test_one_device_strategy_gpu(self): + ds = distribute_utils.get_distribution_strategy('one_device', num_gpus=1) + self.assertEquals(ds.num_replicas_in_sync, 1) + self.assertEquals(len(ds.extended.worker_devices), 1) + self.assertIn('GPU', ds.extended.worker_devices[0]) + + def test_mirrored_strategy(self): + ds = distribute_utils.get_distribution_strategy(num_gpus=5) + self.assertEquals(ds.num_replicas_in_sync, 5) + self.assertEquals(len(ds.extended.worker_devices), 5) + for device in ds.extended.worker_devices: + self.assertIn('GPU', device) + + _ = distribute_utils.get_distribution_strategy( + distribution_strategy='mirrored', + num_gpus=2, + all_reduce_alg='nccl', + num_packs=2) + with self.assertRaisesRegex( + ValueError, + 'When used with `mirrored`, valid values for all_reduce_alg are.*'): + _ = distribute_utils.get_distribution_strategy( + distribution_strategy='mirrored', + num_gpus=2, + all_reduce_alg='dummy', + num_packs=2) + + def test_mwms(self): + distribute_utils.configure_cluster(worker_hosts=None, task_index=-1) + ds = distribute_utils.get_distribution_strategy( + 'multi_worker_mirrored', all_reduce_alg='nccl') + self.assertIsInstance( + ds, tf.distribute.experimental.MultiWorkerMirroredStrategy) + + def test_no_strategy(self): + ds = distribute_utils.get_distribution_strategy('off') + self.assertIs(ds, tf.distribute.get_strategy()) + + def test_invalid_strategy(self): + with self.assertRaisesRegexp( + ValueError, + 'distribution_strategy must be a string but got: False. If'): + distribute_utils.get_distribution_strategy(False) + with self.assertRaisesRegexp( + ValueError, 'distribution_strategy must be a string but got: 1'): + distribute_utils.get_distribution_strategy(1) + + def test_get_strategy_scope(self): + ds = distribute_utils.get_distribution_strategy('one_device', num_gpus=0) + with distribute_utils.get_strategy_scope(ds): + self.assertIs(tf.distribute.get_strategy(), ds) + with distribute_utils.get_strategy_scope(None): + self.assertIsNot(tf.distribute.get_strategy(), ds) + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/common/flags.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/common/flags.py new file mode 100644 index 0000000000000000000000000000000000000000..01ddf57af3872b0ad6f425602b5029ece9def707 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/common/flags.py @@ -0,0 +1,110 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The central place to define flags.""" + +from absl import flags + + +def define_flags(): + """Defines flags. + + All flags are defined as optional, but in practice most models use some of + these flags and so mark_flags_as_required() should be called after calling + this function. Typically, 'experiment', 'mode', and 'model_dir' are required. + For example: + + ``` + from absl import flags + from official.common import flags as tfm_flags # pylint: disable=line-too-long + ... + tfm_flags.define_flags() + flags.mark_flags_as_required(['experiment', 'mode', 'model_dir']) + ``` + + The reason all flags are optional is because unit tests often do not set or + use any of the flags. + """ + flags.DEFINE_string( + 'experiment', default=None, help= + 'The experiment type registered, specifying an ExperimentConfig.') + + flags.DEFINE_enum( + 'mode', + default=None, + enum_values=[ + 'train', 'eval', 'train_and_eval', 'continuous_eval', + 'continuous_train_and_eval', 'train_and_validate' + ], + help='Mode to run: `train`, `eval`, `train_and_eval`, ' + '`continuous_eval`, `continuous_train_and_eval` and ' + '`train_and_validate` (which is not implemented in ' + 'the open source version).') + + flags.DEFINE_string( + 'model_dir', + default=None, + help='The directory where the model and training/evaluation summaries' + 'are stored.') + + flags.DEFINE_multi_string( + 'config_file', + default=None, + help='YAML/JSON files which specifies overrides. The override order ' + 'follows the order of args. Note that each file ' + 'can be used as an override template to override the default parameters ' + 'specified in Python. If the same parameter is specified in both ' + '`--config_file` and `--params_override`, `config_file` will be used ' + 'first, followed by params_override.') + + flags.DEFINE_string( + 'params_override', + default=None, + help='a YAML/JSON string or a YAML file which specifies additional ' + 'overrides over the default parameters and those specified in ' + '`--config_file`. Note that this is supposed to be used only to override ' + 'the model parameters, but not the parameters like TPU specific flags. ' + 'One canonical use case of `--config_file` and `--params_override` is ' + 'users first define a template config file using `--config_file`, then ' + 'use `--params_override` to adjust the minimal set of tuning parameters, ' + 'for example setting up different `train_batch_size`. The final override ' + 'order of parameters: default_model_params --> params from config_file ' + '--> params in params_override. See also the help message of ' + '`--config_file`.') + + # The libraries rely on gin often make mistakes that include flags inside + # the library files which causes conflicts. + try: + flags.DEFINE_multi_string( + 'gin_file', default=None, help='List of paths to the config files.') + except flags.DuplicateFlagError: + pass + + try: + flags.DEFINE_multi_string( + 'gin_params', + default=None, + help='Newline separated list of Gin parameter bindings.') + except flags.DuplicateFlagError: + pass + + flags.DEFINE_string( + 'tpu', + default=None, + help='The Cloud TPU to use for training. This should be either the name ' + 'used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 ' + 'url.') + + flags.DEFINE_string( + 'tf_data_service', default=None, help='The tf.data service address') diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/common/registry_imports.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/common/registry_imports.py new file mode 100644 index 0000000000000000000000000000000000000000..06f3384db6283cbef08070f3678d0afe36e50c08 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/common/registry_imports.py @@ -0,0 +1,20 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""All necessary imports for registration.""" +# pylint: disable=unused-import +from official.nlp import tasks +from official.nlp.configs import experiment_configs +from official.utils.testing import mock_task +from official.vision import beta diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/common/streamz_counters.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/common/streamz_counters.py new file mode 100644 index 0000000000000000000000000000000000000000..ab3df36ce6077d2dafd25eb199fc0370852795e5 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/common/streamz_counters.py @@ -0,0 +1,27 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Global streamz counters.""" + +from tensorflow.python.eager import monitoring + + +progressive_policy_creation_counter = monitoring.Counter( + "/tensorflow/training/fast_training/progressive_policy_creation", + "Counter for the number of ProgressivePolicy creations.") + + +stack_vars_to_vars_call_counter = monitoring.Counter( + "/tensorflow/training/fast_training/tf_vars_to_vars", + "Counter for the number of low-level stacking API calls.") diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/__init__.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e419af524b5f349fe04abfa820c3cb51b777d422 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/actions.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/actions.py new file mode 100644 index 0000000000000000000000000000000000000000..6fc4f943686f1c092cf2b8842b98e67600325989 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/actions.py @@ -0,0 +1,222 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provides TFM orbit actions and associated helper functions/classes.""" + +import os +from typing import List +from absl import logging + +import gin +import orbit +import tensorflow as tf +import tensorflow_model_optimization as tfmot + +from official.core import base_trainer +from official.core import config_definitions +from official.modeling import optimization + + +class PruningActions: + """Train action to updates pruning related information. + + This action updates pruning steps at the end of trainig loop, and log + pruning metrics to tensorboard. + + This action must be used when training a pruned model to avoid pruning error. + """ + + def __init__( + self, + export_dir: str, + model: tf.keras.Model, + optimizer: tf.keras.optimizers.Optimizer, + ): + """Initializes the instance. + + Args: + export_dir: `str` for the export directory of the pruning summaries. + model: `tf.keras.Model` model instance used for training. This will be + used to assign a pruning step to each prunable weight. + optimizer: `tf.keras.optimizers.Optimizer` optimizer instance used for + training. This will be used to find the current training steps. + """ + self._optimizer = optimizer + self.update_pruning_step = tfmot.sparsity.keras.UpdatePruningStep() + self.update_pruning_step.set_model(model) + self.update_pruning_step.on_train_begin() + + self.pruning_summaries = tfmot.sparsity.keras.PruningSummaries( + log_dir=export_dir) + model.optimizer = optimizer + self.pruning_summaries.set_model(model) + + def __call__(self, output: orbit.runner.Output): + """Update pruning step and log pruning summaries. + + Args: + output: The train output to test. + """ + self.update_pruning_step.on_epoch_end(batch=None) + self.pruning_summaries.on_epoch_begin(epoch=None) + + +class EMACheckpointing: + """Eval action to save checkpoint with average weights when EMA is used. + + This action swaps the weights of the model with the average weights, then it + saves the checkpoint under export_dir/ema_checkpoints. Checkpointing is + expensive for large models, so doing this action in eval is more efficient + than training. + """ + + def __init__(self, export_dir: str, optimizer: tf.keras.optimizers.Optimizer, + checkpoint: tf.train.Checkpoint, max_to_keep: int = 1): + """Initializes the instance. + + Args: + export_dir: `str` for the export directory of the EMA average weights. + optimizer: `tf.keras.optimizers.Optimizer` optimizer instance used for + training. This will be used to swap the model weights with the average + weigths. + checkpoint: `tf.train.Checkpoint` instance. + max_to_keep: `int` for max checkpoints to keep in ema_checkpoints subdir. + """ + if not isinstance(optimizer, optimization.ExponentialMovingAverage): + raise ValueError('Optimizer has to be instance of' + 'optimization.ExponentialMovingAverage for' + 'EMACheckpointing action') + + export_dir = os.path.join(export_dir, 'ema_checkpoints') + tf.io.gfile.makedirs( + os.path.dirname(export_dir)) + self._optimizer = optimizer + self._checkpoint = checkpoint + self._checkpoint_manager = tf.train.CheckpointManager( + checkpoint, + directory=export_dir, + max_to_keep=max_to_keep, + checkpoint_name='average_weights') + + def __call__(self, output: orbit.runner.Output): + """Swaps model weights, and saves the checkpoint. + + Args: + output: The train or eval output to test. + """ + self._optimizer.swap_weights() + self._checkpoint_manager.save(checkpoint_number=self._optimizer.iterations) + self._optimizer.swap_weights() + + +class RecoveryAction: + """Train action to recover from loss blowup. + + Checks the loss value by the given threshold. If applicable, recover the + model by reading the checkpoint on disk. + """ + + def __init__(self, checkpoint_manager: tf.train.CheckpointManager): + self.checkpoint_manager = checkpoint_manager + + def __call__(self, _): + """Recovers the training by triggering checkpoint restoration.""" + # Loads the previous good checkpoint. + checkpoint_path = self.checkpoint_manager.restore_or_initialize() + logging.warning('Recovering the model from checkpoint: %s.', + checkpoint_path) + + +class RecoveryCondition: + """Recovery Condition.""" + + def __init__(self, + global_step: tf.Variable, + loss_upper_bound: float, + recovery_begin_steps: int = 0, + recovery_max_trials: int = 3): + self.recover_counter = 0 + self.recovery_begin_steps = recovery_begin_steps + self.recovery_max_trials = recovery_max_trials + self.loss_upper_bound = loss_upper_bound + self.global_step = global_step + + def __call__(self, outputs: orbit.runner.Output): + loss_value = outputs['training_loss'] + if tf.math.is_nan(loss_value): + self.recover_counter += 1 + if self.recover_counter > self.recovery_max_trials: + raise RuntimeError( + 'The loss value is NaN after training loop and it happens %d times.' + % self.recover_counter) + return True + if (self.global_step >= self.recovery_begin_steps and + loss_value > self.loss_upper_bound): + self.recover_counter += 1 + if self.recover_counter > self.recovery_max_trials: + raise RuntimeError( + f'The loss value is {loss_value}, which is larger than the bound {self.loss_upper_bound}, happens {self.recover_counter} times.' + ) + return True + return False + + +@gin.configurable +def get_eval_actions( + params: config_definitions.ExperimentConfig, + trainer: base_trainer.Trainer, + model_dir: str) -> List[orbit.Action]: + """Gets eval actions for TFM trainer.""" + eval_actions = [] + # Adds ema checkpointing action to save the average weights under + # ema_checkpoints subdir. + if isinstance(trainer.optimizer, optimization.ExponentialMovingAverage): + eval_actions.append( + EMACheckpointing( + export_dir=model_dir, + optimizer=trainer.optimizer, + checkpoint=trainer.checkpoint, + max_to_keep=params.trainer.max_to_keep)) + + return eval_actions + + +@gin.configurable +def get_train_actions( + params: config_definitions.ExperimentConfig, trainer: base_trainer.Trainer, + model_dir: str, + checkpoint_manager: tf.train.CheckpointManager) -> List[orbit.Action]: + """Gets train actions for TFM trainer.""" + train_actions = [] + # Adds pruning callback actions. + if hasattr(params.task, 'pruning'): + train_actions.append( + PruningActions( + export_dir=model_dir, + model=trainer.model, + optimizer=trainer.optimizer)) + + if params.trainer.recovery_max_trials >= 0: + recovery_condition = RecoveryCondition( + global_step=trainer.global_step, + loss_upper_bound=params.trainer.loss_upper_bound, + recovery_begin_steps=params.trainer.recovery_begin_steps, + recovery_max_trials=params.trainer.recovery_max_trials, + ) + recover_action = orbit.actions.ConditionalAction( + condition=recovery_condition, + action=RecoveryAction(checkpoint_manager), + ) + train_actions.append(recover_action) + return train_actions diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/actions_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/actions_test.py new file mode 100644 index 0000000000000000000000000000000000000000..22cb55bd7fc3e558df42ac2ad6a55c42ce0554c3 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/actions_test.py @@ -0,0 +1,107 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for TFM actions.""" + +import os + +from absl.testing import parameterized +import numpy as np +import orbit +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.core import actions +from official.modeling import optimization + + +class TestModel(tf.Module): + + def __init__(self): + self.value = tf.Variable(0) + + @tf.function(input_signature=[]) + def __call__(self): + return self.value + + +class ActionsTest(tf.test.TestCase, parameterized.TestCase): + + @combinations.generate( + combinations.combine( + distribution=[ + strategy_combinations.cloud_tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ],)) + def test_ema_checkpointing(self, distribution): + with distribution.scope(): + directory = self.create_tempdir() + model = TestModel() + optimizer = tf.keras.optimizers.SGD() + optimizer = optimization.ExponentialMovingAverage( + optimizer, trainable_weights_only=False) + + # Creats average weights for the model variables. Average weights are + # initialized to zero. + optimizer.shadow_copy(model) + checkpoint = tf.train.Checkpoint(model=model) + + # Changes model.value to 3, average value is still 0. + model.value.assign(3) + + # Checks model.value is 3 + self.assertEqual(model(), 3) + ema_action = actions.EMACheckpointing(directory, optimizer, checkpoint) + + ema_action({}) + self.assertNotEmpty( + tf.io.gfile.glob(os.path.join(directory, 'ema_checkpoints'))) + + checkpoint.read(tf.train.latest_checkpoint( + os.path.join(directory, 'ema_checkpoints'))) + + # Checks model.value is 0 after swapping. + self.assertEqual(model(), 0) + + @combinations.generate( + combinations.combine( + distribution=[ + strategy_combinations.default_strategy, + strategy_combinations.cloud_tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ],)) + def test_recovery_condition(self, distribution): + with distribution.scope(): + global_step = orbit.utils.create_global_step() + recover_condition = actions.RecoveryCondition( + global_step, loss_upper_bound=0.5, recovery_max_trials=2) + outputs = {'training_loss': 0.6} + self.assertTrue(recover_condition(outputs)) + self.assertTrue(recover_condition(outputs)) + with self.assertRaises(RuntimeError): + recover_condition(outputs) + + global_step = orbit.utils.create_global_step() + recover_condition = actions.RecoveryCondition( + global_step, loss_upper_bound=0.5, recovery_max_trials=2) + outputs = {'training_loss': tf.constant([np.nan], tf.float32)} + self.assertTrue(recover_condition(outputs)) + self.assertTrue(recover_condition(outputs)) + with self.assertRaises(RuntimeError): + recover_condition(outputs) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/base_task.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/base_task.py new file mode 100644 index 0000000000000000000000000000000000000000..db29395d66eb24f0ea0465838ea5947c2545fb6b --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/base_task.py @@ -0,0 +1,335 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Defines the base task abstraction.""" +import abc +from typing import Optional + +from absl import logging +import tensorflow as tf + +from official.core import config_definitions +from official.modeling import optimization +from official.modeling import performance + +OptimizationConfig = optimization.OptimizationConfig +RuntimeConfig = config_definitions.RuntimeConfig + + +class Task(tf.Module, metaclass=abc.ABCMeta): + """A single-replica view of training procedure. + + Tasks provide artifacts for training/validation procedures, including + loading/iterating over Datasets, training/validation steps, calculating the + loss and customized metrics with reduction. + """ + + # Special keys in train/validate step returned logs. + loss = "loss" + + def __init__(self, + params, + logging_dir: Optional[str] = None, + name: Optional[str] = None): + """Task initialization. + + Args: + params: the task configuration instance, which can be any of dataclass, + ConfigDict, namedtuple, etc. + logging_dir: a string pointing to where the model, summaries etc. will be + saved. You can also write additional stuff in this directory. + name: the task name. + """ + super().__init__(name=name) + self._task_config = params + self._logging_dir = logging_dir + + @property + def task_config(self): + return self._task_config + + @property + def logging_dir(self) -> str: + return self._logging_dir + + @classmethod + def create_optimizer(cls, optimizer_config: OptimizationConfig, + runtime_config: Optional[RuntimeConfig] = None): + """Creates an TF optimizer from configurations. + + Args: + optimizer_config: the parameters of the Optimization settings. + runtime_config: the parameters of the runtime. + + Returns: + A tf.optimizers.Optimizer object. + """ + opt_factory = optimization.OptimizerFactory(optimizer_config) + optimizer = opt_factory.build_optimizer(opt_factory.build_learning_rate()) + # Configuring optimizer when loss_scale is set in runtime config. This helps + # avoiding overflow/underflow for float16 computations. + if runtime_config: + optimizer = performance.configure_optimizer( + optimizer, + use_float16=runtime_config.mixed_precision_dtype == "float16", + loss_scale=runtime_config.loss_scale) + + return optimizer + + def initialize(self, model: tf.keras.Model): + """[Optional] A callback function used as CheckpointManager's init_fn. + + This function will be called when no checkpoint is found for the model. + If there is a checkpoint, the checkpoint will be loaded and this function + will not be called. You can use this callback function to load a pretrained + checkpoint, saved under a directory other than the model_dir. + + Args: + model: The keras.Model built or used by this task. + """ + ckpt_dir_or_file = self.task_config.init_checkpoint + logging.info("Trying to load pretrained checkpoint from %s", + ckpt_dir_or_file) + if tf.io.gfile.isdir(ckpt_dir_or_file): + ckpt_dir_or_file = tf.train.latest_checkpoint(ckpt_dir_or_file) + if not ckpt_dir_or_file: + return + + if hasattr(model, "checkpoint_items"): + checkpoint_items = model.checkpoint_items + else: + checkpoint_items = dict(model=model) + ckpt = tf.train.Checkpoint(**checkpoint_items) + status = ckpt.read(ckpt_dir_or_file) + status.expect_partial().assert_existing_objects_matched() + logging.info("Finished loading pretrained checkpoint from %s", + ckpt_dir_or_file) + + def build_model(self) -> tf.keras.Model: + """[Optional] Creates model architecture. + + Returns: + A model instance. + """ # pytype: disable=bad-return-type # typed-keras + + @abc.abstractmethod + def build_inputs(self, + params, + input_context: Optional[tf.distribute.InputContext] = None): + """Returns a dataset or a nested structure of dataset functions. + + Dataset functions define per-host datasets with the per-replica batch size. + With distributed training, this method runs on remote hosts. + + Args: + params: hyperparams to create input pipelines, which can be any of + dataclass, ConfigDict, namedtuple, etc. + input_context: optional distribution input pipeline context. + + Returns: + A nested structure of per-replica input functions. + """ + + def build_losses(self, labels, model_outputs, aux_losses=None) -> tf.Tensor: + """Standard interface to compute losses. + + Args: + labels: optional label tensors. + model_outputs: a nested structure of output tensors. + aux_losses: auxiliary loss tensors, i.e. `losses` in keras.Model. + + Returns: + The total loss tensor. + """ + del model_outputs, labels + + if aux_losses is None: + losses = [tf.constant(0.0, dtype=tf.float32)] + else: + losses = aux_losses + total_loss = tf.add_n(losses) + return total_loss + + def build_metrics(self, training: bool = True): + """Gets streaming metrics for training/validation.""" + del training + return [] + + def process_metrics(self, metrics, labels, model_outputs, **kwargs): + """Process and update metrics. + + Called when using custom training loop API. + + Args: + metrics: a nested structure of metrics objects. The return of function + self.build_metrics. + labels: a tensor or a nested structure of tensors. + model_outputs: a tensor or a nested structure of tensors. For example, + output of the keras model built by self.build_model. + **kwargs: other args. + """ + for metric in metrics: + metric.update_state(labels, model_outputs) + + def process_compiled_metrics(self, compiled_metrics, labels, model_outputs): + """Process and update compiled_metrics. + + call when using compile/fit API. + + Args: + compiled_metrics: the compiled metrics (model.compiled_metrics). + labels: a tensor or a nested structure of tensors. + model_outputs: a tensor or a nested structure of tensors. For example, + output of the keras model built by self.build_model. + """ + compiled_metrics.update_state(labels, model_outputs) + + def train_step(self, + inputs, + model: tf.keras.Model, + optimizer: tf.keras.optimizers.Optimizer, + metrics=None): + """Does forward and backward. + + With distribution strategies, this method runs on devices. + + Args: + inputs: a dictionary of input tensors. + model: the model, forward pass definition. + optimizer: the optimizer for this training step. + metrics: a nested structure of metrics objects. + + Returns: + A dictionary of logs. + """ + if isinstance(inputs, tuple) and len(inputs) == 2: + features, labels = inputs + else: + features, labels = inputs, inputs + with tf.GradientTape() as tape: + outputs = model(features, training=True) + # Computes per-replica loss. + if model.compiled_loss: + loss = model.compiled_loss( + labels, outputs, regularization_losses=model.losses) + loss += self.build_losses( + labels=labels, model_outputs=outputs, aux_losses=None) + else: + loss = self.build_losses( + labels=labels, model_outputs=outputs, aux_losses=model.losses) + # Scales loss as the default gradients allreduce performs sum inside the + # optimizer. + scaled_loss = loss / tf.distribute.get_strategy().num_replicas_in_sync + + # For mixed precision, when a LossScaleOptimizer is used, the loss is + # scaled to avoid numeric underflow. + if isinstance(optimizer, + tf.keras.mixed_precision.LossScaleOptimizer): + scaled_loss = optimizer.get_scaled_loss(scaled_loss) + + tvars = model.trainable_variables + grads = tape.gradient(scaled_loss, tvars) + + if isinstance(optimizer, + tf.keras.mixed_precision.LossScaleOptimizer): + grads = optimizer.get_unscaled_gradients(grads) + optimizer.apply_gradients(list(zip(grads, tvars))) + logs = {self.loss: loss} + if metrics: + self.process_metrics(metrics, labels, outputs) + if model.compiled_metrics: + self.process_compiled_metrics(model.compiled_metrics, labels, outputs) + logs.update({m.name: m.result() for m in metrics or []}) + logs.update({m.name: m.result() for m in model.metrics}) + return logs + + def validation_step(self, inputs, model: tf.keras.Model, metrics=None): + """Validation step. + + With distribution strategies, this method runs on devices. + + Args: + inputs: a dictionary of input tensors. + model: the keras.Model. + metrics: a nested structure of metrics objects. + + Returns: + A dictionary of logs. + """ + if isinstance(inputs, tuple) and len(inputs) == 2: + features, labels = inputs + else: + features, labels = inputs, inputs + outputs = self.inference_step(features, model) + loss = self.build_losses( + labels=labels, model_outputs=outputs, aux_losses=model.losses) + logs = {self.loss: loss} + if metrics: + self.process_metrics(metrics, labels, outputs) + if model.compiled_metrics: + self.process_compiled_metrics(model.compiled_metrics, labels, outputs) + logs.update({m.name: m.result() for m in metrics or []}) + logs.update({m.name: m.result() for m in model.metrics}) + return logs + + def inference_step(self, inputs, model: tf.keras.Model): + """Performs the forward step. + + With distribution strategies, this method runs on devices. + + Args: + inputs: a dictionary of input tensors. + model: the keras.Model. + + Returns: + Model outputs. + """ + return model(inputs, training=False) + + def aggregate_logs(self, state, step_logs): + """Optional aggregation over logs returned from a validation step. + + Given step_logs from a validation step, this function aggregates the logs + after each eval_step() (see eval_reduce() function in + official/core/base_trainer.py). It runs on CPU and can be used to aggregate + metrics during validation, when there are too many metrics that cannot fit + into TPU memory. Note that this may increase latency due to data transfer + between TPU and CPU. Also, the step output from a validation step may be a + tuple with elements from replicas, and a concatenation of the elements is + needed in such case. + + Args: + state: The current state of training, for example, it can be a sequence of + metrics. + step_logs: Logs from a validation step. Can be a dictionary. + """ + pass + + def reduce_aggregated_logs(self, + aggregated_logs, + global_step: Optional[tf.Tensor] = None): + """Optional reduce of aggregated logs over validation steps. + + This function reduces aggregated logs at the end of validation, and can be + used to compute the final metrics. It runs on CPU and in each eval_end() in + base trainer (see eval_end() function in official/core/base_trainer.py). + + Args: + aggregated_logs: Aggregated logs over multiple validation steps. + global_step: An optional variable of global step. + + Returns: + A dictionary of reduced results. + """ + return {} diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/base_trainer.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/base_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..c2983314dbd7672aa893461f50c42bca5fe9cdf1 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/base_trainer.py @@ -0,0 +1,477 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Standard Trainer implementation. + +The base trainer implements the Orbit `StandardTrainable` and +`StandardEvaluable` interfaces. Trainers inside this project should be +interchangable and independent on model architectures and tasks. +""" +import functools +from typing import Union, Optional +from absl import logging +import gin +import orbit +import tensorflow as tf + +from official.core import base_task +from official.core import config_definitions +from official.modeling import optimization + +ExperimentConfig = config_definitions.ExperimentConfig +TrainerConfig = config_definitions.TrainerConfig + + +class Recovery: + """Built-in model blowup recovery module. + + Checks the loss value by the given threshold. If applicable, recover the + model by reading the checkpoint on disk. + """ + + def __init__(self, + loss_upper_bound: float, + checkpoint_manager: tf.train.CheckpointManager, + recovery_begin_steps: int = 0, + recovery_max_trials: int = 3): + self.recover_counter = 0 + self.recovery_begin_steps = recovery_begin_steps + self.recovery_max_trials = recovery_max_trials + self.loss_upper_bound = loss_upper_bound + self.checkpoint_manager = checkpoint_manager + + def should_recover(self, loss_value, global_step): + if tf.math.is_nan(loss_value): + return True + if (global_step >= self.recovery_begin_steps and + loss_value > self.loss_upper_bound): + return True + return False + + def maybe_recover(self, loss_value, global_step): + """Conditionally recovers the training by triggering checkpoint restoration. + + Args: + loss_value: the loss value as a float. + global_step: the number of global training steps. + + Raises: + RuntimeError: when recovery happens more than the max number of trials, + the job should crash. + """ + if not self.should_recover(loss_value, global_step): + return + self.recover_counter += 1 + if self.recover_counter > self.recovery_max_trials: + raise RuntimeError( + "The loss value is NaN or out of range after training loop and " + f"this happens {self.recover_counter} times.") + # Loads the previous good checkpoint. + checkpoint_path = self.checkpoint_manager.restore_or_initialize() + logging.warning( + "Recovering the model from checkpoint: %s. The loss value becomes " + "%f at step %d.", checkpoint_path, loss_value, global_step) + + +class _AsyncTrainer(orbit.StandardTrainer, orbit.StandardEvaluator): + """Trainer class for both sync and async Strategy.""" + + def init_async(self): + """Initializes the Async Trainer base class.""" + assert isinstance(self._strategy, tf.distribute.Strategy) + self._is_async = isinstance( + self._strategy, tf.distribute.experimental.ParameterServerStrategy) + self._coordinator = None + if self._is_async: + self._coordinator = ( + tf.distribute.experimental.coordinator.ClusterCoordinator( + self._strategy)) + + def join(self): + """Join all async steps. Only useful in aysnc training.""" + if getattr(self, "_is_async", False): + self._coordinator.join() + + def create_train_loop_fn(self): + """Creates a eval loop from the given step function and options.""" + train_loop_fn = super().create_train_loop_fn() + if getattr(self, "_is_async", False): + + def _async_loop_fn(iterator, num_steps): + self._coordinator.schedule(train_loop_fn, args=(iterator, num_steps)) + + return _async_loop_fn + else: + return train_loop_fn + + def create_eval_loop_fn(self, has_state: bool): + """Creates a training loop from the given step function and options.""" + eval_loop_fn = super().create_eval_loop_fn(has_state) + + if getattr(self, "_is_async", False): + if has_state: + raise ValueError( + "Stateful eval loop is not supported in async training.") + + def _async_loop_fn(iterator, num_steps, state=None, reduce_fn=None): + assert state is None + assert reduce_fn is None + self._coordinator.schedule(eval_loop_fn, args=(iterator, num_steps)) + + return _async_loop_fn + else: + return eval_loop_fn + + def distribute_dataset(self, dataset_or_fn, *args, **kwargs): + """A utility function to help create a `tf.distribute.DistributedDataset`. + + Args: + dataset_or_fn: A instance of `tf.data.Dataset`, or a "dataset function" + returning a `tf.data.Dataset`. If it is a function, it may optionally + have an argument named `input_context` which will be passed a + `tf.distribute.InputContext` instance. + *args: Any positional arguments to pass through to `dataset_or_fn`. + **kwargs: Any keyword arguments to pass through to `dataset_or_fn`. + + Returns: + A distributed Dataset. + """ + if getattr(self, "_is_async", False): + per_worker_dataset_fn = functools.partial( + orbit.utils.make_distributed_dataset, self._strategy, dataset_or_fn, + *args, **kwargs) + per_worker_dataset_fn = tf.function(per_worker_dataset_fn) + + return self._coordinator.create_per_worker_dataset(per_worker_dataset_fn) + else: + return orbit.utils.make_distributed_dataset(self._strategy, dataset_or_fn, + *args, **kwargs) + + +def get_runtime_options(config: ExperimentConfig): + """Get tf.distribute.RunOptions from config.""" + xla_options = {} + if config.runtime.tpu_enable_xla_dynamic_padder is not None: + xla_options["enable_xla_dynamic_padder"] = ( + config.runtime.tpu_enable_xla_dynamic_padder) + return tf.distribute.RunOptions( + experimental_xla_options=tf.tpu.XLAOptions(**xla_options)) + + +@gin.configurable +class Trainer(_AsyncTrainer): + """Implements the common trainer shared for TensorFlow models.""" + + # pylint: disable=super-init-not-called + def __init__( + self, + config: ExperimentConfig, + task: base_task.Task, + model: tf.keras.Model, + optimizer: tf.optimizers.Optimizer, + train: bool = True, + evaluate: bool = True, + train_dataset: Optional[Union[tf.data.Dataset, + tf.distribute.DistributedDataset]] = None, + validation_dataset: Optional[Union[ + tf.data.Dataset, tf.distribute.DistributedDataset]] = None, + checkpoint_exporter=None): + """Initialize common trainer for TensorFlow models. + + Args: + config: An `ExperimentConfig` instance specifying experiment config. + task: A base_task.Task instance. + model: The model instance, e.g. a tf.keras.Model instance. + optimizer: tf.optimizers.Optimizer instance. + train: bool, whether or not this trainer will be used for training. + default to True. + evaluate: bool, whether or not this trainer will be used for evaluation. + default to True. + train_dataset: a dataset object created for training. With tf.distribute, + it needs to be a `DistributedDataset`. + validation_dataset: a dataset object created for evaluation. With + tf.distribute, it needs to be a `DistributedDataset`. The evaluator will + create a dataset iterator for each eval round, so the dataset does not + need to repeat. + checkpoint_exporter: an object that has the `maybe_export_checkpoint` + interface. + """ + # Gets the current distribution strategy. If not inside any strategy scope, + # it gets a single-replica no-op strategy. + self._strategy = tf.distribute.get_strategy() + self._validate_params( + config, + check_train_data=train_dataset is None, + check_validation_data=validation_dataset is None) + self._config = config + self._task = task + self._model = model + self._optimizer = optimizer + self._checkpoint_exporter = checkpoint_exporter + self._recovery = None + # Runtime options are only applied to train_step. + # We use default for eval_step. + self._runtime_options = get_runtime_options(config) + + # Creates a shadow copy of the weights to store weights moving average. + if isinstance(self._optimizer, optimization.ExponentialMovingAverage + ) and not self._optimizer.has_shadow_copy: + self._optimizer.shadow_copy(self._model) + + # global_step increases by 1 after each training iteration. + # We should have global_step.numpy() == self.optimizer.iterations.numpy() + # when there is only 1 optimizer. + self._global_step = orbit.utils.create_global_step() + if hasattr(self.model, "checkpoint_items"): + checkpoint_items = self.model.checkpoint_items + else: + checkpoint_items = {} + self._checkpoint = tf.train.Checkpoint( + global_step=self.global_step, + model=self.model, + optimizer=self.optimizer, + **checkpoint_items) + + self._train_loss = tf.keras.metrics.Mean("training_loss", dtype=tf.float32) + self._validation_loss = tf.keras.metrics.Mean( + "validation_loss", dtype=tf.float32) + model_metrics = model.metrics if hasattr(model, "metrics") else [] + self._train_metrics = self.task.build_metrics( + training=True) + model_metrics + self._validation_metrics = self.task.build_metrics( + training=False) + model_metrics + + self.init_async() + + if train: + train_dataset = train_dataset or self.distribute_dataset( + self.task.build_inputs, self.config.task.train_data) + orbit.StandardTrainer.__init__( + self, + train_dataset, + options=orbit.StandardTrainerOptions( + use_tf_while_loop=config.trainer.train_tf_while_loop, + use_tf_function=config.trainer.train_tf_function, + use_tpu_summary_optimization=config.trainer.allow_tpu_summary)) + + if evaluate: + validation_dataset = validation_dataset or self.distribute_dataset( + self.task.build_inputs, self.config.task.validation_data) + orbit.StandardEvaluator.__init__( + self, + validation_dataset, + options=orbit.StandardEvaluatorOptions( + use_tf_function=config.trainer.eval_tf_function, + use_tf_while_loop=config.trainer.eval_tf_while_loop)) + + def _validate_params(self, + config, + check_train_data=True, + check_validation_data=True): + r"""Validates if the configuration object passed to the Trainer. + + The experiment configuration should be structured as: + \trainer + \task + \train_data + \validation_data + + Args: + config: a namedtuple, dataclass, ConfigDict, etc. + check_train_data: whether to check task.train_data field. + check_validation_data: whether to check task.validation_data field. + """ + if not hasattr(config, "trainer"): + raise AttributeError("The trainer requires the configuration contains an" + " attribute `trainer`.") + + if not hasattr(config, "task"): + raise AttributeError("The trainer requires the configuration contains an" + " attribute `task`.") + + if check_train_data and not hasattr(config.task, "train_data"): + raise AttributeError("The trainer requires the configuration contains an" + " attribute `task.train_data`.") + + if check_validation_data and not hasattr(config.task, "validation_data"): + raise AttributeError("The trainer requires the configuration contains an" + " attribute `task.validation_data`.") + + @property + def strategy(self): + return self._strategy + + @property + def config(self): + return self._config + + @property + def task(self): + return self._task + + @property + def model(self): + return self._model + + @property + def optimizer(self): + if hasattr(self, "_optimizer"): + return self._optimizer + else: + return None + + @property + def global_step(self): + return self._global_step + + @property + def train_loss(self): + """Accesses the training loss metric object.""" + return self._train_loss + + @property + def validation_loss(self): + """Accesses the validation loss metric object.""" + return self._validation_loss + + @property + def train_metrics(self): + """Accesses all training metric objects.""" + return self._train_metrics + + @property + def validation_metrics(self): + """Accesses all validation metric metric objects.""" + return self._validation_metrics + + def initialize(self): + """A callback function. + + This function will be called when no checkpoint found for the model. + If there is a checkpoint, the checkpoint will be loaded and this function + will not be called. Tasks may use this callback function to load a + pretrained checkpoint, saved under a directory other than the model_dir. + """ + self.task.initialize(self.model) + + @property + def checkpoint(self): + """Accesses the training checkpoint.""" + return self._checkpoint + + # TODO(yejiayu): Remove this once all deps are fixed. + def add_recovery(self, params: TrainerConfig, + checkpoint_manager: tf.train.CheckpointManager): + if params.recovery_max_trials >= 0: + self._recovery = Recovery( + loss_upper_bound=params.loss_upper_bound, + recovery_begin_steps=params.recovery_begin_steps, + recovery_max_trials=params.recovery_max_trials, + checkpoint_manager=checkpoint_manager) + + def train_loop_end(self): + """See base class.""" + self.join() + logs = {} + for metric in self.train_metrics + [self.train_loss]: + logs[metric.name] = metric.result() + metric.reset_states() + if callable(self.optimizer.learning_rate): + # Maybe a self-implemented optimizer does not have `optimizer.iterations`. + # So just to be safe here. + if hasattr(self.optimizer, "iterations"): + logs["learning_rate"] = self.optimizer.learning_rate( + self.optimizer.iterations) + else: + logs["learning_rate"] = self.optimizer.learning_rate(self.global_step) + else: + logs["learning_rate"] = self.optimizer.learning_rate + return logs + + def train_step(self, iterator): + """See base class.""" + + def step_fn(inputs): + if self.config.runtime.enable_xla and (self.config.runtime.num_gpus > 0): + task_train_step = tf.function(self.task.train_step, jit_compile=True) + else: + task_train_step = self.task.train_step + logs = task_train_step( + inputs, + model=self.model, + optimizer=self.optimizer, + metrics=self.train_metrics) + self._train_loss.update_state(logs[self.task.loss]) + self.global_step.assign_add(1) + + self.strategy.run( + step_fn, args=(next(iterator),), options=self._runtime_options) + + def eval_begin(self): + """Sets up metrics.""" + for metric in self.validation_metrics + [self.validation_loss]: + metric.reset_states() + # Swaps weights to test on weights moving average. + if self.optimizer and isinstance(self.optimizer, + optimization.ExponentialMovingAverage): + self.optimizer.swap_weights() + + def eval_step(self, iterator): + """See base class.""" + + def step_fn(inputs): + logs = self.task.validation_step( + inputs, model=self.model, metrics=self.validation_metrics) + if self.task.loss in logs: + self._validation_loss.update_state(logs[self.task.loss]) + return logs + + distributed_outputs = self.strategy.run(step_fn, args=(next(iterator),)) + return tf.nest.map_structure(self.strategy.experimental_local_results, + distributed_outputs) + + def eval_end(self, aggregated_logs=None): + """Processes evaluation results.""" + self.join() + logs = {} + for metric in self.validation_metrics: + logs[metric.name] = metric.result() + if self.validation_loss.count.numpy() != 0: + logs[self.validation_loss.name] = self.validation_loss.result() + else: + # `self.validation_loss` metric was not updated, because the validation + # loss was not returned from the task's `validation_step` method. + logging.info("The task did not report validation loss.") + if aggregated_logs: + metrics = self.task.reduce_aggregated_logs( + aggregated_logs, global_step=self.global_step) + logs.update(metrics) + + if self._checkpoint_exporter: + self._checkpoint_exporter.maybe_export_checkpoint( + self.checkpoint, logs, self.global_step.numpy()) + metric_name = self.config.trainer.best_checkpoint_eval_metric + logs["best_" + + metric_name] = self._checkpoint_exporter.best_ckpt_logs[metric_name] + + # Swaps back weights after testing when EMA is used. + # This happens after best checkpoint export so that average weights used for + # eval are exported instead of regular weights. + if self.optimizer and isinstance(self.optimizer, + optimization.ExponentialMovingAverage): + self.optimizer.swap_weights() + return logs + + def eval_reduce(self, state=None, step_outputs=None): + return self.task.aggregate_logs(state, step_outputs) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/base_trainer_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/base_trainer_test.py new file mode 100644 index 0000000000000000000000000000000000000000..e50a5bcb7c2889e2aceda3b82c8916b65718eb05 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/base_trainer_test.py @@ -0,0 +1,385 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for tensorflow_models.core.trainers.trainer.""" +# pylint: disable=g-direct-tensorflow-import +import gc +import multiprocessing +import os +import sys + +from absl.testing import parameterized +import orbit +import portpicker +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.core import base_trainer as trainer_lib +from official.core import config_definitions as cfg +from official.core import train_lib +from official.utils.testing import mock_task + +TPU_TEST = 'test_tpu' in sys.argv[0] +GPU_TEST = 'test_gpu' in sys.argv[0] + + +def all_strategy_combinations(): + return combinations.combine( + distribution=[ + strategy_combinations.default_strategy, + strategy_combinations.cloud_tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ],) + + +def create_in_process_cluster(num_workers, num_ps): + """Creates and starts local servers and returns the cluster_resolver.""" + worker_ports = [portpicker.pick_unused_port() for _ in range(num_workers)] + ps_ports = [portpicker.pick_unused_port() for _ in range(num_ps)] + + cluster_dict = {} + cluster_dict['worker'] = ['localhost:%s' % port for port in worker_ports] + if num_ps > 0: + cluster_dict['ps'] = ['localhost:%s' % port for port in ps_ports] + + cluster_spec = tf.train.ClusterSpec(cluster_dict) + + # Workers need some inter_ops threads to work properly. + worker_config = tf.compat.v1.ConfigProto() + if multiprocessing.cpu_count() < num_workers + 1: + worker_config.inter_op_parallelism_threads = num_workers + 1 + + for i in range(num_workers): + tf.distribute.Server( + cluster_spec, + job_name='worker', + task_index=i, + config=worker_config, + protocol='grpc') + + for i in range(num_ps): + tf.distribute.Server( + cluster_spec, job_name='ps', task_index=i, protocol='grpc') + + cluster_resolver = tf.distribute.cluster_resolver.SimpleClusterResolver( + cluster_spec, rpc_layer='grpc') + return cluster_resolver + + +def dataset_fn(input_context=None): + del input_context + + def dummy_data(_): + return tf.zeros((1, 1), dtype=tf.float32) + + dataset = tf.data.Dataset.range(1) + dataset = dataset.repeat() + dataset = dataset.map( + dummy_data, num_parallel_calls=tf.data.experimental.AUTOTUNE) + return dataset + + +class MockAsyncTrainer(trainer_lib._AsyncTrainer): + """Mock AsyncTrainer to test the _AsyncTrainer class.""" + + def __init__(self): + self._strategy = tf.distribute.get_strategy() + self.init_async() + + self.global_step = tf.Variable( + 0, + dtype=tf.int64, + name='global_step', + trainable=False, + aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA) + self.eval_global_step = tf.Variable( + 0, + dtype=tf.int64, + name='eval_global_step', + trainable=False, + aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA) + + train_dataset = self.distribute_dataset(dataset_fn) + orbit.StandardTrainer.__init__( + self, train_dataset, options=orbit.StandardTrainerOptions()) + + validation_dataset = self.distribute_dataset(dataset_fn) + orbit.StandardEvaluator.__init__( + self, + validation_dataset, + options=orbit.StandardEvaluatorOptions(use_tf_while_loop=True)) + + def train_loop_begin(self): + self.global_step.assign(0) + + def train_step(self, iterator): + + def replica_step(_): + self.global_step.assign_add(1) + + self._strategy.run(replica_step, args=(next(iterator),)) + + def train_loop_end(self): + self.join() + return self.global_step.numpy() + + def eval_begin(self): + self.eval_global_step.assign(0) + + def eval_step(self, iterator): + + def replica_step(_): + self.eval_global_step.assign_add(1) + + self._strategy.run(replica_step, args=(next(iterator),)) + + def eval_end(self): + self.join() + return self.eval_global_step.numpy() + + +class RecoveryTest(tf.test.TestCase): + + def test_recovery_module(self): + ckpt = tf.train.Checkpoint(v=tf.Variable(1, dtype=tf.int32)) + model_dir = self.get_temp_dir() + manager = tf.train.CheckpointManager(ckpt, model_dir, max_to_keep=1) + recovery_module = trainer_lib.Recovery( + loss_upper_bound=1.0, + checkpoint_manager=manager, + recovery_begin_steps=1, + recovery_max_trials=1) + self.assertFalse(recovery_module.should_recover(1.1, 0)) + self.assertFalse(recovery_module.should_recover(0.1, 1)) + self.assertTrue(recovery_module.should_recover(1.1, 2)) + + # First triggers the recovery once. + recovery_module.maybe_recover(1.1, 10) + + # Second time, it raises. + with self.assertRaisesRegex( + RuntimeError, 'The loss value is NaN .*'): + recovery_module.maybe_recover(1.1, 10) + + +class TrainerTest(tf.test.TestCase, parameterized.TestCase): + + def setUp(self): + super().setUp() + self._config = cfg.ExperimentConfig( + trainer=cfg.TrainerConfig( + optimizer_config=cfg.OptimizationConfig({ + 'optimizer': { + 'type': 'sgd' + }, + 'learning_rate': { + 'type': 'constant' + } + }))) + + def tearDown(self): + gc.collect() + # This will only contain uncollectable garbage, i.e. reference cycles + # involving objects with __del__ defined. + self.assertEmpty(gc.garbage) + super().tearDown() + + def create_test_trainer(self, config, model_dir=None, task=None): + task = task or mock_task.MockTask(config.task, logging_dir=model_dir) + ckpt_exporter = train_lib.maybe_create_best_ckpt_exporter(config, model_dir) + trainer = trainer_lib.Trainer( + config, + task, + model=task.build_model(), + optimizer=task.create_optimizer(config.trainer.optimizer_config, + config.runtime), + checkpoint_exporter=ckpt_exporter) + return trainer + + @combinations.generate(all_strategy_combinations()) + def test_trainer_train(self, distribution): + with distribution.scope(): + trainer = self.create_test_trainer(self._config) + logs = trainer.train(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertIn('training_loss', logs) + self.assertIn('learning_rate', logs) + + @combinations.generate(all_strategy_combinations()) + def test_trainer_passing_datasets(self, distribution): + with distribution.scope(): + task = mock_task.MockTask(self._config) + train_dataset = orbit.utils.make_distributed_dataset( + distribution, task.build_inputs, self._config.task.train_data) + validation_dataset = orbit.utils.make_distributed_dataset( + distribution, task.build_inputs, self._config.task.validation_data) + self._config.task.train_data = None + self._config.task.validation_data = None + trainer = trainer_lib.Trainer( + self._config, + task, + model=task.build_model(), + optimizer=task.create_optimizer(self._config.trainer.optimizer_config, + self._config.runtime), + train_dataset=train_dataset, + validation_dataset=validation_dataset) + logs = trainer.train(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertIn('training_loss', logs) + self.assertIn('learning_rate', logs) + logs = trainer.evaluate(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertIn('validation_loss', logs) + + def test_base_async_trainer(self): + if TPU_TEST or GPU_TEST: + self.skipTest('Aysnc training is not available on GPU/GPU.') + num_workers = 3 + num_ps = 2 + cluster_resolver = create_in_process_cluster(num_workers, num_ps) + distribution = tf.distribute.experimental.ParameterServerStrategy( + cluster_resolver) + with distribution.scope(): + trainer = MockAsyncTrainer() + trainer.init_async() + self.assertIsInstance( + trainer._coordinator, + tf.distribute.experimental.coordinator.ClusterCoordinator) + self.assertEqual(trainer.train(tf.constant(10)), 10) + self.assertEqual(trainer.evaluate(tf.constant(11)), 11) + + def test_async_trainer_train(self): + if TPU_TEST or GPU_TEST: + self.skipTest('Aysnc training is not available on GPU/TPU.') + num_workers = 3 + num_ps = 2 + cluster_resolver = create_in_process_cluster(num_workers, num_ps) + distribution = tf.distribute.experimental.ParameterServerStrategy( + cluster_resolver) + with distribution.scope(): + config = cfg.ExperimentConfig(**self._config.as_dict()) + config.trainer.eval_tf_while_loop = True + trainer = self.create_test_trainer(config) + logs = trainer.train(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertIn('training_loss', logs) + self.assertIn('learning_rate', logs) + + def test_async_trainer_validate(self): + if TPU_TEST or GPU_TEST: + self.skipTest('Aysnc training is not available on GPU/GPU.') + num_workers = 3 + num_ps = 2 + cluster_resolver = create_in_process_cluster(num_workers, num_ps) + distribution = tf.distribute.experimental.ParameterServerStrategy( + cluster_resolver) + with distribution.scope(): + config = cfg.ExperimentConfig(**self._config.as_dict()) + config.trainer.eval_tf_while_loop = True + trainer = self.create_test_trainer(config) + logs = trainer.evaluate(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertIn('acc', logs) + self.assertIn('validation_loss', logs) + + @combinations.generate(all_strategy_combinations()) + def test_trainer_validate(self, distribution): + with distribution.scope(): + trainer = self.create_test_trainer(self._config) + logs = trainer.evaluate(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertEqual(logs['counter'], 5. * distribution.num_replicas_in_sync) + self.assertIn('validation_loss', logs) + + @combinations.generate(all_strategy_combinations()) + def test_trainer_validate_without_loss(self, distribution): + + class MockTaskWithoutValidationLoss(mock_task.MockTask): + + def validation_step(self, inputs, model, metrics=None): + # Disable validation loss. + logs = super().validation_step(inputs, model) + del logs[self.loss] + return logs + + with distribution.scope(): + task = MockTaskWithoutValidationLoss() + trainer = self.create_test_trainer(self._config, task=task) + logs = trainer.evaluate(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertEqual(logs['counter'], 5. * distribution.num_replicas_in_sync) + self.assertNotIn('validation_loss', logs) + + @combinations.generate( + combinations.combine( + mixed_precision_dtype=['float32', 'bfloat16', 'float16'], + loss_scale=[None, 'dynamic', 128, 256], + )) + def test_configure_optimizer(self, mixed_precision_dtype, loss_scale): + config = cfg.ExperimentConfig( + runtime=cfg.RuntimeConfig( + mixed_precision_dtype=mixed_precision_dtype, loss_scale=loss_scale), + trainer=cfg.TrainerConfig( + optimizer_config=cfg.OptimizationConfig({ + 'optimizer': { + 'type': 'sgd' + }, + 'learning_rate': { + 'type': 'constant' + }, + }))) + trainer = self.create_test_trainer(config) + if mixed_precision_dtype == 'float16': + self.assertIsInstance(trainer.optimizer, + tf.keras.mixed_precision.LossScaleOptimizer) + if loss_scale in (None, 'dynamic'): + self.assertTrue(trainer.optimizer.dynamic) + else: + self.assertFalse(trainer.optimizer.dynamic) + self.assertEqual(trainer.optimizer.initial_scale, loss_scale) + else: + self.assertIsInstance(trainer.optimizer, tf.keras.optimizers.SGD) + + metrics = trainer.train(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertIn('training_loss', metrics) + + def test_export_best_ckpt(self): + config = cfg.ExperimentConfig( + trainer=cfg.TrainerConfig( + best_checkpoint_export_subdir='best_ckpt', + best_checkpoint_eval_metric='acc', + optimizer_config=cfg.OptimizationConfig({ + 'optimizer': { + 'type': 'sgd' + }, + 'learning_rate': { + 'type': 'constant' + } + }))) + model_dir = self.get_temp_dir() + trainer = self.create_test_trainer(config, model_dir=model_dir) + trainer.train(tf.convert_to_tensor(1, dtype=tf.int32)) + trainer.evaluate(tf.convert_to_tensor(1, dtype=tf.int32)) + self.assertTrue( + tf.io.gfile.exists(os.path.join(model_dir, 'best_ckpt', 'info.json'))) + + def test_model_with_compiled_loss(self): + task = mock_task.MockTask() + model = task.build_model() + model.compile(loss=tf.keras.losses.CategoricalCrossentropy()) + trainer = trainer_lib.Trainer( + self._config, + task, + model=model, + optimizer=task.create_optimizer(self._config.trainer.optimizer_config)) + logs = trainer.train(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertIn('training_loss', logs) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/config_definitions.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/config_definitions.py new file mode 100644 index 0000000000000000000000000000000000000000..3bca789b5221d7e51ed112cfa753613febae11c7 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/config_definitions.py @@ -0,0 +1,255 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common configuration settings.""" + +import dataclasses +from typing import Optional, Sequence, Union + +from official.modeling.hyperparams import base_config +from official.modeling.optimization.configs import optimization_config + +OptimizationConfig = optimization_config.OptimizationConfig + + +@dataclasses.dataclass +class DataConfig(base_config.Config): + """The base configuration for building datasets. + + Attributes: + input_path: The path to the input. It can be either (1) a str indicating a + file path/pattern, or (2) a str indicating multiple file paths/patterns + separated by comma (e.g "a, b, c" or no spaces "a,b,c"), or (3) a list of + str, each of which is a file path/pattern or multiple file paths/patterns + separated by comma, or (4) a dictionary of the previous three approaches + for more advanced data mixing using named access. It should not be + specified when the following `tfds_name` is specified. + tfds_name: The name of the tensorflow dataset (TFDS). It should not be + specified when the above `input_path` is specified. + tfds_split: A str indicating which split of the data to load from TFDS. It + is required when above `tfds_name` is specified. + global_batch_size: The global batch size across all replicas. + is_training: Whether this data is used for training or not. This flag is + useful for consumers of this object to determine whether the data should + be repeated or shuffled. + drop_remainder: Whether the last batch should be dropped in the case it has + fewer than `global_batch_size` elements. + shuffle_buffer_size: The buffer size used for shuffling training data. + cache: Whether to cache dataset examples. If `True`, we will cache the + dataset after applying the decode_fn and parse_fn. It can be used to avoid + re-reading from disk, re-decoding and re-parsing the example on the second + epoch, but it requires significant memory overhead. + cycle_length: The number of files that will be processed concurrently when + interleaving files. + block_length: The number of consecutive elements to produce from each input + element before cycling to another input element when interleaving files. + deterministic: A boolean controlling whether determinism should be enforced. + sharding: Whether sharding is used in the input pipeline. + enable_tf_data_service: A boolean indicating whether to enable tf.data + service for the input pipeline. + tf_data_service_address: The URI of a tf.data service to offload + preprocessing onto during training. The URI should be in the format + "protocol://address", e.g. "grpc://tf-data-service:5050". It can be + overridden by `FLAGS.tf_data_service` flag in the binary. + tf_data_service_job_name: The name of the tf.data service job. This argument + makes it possible for multiple datasets to share the same job. The default + behavior is that the dataset creates anonymous, exclusively owned jobs. + tfds_data_dir: A str specifying the directory to read/write TFDS data. + tfds_as_supervised: A bool. When loading dataset from TFDS, if True, the + returned tf.data.Dataset will have a 2-tuple structure (input, label) + according to builder.info.supervised_keys; if False, the default, the + returned tf.data.Dataset will have a dictionary with all the features. + tfds_skip_decoding_feature: A str to indicate which features are skipped for + decoding when loading dataset from TFDS. Use comma to separate multiple + features. The main use case is to skip the image/video decoding for better + performance. + seed: An optional seed to use for deterministic shuffling/preprocessing. + """ + input_path: Union[Sequence[str], str, base_config.Config] = "" + tfds_name: str = "" + tfds_split: str = "" + global_batch_size: int = 0 + is_training: bool = None + drop_remainder: bool = True + shuffle_buffer_size: int = 100 + cache: bool = False + cycle_length: Optional[int] = None + block_length: int = 1 + deterministic: Optional[bool] = None + sharding: bool = True + enable_tf_data_service: bool = False + tf_data_service_address: Optional[str] = None + tf_data_service_job_name: Optional[str] = None + tfds_data_dir: str = "" + tfds_as_supervised: bool = False + tfds_skip_decoding_feature: str = "" + seed: Optional[int] = None + + +@dataclasses.dataclass +class RuntimeConfig(base_config.Config): + """High-level configurations for Runtime. + + These include parameters that are not directly related to the experiment, + e.g. directories, accelerator type, etc. + + Attributes: + distribution_strategy: e.g. 'mirrored', 'tpu', etc. + enable_xla: Whether or not to enable XLA. + per_gpu_thread_count: thread count per GPU. + gpu_thread_mode: Whether and how the GPU device uses its own threadpool. + dataset_num_private_threads: Number of threads for a private threadpool + created for all datasets computation. + tpu: The address of the TPU to use, if any. + num_gpus: The number of GPUs to use, if any. + worker_hosts: comma-separated list of worker ip:port pairs for running + multi-worker models with DistributionStrategy. + task_index: If multi-worker training, the task index of this worker. + all_reduce_alg: Defines the algorithm for performing all-reduce. + num_packs: Sets `num_packs` in the cross device ops used in + MirroredStrategy. For details, see tf.distribute.NcclAllReduce. + mixed_precision_dtype: dtype of mixed precision policy. It can be 'float32', + 'float16', or 'bfloat16'. + loss_scale: The type of loss scale, or 'float' value. This is used when + setting the mixed precision policy. + run_eagerly: Whether or not to run the experiment eagerly. + batchnorm_spatial_persistent: Whether or not to enable the spatial + persistent mode for CuDNN batch norm kernel for improved GPU performance. + """ + distribution_strategy: str = "mirrored" + enable_xla: bool = False + gpu_thread_mode: Optional[str] = None + dataset_num_private_threads: Optional[int] = None + per_gpu_thread_count: int = 0 + tpu: Optional[str] = None + num_gpus: int = 0 + worker_hosts: Optional[str] = None + task_index: int = -1 + all_reduce_alg: Optional[str] = None + num_packs: int = 1 + mixed_precision_dtype: Optional[str] = None + loss_scale: Optional[Union[str, float]] = None + run_eagerly: bool = False + batchnorm_spatial_persistent: bool = False + + # XLA runtime params. + # XLA params are only applied to the train_step. + # These augments can improve training speed. They can also improve eval, but + # may reduce usability and users would need to make changes to code. + + # Whether to enable XLA dynamic padder + # infrastructure to handle dynamic shapes inputs inside XLA. True by + # default. Disabling this may cause correctness issues with dynamic shapes + # inputs, as XLA will just assume the inputs are with padded shapes. However + # users can optionally set it to False to improve device time if masking is + # already handled in the user side. + # If None, will respect XLA default. + tpu_enable_xla_dynamic_padder: Optional[bool] = None + + # Global model parallelism configurations. + num_cores_per_replica: int = 1 + default_shard_dim: int = -1 + + def model_parallelism(self): + return dict( + num_cores_per_replica=self.num_cores_per_replica, + default_shard_dim=self.default_shard_dim) + + +@dataclasses.dataclass +class TrainerConfig(base_config.Config): + """Configuration for trainer. + + Attributes: + optimizer_config: optimizer config, it includes optimizer, learning rate, + and warmup schedule configs. + train_tf_while_loop: whether or not to use tf while loop. + train_tf_function: whether or not to use tf_function for training loop. + eval_tf_function: whether or not to use tf_function for eval. + allow_tpu_summary: Whether to allow summary happen inside the XLA program + runs on TPU through automatic outside compilation. + steps_per_loop: number of steps per loop to report training metrics. This + can also be used to reduce host worker communication in a TPU setup. + summary_interval: number of steps between each summary. + checkpoint_interval: number of steps between checkpoints. + max_to_keep: max checkpoints to keep. + continuous_eval_timeout: maximum number of seconds to wait between + checkpoints, if set to None, continuous eval will wait indefinitely. This + is only used continuous_train_and_eval and continuous_eval modes. Default + value is 1 hrs. + train_steps: number of train steps. + validation_steps: number of eval steps. If `None`, the entire eval dataset + is used. + validation_interval: number of training steps to run between evaluations. + best_checkpoint_export_subdir: if set, the trainer will keep track of the + best evaluation metric, and export the corresponding best checkpoint under + `model_dir/best_checkpoint_export_subdir`. Note that this only works if + mode contains eval (such as `train_and_eval`, `continuous_eval`, and + `continuous_train_and_eval`). + best_checkpoint_eval_metric: for exporting the best checkpoint, which + evaluation metric the trainer should monitor. This can be any evaluation + metric appears on tensorboard. + best_checkpoint_metric_comp: for exporting the best checkpoint, how the + trainer should compare the evaluation metrics. This can be either `higher` + (higher the better) or `lower` (lower the better). + validation_summary_subdir: A 'str', sub directory for saving eval summary. + """ + optimizer_config: OptimizationConfig = OptimizationConfig() + # Orbit settings. + train_tf_while_loop: bool = True + train_tf_function: bool = True + eval_tf_function: bool = True + eval_tf_while_loop: bool = False + allow_tpu_summary: bool = False + # Trainer intervals. + steps_per_loop: int = 1000 + summary_interval: int = 1000 + checkpoint_interval: int = 1000 + # Checkpoint manager. + max_to_keep: int = 5 + continuous_eval_timeout: int = 60 * 60 + # Train/Eval routines. + train_steps: int = 0 + # Sets validation steps to be -1 to evaluate the entire dataset. + validation_steps: int = -1 + validation_interval: int = 1000 + # Best checkpoint export. + best_checkpoint_export_subdir: str = "" + best_checkpoint_eval_metric: str = "" + best_checkpoint_metric_comp: str = "higher" + # Blowup recovery. + loss_upper_bound: float = 1e6 + recovery_begin_steps: int = 0 # Enforcing the loss bound after these steps. + # When max trials < 0, no recovery module; max trials = 0, we will check + # the condition and fail the job if the condition happens; max trials > 0, + # we will retore the model states. + recovery_max_trials: int = 0 + validation_summary_subdir: str = "validation" + + +@dataclasses.dataclass +class TaskConfig(base_config.Config): + init_checkpoint: str = "" + model: Optional[base_config.Config] = None + train_data: DataConfig = DataConfig() + validation_data: DataConfig = DataConfig() + name: Optional[str] = None + + +@dataclasses.dataclass +class ExperimentConfig(base_config.Config): + """Top-level configuration.""" + task: TaskConfig = TaskConfig() + trainer: TrainerConfig = TrainerConfig() + runtime: RuntimeConfig = RuntimeConfig() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/exp_factory.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/exp_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..b10d49acbdaeee211194ed0c018cb91493d74d21 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/exp_factory.py @@ -0,0 +1,32 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Experiment factory methods.""" + +from official.core import config_definitions as cfg +from official.core import registry + + +_REGISTERED_CONFIGS = {} + + +def register_config_factory(name): + """Register ExperimentConfig factory method.""" + return registry.register(_REGISTERED_CONFIGS, name) + + +def get_exp_config(exp_name: str) -> cfg.ExperimentConfig: + """Looks up the `ExperimentConfig` according to the `exp_name`.""" + exp_creater = registry.lookup(_REGISTERED_CONFIGS, exp_name) + return exp_creater() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/export_base.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/export_base.py new file mode 100644 index 0000000000000000000000000000000000000000..ebfefd0e1b92c505a628495bbe623b568aab985f --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/export_base.py @@ -0,0 +1,133 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Base class for model export.""" + +import abc +import functools +from typing import Any, Callable, Dict, Mapping, List, Optional, Text, Union + +import tensorflow as tf +from tensorflow.python.saved_model.model_utils import export_utils + + +class ExportModule(tf.Module, metaclass=abc.ABCMeta): + """Base Export Module.""" + + def __init__(self, + params, + model: Union[tf.Module, tf.keras.Model], + inference_step: Optional[Callable[..., Any]] = None, + *, + preprocessor: Optional[Callable[..., Any]] = None, + postprocessor: Optional[Callable[..., Any]] = None): + """Instantiates an ExportModel. + + Examples: + + `inference_step` must be a function that has `model` as an kwarg or the + second positional argument. + ``` + def _inference_step(inputs, model=None): + return model(inputs, training=False) + + module = ExportModule(params, model, inference_step=_inference_step) + ``` + + `preprocessor` and `postprocessor` could be either functions or `tf.Module`. + The usages of preprocessor and postprocessor are managed by the + implementation of `serve()` method. + + Args: + params: A dataclass for parameters to the module. + model: A model instance which contains weights and forward computation. + inference_step: An optional callable to forward-pass the model. If not + specified, it creates a parital function with `model` as an required + kwarg. + preprocessor: An optional callable to preprocess the inputs. + postprocessor: An optional callable to postprocess the model outputs. + """ + super().__init__(name=None) + self.model = model + self.params = params + + if inference_step is not None: + self.inference_step = functools.partial(inference_step, model=self.model) + else: + self.inference_step = functools.partial( + self.model.__call__, training=False) + self.preprocessor = preprocessor + self.postprocessor = postprocessor + + @abc.abstractmethod + def serve(self) -> Mapping[Text, tf.Tensor]: + """The bare inference function which should run on all devices. + + Expecting tensors are passed in through keyword arguments. Returns a + dictionary of tensors, when the keys will be used inside the SignatureDef. + """ + + @abc.abstractmethod + def get_inference_signatures( + self, function_keys: Dict[Text, Text]) -> Mapping[Text, Any]: + """Get defined function signatures.""" + + +def export(export_module: ExportModule, + function_keys: Union[List[Text], Dict[Text, Text]], + export_savedmodel_dir: Text, + checkpoint_path: Optional[Text] = None, + timestamped: bool = True, + save_options: Optional[tf.saved_model.SaveOptions] = None) -> Text: + """Exports to SavedModel format. + + Args: + export_module: a ExportModule with the keras Model and serving tf.functions. + function_keys: a list of string keys to retrieve pre-defined serving + signatures. The signaute keys will be set with defaults. If a dictionary + is provided, the values will be used as signature keys. + export_savedmodel_dir: Output saved model directory. + checkpoint_path: Object-based checkpoint path or directory. + timestamped: Whether to export the savedmodel to a timestamped directory. + save_options: `SaveOptions` for `tf.saved_model.save`. + + Returns: + The savedmodel directory path. + """ + ckpt_dir_or_file = checkpoint_path + if ckpt_dir_or_file is not None and tf.io.gfile.isdir(ckpt_dir_or_file): + ckpt_dir_or_file = tf.train.latest_checkpoint(ckpt_dir_or_file) + if ckpt_dir_or_file: + checkpoint = tf.train.Checkpoint(model=export_module.model) + checkpoint.read( + ckpt_dir_or_file).assert_existing_objects_matched().expect_partial() + if isinstance(function_keys, list): + if len(function_keys) == 1: + function_keys = { + function_keys[0]: tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY + } + else: + raise ValueError( + "If the function_keys is a list, it must contain a single element. %s" + % function_keys) + + signatures = export_module.get_inference_signatures(function_keys) + if timestamped: + export_dir = export_utils.get_timestamped_export_dir( + export_savedmodel_dir).decode("utf-8") + else: + export_dir = export_savedmodel_dir + tf.saved_model.save( + export_module, export_dir, signatures=signatures, options=save_options) + return export_dir diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/export_base_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/export_base_test.py new file mode 100644 index 0000000000000000000000000000000000000000..8071ec66392a6267f033db9a98dc2e7d0a24a7be --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/export_base_test.py @@ -0,0 +1,126 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for official.core.export_base.""" +import os +from typing import Any, Dict, Mapping, Text + +import tensorflow as tf + +from official.core import export_base + + +class TestModule(export_base.ExportModule): + + @tf.function + def serve(self, inputs: tf.Tensor) -> Mapping[Text, tf.Tensor]: + x = inputs if self.preprocessor is None else self.preprocessor( + inputs=inputs) + x = self.inference_step(x) + x = self.postprocessor(x) if self.postprocessor else x + return {'outputs': x} + + def get_inference_signatures( + self, function_keys: Dict[Text, Text]) -> Mapping[Text, Any]: + input_signature = tf.TensorSpec(shape=[None, None], dtype=tf.float32) + return {'foo': self.serve.get_concrete_function(input_signature)} + + +class ExportBaseTest(tf.test.TestCase): + + def test_export_module(self): + tmp_dir = self.get_temp_dir() + model = tf.keras.layers.Dense(2) + inputs = tf.ones([2, 4], tf.float32) + expected_output = model(inputs, training=False) + module = TestModule(params=None, model=model) + ckpt_path = tf.train.Checkpoint(model=model).save( + os.path.join(tmp_dir, 'ckpt')) + export_dir = export_base.export( + module, ['foo'], + export_savedmodel_dir=tmp_dir, + checkpoint_path=ckpt_path, + timestamped=True) + self.assertTrue(os.path.exists(os.path.join(export_dir, 'saved_model.pb'))) + self.assertTrue( + os.path.exists( + os.path.join(export_dir, 'variables', 'variables.index'))) + self.assertTrue( + os.path.exists( + os.path.join(export_dir, 'variables', + 'variables.data-00000-of-00001'))) + + imported = tf.saved_model.load(export_dir) + output = imported.signatures['foo'](inputs) + self.assertAllClose(output['outputs'].numpy(), expected_output.numpy()) + + def test_custom_inference_step(self): + tmp_dir = self.get_temp_dir() + model = tf.keras.layers.Dense(2) + inputs = tf.ones([2, 4], tf.float32) + + def _inference_step(inputs, model): + return tf.nn.softmax(model(inputs, training=False)) + + module = TestModule( + params=None, model=model, inference_step=_inference_step) + expected_output = _inference_step(inputs, model) + ckpt_path = tf.train.Checkpoint(model=model).save( + os.path.join(tmp_dir, 'ckpt')) + export_dir = export_base.export( + module, ['foo'], + export_savedmodel_dir=tmp_dir, + checkpoint_path=ckpt_path, + timestamped=False) + imported = tf.saved_model.load(export_dir) + output = imported.signatures['foo'](inputs) + self.assertAllClose(output['outputs'].numpy(), expected_output.numpy()) + + def test_processors(self): + model = tf.Module() + inputs = tf.zeros((), tf.float32) + + def _inference_step(inputs, model): + del model + return inputs + 1.0 + + def _preprocessor(inputs): + print(inputs) + return inputs + 0.1 + + module = TestModule( + params=None, + model=model, + inference_step=_inference_step, + preprocessor=_preprocessor) + output = module.serve(inputs) + self.assertAllClose(output['outputs'].numpy(), 1.1) + + class _PostProcessor(tf.Module): + + def __call__(self, inputs): + return inputs + 0.01 + + module = TestModule( + params=None, + model=model, + inference_step=_inference_step, + preprocessor=_preprocessor, + postprocessor=_PostProcessor()) + output = module.serve(inputs) + self.assertAllClose(output['outputs'].numpy(), 1.11) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/input_reader.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/input_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..736172b6a25723d6419cbfc267f567b242483a1e --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/input_reader.py @@ -0,0 +1,478 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A common dataset reader.""" +import random +from typing import Any, Callable, Dict, List, Optional, Sequence, Text, Union + +from absl import logging +import tensorflow as tf +import tensorflow_datasets as tfds + +from official.core import config_definitions as cfg + + +def _get_random_integer(): + return random.randint(0, (1 << 31) - 1) + + +def _maybe_map_fn(dataset: tf.data.Dataset, + fn: Optional[Callable[..., Any]] = None) -> tf.data.Dataset: + """Calls dataset.map if a valid function is passed in.""" + return dataset if fn is None else dataset.map( + fn, num_parallel_calls=tf.data.experimental.AUTOTUNE) + + +def match_files(input_path: Union[Sequence[str], str]) -> List[str]: + """Matches files from an input_path.""" + matched_files = [] + # Read dataset from files. + usage = ('`input_path` should be either (1) a str indicating a file ' + 'path/pattern, or (2) a str indicating multiple file ' + 'paths/patterns separated by comma (e.g "a, b, c" or no spaces ' + '"a,b,c", or (3) a list of str, each of which is a file ' + 'path/pattern or multiple file paths/patterns separated by ' + 'comma, but got: %s') + if isinstance(input_path, str): + input_path_list = [input_path] + elif isinstance(input_path, (list, tuple)): + if any(not isinstance(x, str) for x in input_path): + raise ValueError(usage % input_path) + input_path_list = input_path + else: + raise ValueError(usage % input_path) + + for input_path in input_path_list: + input_patterns = input_path.strip().split(',') + for input_pattern in input_patterns: + input_pattern = input_pattern.strip() + if not input_pattern: + continue + if '*' in input_pattern or '?' in input_pattern: + tmp_matched_files = tf.io.gfile.glob(input_pattern) + if not tmp_matched_files: + raise ValueError('%s does not match any files.' % input_pattern) + matched_files.extend(tmp_matched_files) + else: + matched_files.append(input_pattern) + + if not matched_files: + raise ValueError('%s does not match any files.' % input_path) + + return matched_files + + +def _read_files_then_shard(matched_files: List[str], + dataset_fn, + input_context: Optional[ + tf.distribute.InputContext] = None, + sharding: bool = False, + repeat: bool = False) -> tf.data.Dataset: + """Sends all data files to every worker and then shard by data.""" + dataset = dataset_fn(matched_files) + + # When `input_file` is a path to a single file or the number of files is + # less than the number of input pipelines, disable auto sharding + # so that same input file is sent to all workers. + options = tf.data.Options() + options.experimental_distribute.auto_shard_policy = ( + tf.data.experimental.AutoShardPolicy.OFF) + dataset = dataset.with_options(options) + # Do not enable sharding if tf.data service is enabled, as sharding will be + # handled inside tf.data service. + if sharding and input_context and (input_context.num_input_pipelines > 1): + dataset = dataset.shard(input_context.num_input_pipelines, + input_context.input_pipeline_id) + + if repeat: + dataset = dataset.repeat() + return dataset + + +def _shard_files_then_read(matched_files: List[str], + dataset_fn, + input_context: Optional[ + tf.distribute.InputContext] = None, + seed: Optional[Union[int, tf.Tensor]] = None, + is_training: bool = False, + sharding: bool = False, + cache: bool = False, + cycle_length: Optional[int] = None, + block_length: Optional[int] = None, + deterministic: bool = False) -> tf.data.Dataset: + """Shards the data files and then sent a split to every worker to read.""" + dataset = tf.data.Dataset.from_tensor_slices(matched_files) + + # Shuffle and repeat at file level. + # If cache is enabled, `reshuffle_each_iteration` is set to False, + # because we will read the same cached data in every iteration anyway. + if is_training: + # We need a seed to shuffle the files so that when each TPU workers gets + # its own shard the files do not overlap. + if sharding and seed is None: + seed = _get_random_integer() + dataset = dataset.shuffle( + len(matched_files), + seed=seed, + reshuffle_each_iteration=True if not cache else False) + + # Do not enable sharding if tf.data service is enabled, as sharding will be + # handled inside tf.data service. + if sharding and input_context and (input_context.num_input_pipelines > 1): + dataset = dataset.shard(input_context.num_input_pipelines, + input_context.input_pipeline_id) + + # If cache is enabled, we will call `repeat()` later after `cache()`. + if is_training and not cache: + dataset = dataset.repeat() + + dataset = dataset.interleave( + map_func=dataset_fn, + cycle_length=cycle_length, + block_length=block_length, + num_parallel_calls=(cycle_length + if cycle_length else tf.data.experimental.AUTOTUNE), + deterministic=deterministic) + return dataset + + +def _read_tfds(tfds_builder: tfds.core.DatasetBuilder, + tfds_split: Text, + tfds_skip_decoding_feature: Text, + tfds_as_supervised: bool, + input_context: Optional[tf.distribute.InputContext] = None, + seed: Optional[Union[int, tf.Tensor]] = None, + is_training: bool = False, + cache: bool = False, + cycle_length: Optional[int] = None, + block_length: Optional[int] = None) -> tf.data.Dataset: + """Reads a dataset from tfds.""" + # No op if exist. + tfds_builder.download_and_prepare() + + read_config = tfds.ReadConfig( + interleave_cycle_length=cycle_length, + interleave_block_length=block_length, + input_context=input_context, + shuffle_seed=seed) + decoders = {} + if tfds_skip_decoding_feature: + for skip_feature in tfds_skip_decoding_feature.split(','): + decoders[skip_feature.strip()] = tfds.decode.SkipDecoding() + dataset = tfds_builder.as_dataset( + split=tfds_split, + shuffle_files=is_training, + as_supervised=tfds_as_supervised, + decoders=decoders, + read_config=read_config) + + if is_training and not cache: + dataset = dataset.repeat() + return dataset + + +class InputReader: + """Input reader that returns a tf.data.Dataset instance.""" + + # A static random number which is the same across different InputReader + # instances. + static_randnum = _get_random_integer() + + def __init__(self, + params: cfg.DataConfig, + dataset_fn=tf.data.TFRecordDataset, + decoder_fn: Optional[Callable[..., Any]] = None, + combine_fn: Optional[Callable[..., Any]] = None, + sample_fn: Optional[Callable[..., Any]] = None, + parser_fn: Optional[Callable[..., Any]] = None, + transform_and_batch_fn: Optional[Callable[ + [tf.data.Dataset, Optional[tf.distribute.InputContext]], + tf.data.Dataset]] = None, + postprocess_fn: Optional[Callable[..., Any]] = None): + """Initializes an InputReader instance. + + Args: + params: A config_definitions.DataConfig object. + dataset_fn: A `tf.data.Dataset` that consumes the input files. For + example, it can be `tf.data.TFRecordDataset`. + decoder_fn: An optional `callable` that takes the serialized data string + and decodes them into the raw tensor dictionary. + combine_fn: An optional `callable` that takes a dictionarty of + `tf.data.Dataset` objects as input and outputs a combined dataset. It + will be executed after the decoder_fn and before the sample_fn. + sample_fn: An optional `callable` that takes a `tf.data.Dataset` object as + input and outputs the transformed dataset. It performs sampling on the + decoded raw tensors dict before the parser_fn. + parser_fn: An optional `callable` that takes the decoded raw tensors dict + and parse them into a dictionary of tensors that can be consumed by the + model. It will be executed after decoder_fn. + transform_and_batch_fn: An optional `callable` that takes a + `tf.data.Dataset` object and an optional `tf.distribute.InputContext` as + input, and returns a `tf.data.Dataset` object. It will be executed after + `parser_fn` to transform and batch the dataset; if None, after + `parser_fn` is executed, the dataset will be batched into per-replica + batch size. + postprocess_fn: A optional `callable` that processes batched tensors. It + will be executed after batching. + """ + if params.input_path and params.tfds_name: + raise ValueError('At most one of `input_path` and `tfds_name` can be ' + 'specified, but got %s and %s.' % + (params.input_path, params.tfds_name)) + + if isinstance(params.input_path, + cfg.base_config.Config) and combine_fn is None: + raise ValueError( + 'A `combine_fn` is required if the `input_path` is a dictionary.') + + self._tfds_builder = None + self._matched_files = None + if not params.input_path: + # Read dataset from TFDS. + if not params.tfds_split: + raise ValueError( + '`tfds_name` is %s, but `tfds_split` is not specified.' % + params.tfds_name) + self._tfds_builder = tfds.builder( + params.tfds_name, data_dir=params.tfds_data_dir) + else: + self._matched_files = self.get_files(params.input_path) + + self._global_batch_size = params.global_batch_size + self._is_training = params.is_training + self._drop_remainder = params.drop_remainder + self._shuffle_buffer_size = params.shuffle_buffer_size + self._cache = params.cache + self._cycle_length = params.cycle_length + self._block_length = params.block_length + self._deterministic = params.deterministic + self._sharding = params.sharding + self._tfds_split = params.tfds_split + self._tfds_as_supervised = params.tfds_as_supervised + self._tfds_skip_decoding_feature = params.tfds_skip_decoding_feature + + self._dataset_fn = dataset_fn + self._decoder_fn = decoder_fn + self._combine_fn = combine_fn + self._sample_fn = sample_fn + self._parser_fn = parser_fn + self._transform_and_batch_fn = transform_and_batch_fn + self._postprocess_fn = postprocess_fn + self._seed = params.seed + + # When tf.data service is enabled, each data service worker should get + # different random seeds. Thus, we set `seed` to None. + # Sharding should also be disabled because tf data service handles how + # each worker shard data with `processing_mode` in distribute method. + if params.enable_tf_data_service: + self._seed = None + self._sharding = False + + self._enable_tf_data_service = ( + params.enable_tf_data_service and params.tf_data_service_address) + self._tf_data_service_address = params.tf_data_service_address + if self._enable_tf_data_service: + # Add a random seed as the tf.data service job name suffix, so tf.data + # service doesn't reuse the previous state if TPU worker gets preempted. + self._tf_data_service_job_name = ( + params.tf_data_service_job_name + str(self.static_randnum)) + self._enable_round_robin_tf_data_service = params.get( + 'enable_round_robin_tf_data_service', False) + + @property + def tfds_info(self) -> tfds.core.DatasetInfo: + """Returns TFDS dataset info, if available.""" + if self._tfds_builder: + return self._tfds_builder.info + else: + raise ValueError('tfds_info is not available, because the dataset ' + 'is not loaded from tfds.') + + def get_files(self, input_path): + """Gets matched files. Can be overridden by subclasses.""" + if not input_path: + return None + # we want to combine / mix datasets + if isinstance(input_path, cfg.base_config.Config): + matched_files = {} + for k, v in input_path.as_dict().items(): + matched_files[k] = match_files(v) + # single dataset + else: + matched_files = match_files(input_path) + return matched_files + + def _read_data_source( + self, + matched_files: Union[Dict[str, List[str]], List[str]], + dataset_fn, + input_context: Optional[tf.distribute.InputContext] = None, + tfds_builder: Optional[tfds.core.DatasetBuilder] = None): + """Reads the data source (files/tfds) to a dataset.""" + + def _files_to_dataset(files: List[str]) -> tf.data.Dataset: + if len(files) > 1: + if input_context and (len(files) < input_context.num_input_pipelines): + logging.warn( + 'The number of files %d is less than the number of input pipelines ' + '%d. We will send all input files to every worker. ' + 'Please consider sharding your data into more files.', len(files), + input_context.num_input_pipelines) + return _read_files_then_shard( + files, + dataset_fn, + input_context, + sharding=self._sharding, + repeat=self._is_training and not self._cache) + else: + return _shard_files_then_read( + files, + dataset_fn, + input_context, + seed=self._seed, + is_training=self._is_training, + sharding=self._sharding, + cache=self._cache, + cycle_length=self._cycle_length, + block_length=self._block_length, + deterministic=self._deterministic) + elif len(files) == 1: + return _read_files_then_shard( + files, + dataset_fn, + input_context, + sharding=self._sharding, + repeat=self._is_training and not self._cache) + else: + raise ValueError('It is unexpected that `tfds_builder` is None and ' + 'there is also no `files`.') + + if tfds_builder: + dataset = _read_tfds( + tfds_builder=self._tfds_builder, + tfds_split=self._tfds_split, + tfds_skip_decoding_feature=self._tfds_skip_decoding_feature, + tfds_as_supervised=self._tfds_as_supervised, + input_context=input_context, + seed=self._seed, + is_training=self._is_training, + cache=self._cache, + cycle_length=self._cycle_length, + block_length=self._block_length) + elif isinstance(matched_files, (list, tuple)): + dataset = _files_to_dataset(matched_files) + elif isinstance(matched_files, dict): + dataset = {} + for k, fs in matched_files.items(): + dataset[k] = _files_to_dataset(fs) + else: + raise ValueError('`matched_files` should be a list or dict.') + + return dataset + + def _decode_and_parse_dataset( + self, + dataset: Union[tf.data.Dataset, Dict[Text, tf.data.Dataset]], + batch_size: int, + input_context: Optional[tf.distribute.InputContext] = None + ) -> tf.data.Dataset: + """Returns a tf.data.Dataset object after shuffling, decoding, and parsing.""" + + def _shuffle_and_decode(ds): + # If cache is enabled, we will call `shuffle()` later after `cache()`. + if self._is_training and not self._cache: + ds = ds.shuffle(self._shuffle_buffer_size, seed=self._seed) + # Decode + ds = _maybe_map_fn(ds, self._decoder_fn) + return ds + + dataset = tf.nest.map_structure(_shuffle_and_decode, dataset) + if tf.nest.is_nested(dataset): + dataset = self._combine_fn(dataset) + + if self._sample_fn is not None: + dataset = dataset.apply(self._sample_fn) + dataset = _maybe_map_fn(dataset, self._parser_fn) + + if self._cache: + dataset = dataset.cache() + if self._is_training: + dataset = dataset.repeat() + dataset = dataset.shuffle(self._shuffle_buffer_size, seed=self._seed) + + if self._transform_and_batch_fn is not None: + dataset = self._transform_and_batch_fn(dataset, input_context) + else: + per_replica_batch_size = input_context.get_per_replica_batch_size( + batch_size) if input_context else batch_size + dataset = dataset.batch( + per_replica_batch_size, drop_remainder=self._drop_remainder) + + return dataset + + def _maybe_apply_data_service( + self, + dataset: tf.data.Dataset, + input_context: Optional[tf.distribute.InputContext] = None + ) -> tf.data.Dataset: + """Potentially distributes a dataset.""" + if self._enable_tf_data_service and input_context: + if self._enable_round_robin_tf_data_service: + replicas_per_input_pipeline = input_context.num_replicas_in_sync // ( + input_context.num_input_pipelines) + base_consumer_index = input_context.input_pipeline_id * ( + replicas_per_input_pipeline) + num_consumers = input_context.num_input_pipelines * ( + replicas_per_input_pipeline) + range_dataset = tf.data.Dataset.range(replicas_per_input_pipeline) + dataset = range_dataset.map(lambda i: dataset.apply( # pylint: disable=g-long-lambda + tf.data.experimental.service.distribute( + processing_mode='parallel_epochs', + service=self._tf_data_service_address, + job_name=self._tf_data_service_job_name, + consumer_index=base_consumer_index + i, + num_consumers=num_consumers))) + # Use parallel interleave to read multiple batches from a tf.data + # service worker in parallel. + dataset = dataset.interleave( + lambda x: x, + cycle_length=replicas_per_input_pipeline, + num_parallel_calls=replicas_per_input_pipeline, + deterministic=True) + else: + dataset = dataset.apply( + tf.data.experimental.service.distribute( + processing_mode='parallel_epochs', + service=self._tf_data_service_address, + job_name=self._tf_data_service_job_name)) + return dataset + + def read(self, + input_context: Optional[tf.distribute.InputContext] = None, + dataset: Optional[tf.data.Dataset] = None) -> tf.data.Dataset: + """Generates a tf.data.Dataset object.""" + if dataset is None: + dataset = self._read_data_source( + self._matched_files, self._dataset_fn, input_context, + self._tfds_builder) + dataset = self._decode_and_parse_dataset(dataset, self._global_batch_size, + input_context) + dataset = _maybe_map_fn(dataset, self._postprocess_fn) + dataset = self._maybe_apply_data_service(dataset, input_context) + + if self._deterministic is not None: + options = tf.data.Options() + options.experimental_deterministic = self._deterministic + dataset = dataset.with_options(options) + return dataset.prefetch(tf.data.experimental.AUTOTUNE) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/registry.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..f349710b54f082c8e5d2843b23210c16c8a59023 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/registry.py @@ -0,0 +1,101 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Registry utility.""" + + +def register(registered_collection, reg_key): + """Register decorated function or class to collection. + + Register decorated function or class into registered_collection, in a + hierarchical order. For example, when reg_key="my_model/my_exp/my_config_0" + the decorated function or class is stored under + registered_collection["my_model"]["my_exp"]["my_config_0"]. + This decorator is supposed to be used together with the lookup() function in + this file. + + Args: + registered_collection: a dictionary. The decorated function or class will be + put into this collection. + reg_key: The key for retrieving the registered function or class. If reg_key + is a string, it can be hierarchical like my_model/my_exp/my_config_0 + Returns: + A decorator function + Raises: + KeyError: when function or class to register already exists. + """ + def decorator(fn_or_cls): + """Put fn_or_cls in the dictionary.""" + if isinstance(reg_key, str): + hierarchy = reg_key.split("/") + collection = registered_collection + for h_idx, entry_name in enumerate(hierarchy[:-1]): + if entry_name not in collection: + collection[entry_name] = {} + collection = collection[entry_name] + if not isinstance(collection, dict): + raise KeyError( + "Collection path {} at position {} already registered as " + "a function or class.".format(entry_name, h_idx)) + leaf_reg_key = hierarchy[-1] + else: + collection = registered_collection + leaf_reg_key = reg_key + + if leaf_reg_key in collection: + raise KeyError("Function or class {} registered multiple times.".format( + leaf_reg_key)) + + collection[leaf_reg_key] = fn_or_cls + return fn_or_cls + return decorator + + +def lookup(registered_collection, reg_key): + """Lookup and return decorated function or class in the collection. + + Lookup decorated function or class in registered_collection, in a + hierarchical order. For example, when + reg_key="my_model/my_exp/my_config_0", + this function will return + registered_collection["my_model"]["my_exp"]["my_config_0"]. + + Args: + registered_collection: a dictionary. The decorated function or class will be + retrieved from this collection. + reg_key: The key for retrieving the registered function or class. If reg_key + is a string, it can be hierarchical like my_model/my_exp/my_config_0 + Returns: + The registered function or class. + Raises: + LookupError: when reg_key cannot be found. + """ + if isinstance(reg_key, str): + hierarchy = reg_key.split("/") + collection = registered_collection + for h_idx, entry_name in enumerate(hierarchy): + if entry_name not in collection: + raise LookupError( + f"collection path {entry_name} at position {h_idx} is never " + f"registered. Please make sure the {entry_name} and its library is " + "imported and linked to the trainer binary.") + collection = collection[entry_name] + return collection + else: + if reg_key not in registered_collection: + raise LookupError( + f"registration key {reg_key} is never " + f"registered. Please make sure the {reg_key} and its library is " + "imported and linked to the trainer binary.") + return registered_collection[reg_key] diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/registry_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/registry_test.py new file mode 100644 index 0000000000000000000000000000000000000000..0d0639c6b10d5f9d587593d52dd6f2458c83bcd5 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/registry_test.py @@ -0,0 +1,88 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for registry.""" + +import tensorflow as tf +from official.core import registry + + +class RegistryTest(tf.test.TestCase): + + def test_register(self): + collection = {} + + @registry.register(collection, 'functions/func_0') + def func_test(): + pass + + self.assertEqual(registry.lookup(collection, 'functions/func_0'), func_test) + + @registry.register(collection, 'classes/cls_0') + class ClassRegistryKey: + pass + + self.assertEqual( + registry.lookup(collection, 'classes/cls_0'), ClassRegistryKey) + + @registry.register(collection, ClassRegistryKey) + class ClassRegistryValue: + pass + + self.assertEqual( + registry.lookup(collection, ClassRegistryKey), ClassRegistryValue) + + def test_register_hierarchy(self): + collection = {} + + @registry.register(collection, 'functions/func_0') + def func_test0(): + pass + + @registry.register(collection, 'func_1') + def func_test1(): + pass + + @registry.register(collection, func_test1) + def func_test2(): + pass + + expected_collection = { + 'functions': { + 'func_0': func_test0, + }, + 'func_1': func_test1, + func_test1: func_test2, + } + self.assertEqual(collection, expected_collection) + + def test_register_error(self): + collection = {} + + @registry.register(collection, 'functions/func_0') + def func_test0(): # pylint: disable=unused-variable + pass + + with self.assertRaises(KeyError): + + @registry.register(collection, 'functions/func_0/sub_func') + def func_test1(): # pylint: disable=unused-variable + pass + + with self.assertRaises(LookupError): + registry.lookup(collection, 'non-exist') + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/task_factory.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/task_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..f5862462e0da94ad183e8bb7a5d60a7cad6e1b79 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/task_factory.py @@ -0,0 +1,70 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A global factory to register and access all registered tasks.""" + +from official.core import registry + +_REGISTERED_TASK_CLS = {} + + +# TODO(b/158741360): Add type annotations once pytype checks across modules. +def register_task_cls(task_config_cls): + """Decorates a factory of Tasks for lookup by a subclass of TaskConfig. + + This decorator supports registration of tasks as follows: + + ``` + @dataclasses.dataclass + class MyTaskConfig(TaskConfig): + # Add fields here. + pass + + @register_task_cls(MyTaskConfig) + class MyTask(Task): + # Inherits def __init__(self, task_config). + pass + + my_task_config = MyTaskConfig() + my_task = get_task(my_task_config) # Returns MyTask(my_task_config). + ``` + + Besisdes a class itself, other callables that create a Task from a TaskConfig + can be decorated by the result of this function, as long as there is at most + one registration for each config class. + + Args: + task_config_cls: a subclass of TaskConfig (*not* an instance of TaskConfig). + Each task_config_cls can only be used for a single registration. + + Returns: + A callable for use as class decorator that registers the decorated class + for creation from an instance of task_config_cls. + """ + return registry.register(_REGISTERED_TASK_CLS, task_config_cls) + + +def get_task(task_config, **kwargs): + """Creates a Task (of suitable subclass type) from task_config.""" + # TODO(hongkuny): deprecate the task factory to use config.BUILDER. + if task_config.BUILDER is not None: + return task_config.BUILDER(task_config, **kwargs) + return get_task_cls(task_config.__class__)(task_config, **kwargs) + + +# The user-visible get_task() is defined after classes have been registered. +# TODO(b/158741360): Add type annotations once pytype checks across modules. +def get_task_cls(task_config_cls): + task_cls = registry.lookup(_REGISTERED_TASK_CLS, task_config_cls) + return task_cls diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/test_utils.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/test_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..015373699c5c5917e9f866686d5817a791155d01 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/test_utils.py @@ -0,0 +1,59 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utils for testing.""" + +import tensorflow as tf + + +class FakeKerasModel(tf.keras.Model): + """Fake keras model for testing.""" + + def __init__(self): + super().__init__() + self.dense = tf.keras.layers.Dense(4, activation=tf.nn.relu) + self.dense2 = tf.keras.layers.Dense(4, activation=tf.nn.relu) + + def call(self, inputs): + return self.dense2(self.dense(inputs)) + + +class _Dense(tf.Module): + """A dense layer.""" + + def __init__(self, input_dim, output_size, name=None): + super().__init__(name=name) + with self.name_scope: + self.w = tf.Variable( + tf.random.normal([input_dim, output_size]), name='w') + self.b = tf.Variable(tf.zeros([output_size]), name='b') + + @tf.Module.with_name_scope + def __call__(self, x): + y = tf.matmul(x, self.w) + self.b + return tf.nn.relu(y) + + +class FakeModule(tf.Module): + """Fake model using tf.Module for testing.""" + + def __init__(self, input_size, name=None): + super().__init__(name=name) + with self.name_scope: + self.dense = _Dense(input_size, 4, name='dense') + self.dense2 = _Dense(4, 4, name='dense_1') + + @tf.Module.with_name_scope + def __call__(self, x): + return self.dense2(self.dense(x)) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/train_lib.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/train_lib.py new file mode 100644 index 0000000000000000000000000000000000000000..5f548ea722591bb878e468647b449426105ec74b --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/train_lib.py @@ -0,0 +1,150 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TFM common training driver library.""" +# pytype: disable=attribute-error +import os +from typing import Any, Mapping, Optional, Tuple + +# Import libraries + +from absl import logging +import orbit +import tensorflow as tf + +from official.core import actions +from official.core import base_task +from official.core import base_trainer +from official.core import config_definitions +from official.core import train_utils + +maybe_create_best_ckpt_exporter = train_utils.maybe_create_best_ckpt_exporter + + +def run_experiment( + distribution_strategy: tf.distribute.Strategy, + task: base_task.Task, + mode: str, + params: config_definitions.ExperimentConfig, + model_dir: str, + run_post_eval: bool = False, + save_summary: bool = True, + trainer: Optional[base_trainer.Trainer] = None, + controller_cls=orbit.Controller +) -> Tuple[tf.keras.Model, Mapping[str, Any]]: + """Runs train/eval configured by the experiment params. + + Args: + distribution_strategy: A distribution distribution_strategy. + task: A Task instance. + mode: A 'str', specifying the mode. Can be 'train', 'eval', 'train_and_eval' + or 'continuous_eval'. + params: ExperimentConfig instance. + model_dir: A 'str', a path to store model checkpoints and summaries. + run_post_eval: Whether to run post eval once after training, metrics logs + are returned. + save_summary: Whether to save train and validation summary. + trainer: the base_trainer.Trainer instance. It should be created within the + strategy.scope(). + controller_cls: The controller class to manage the train and eval process. + Must be a orbit.Controller subclass. + + Returns: + A 2-tuple of (model, eval_logs). + model: `tf.keras.Model` instance. + eval_logs: returns eval metrics logs when run_post_eval is set to True, + otherwise, returns {}. + """ + + with distribution_strategy.scope(): + if not trainer: + trainer = train_utils.create_trainer( + params, + task, + train='train' in mode, + evaluate=('eval' in mode) or run_post_eval, + checkpoint_exporter=maybe_create_best_ckpt_exporter( + params, model_dir)) + + if trainer.checkpoint: + if model_dir is None: + raise ValueError('model_dir must be specified, but got None') + checkpoint_manager = tf.train.CheckpointManager( + trainer.checkpoint, + directory=model_dir, + max_to_keep=params.trainer.max_to_keep, + step_counter=trainer.global_step, + checkpoint_interval=params.trainer.checkpoint_interval, + init_fn=trainer.initialize) + else: + checkpoint_manager = None + + controller = controller_cls( + strategy=distribution_strategy, + trainer=trainer if 'train' in mode else None, + evaluator=trainer, + global_step=trainer.global_step, + steps_per_loop=params.trainer.steps_per_loop, + checkpoint_manager=checkpoint_manager, + summary_dir=os.path.join(model_dir, 'train') if (save_summary) else None, + eval_summary_dir=os.path.join(model_dir, + params.trainer.validation_summary_subdir) if + (save_summary) else None, + summary_interval=params.trainer.summary_interval if + (save_summary) else None, + train_actions=actions.get_train_actions( + params, trainer, model_dir, checkpoint_manager=checkpoint_manager), + eval_actions=actions.get_eval_actions(params, trainer, model_dir)) + + logging.info('Starts to execute mode: %s', mode) + with distribution_strategy.scope(): + if mode == 'train': + controller.train(steps=params.trainer.train_steps) + elif mode == 'train_and_eval': + controller.train_and_evaluate( + train_steps=params.trainer.train_steps, + eval_steps=params.trainer.validation_steps, + eval_interval=params.trainer.validation_interval) + elif mode == 'eval': + controller.evaluate(steps=params.trainer.validation_steps) + elif mode == 'continuous_eval': + + def timeout_fn(): + if trainer.global_step.numpy() >= params.trainer.train_steps: + return True + return False + + controller.evaluate_continuously( + steps=params.trainer.validation_steps, + timeout=params.trainer.continuous_eval_timeout, + timeout_fn=timeout_fn) + else: + raise NotImplementedError('The mode is not implemented: %s' % mode) + + num_params = train_utils.try_count_params(trainer.model) + if num_params is not None: + logging.info('Number of trainable params in model: %f Millions.', + num_params / 10.**6) + + flops = train_utils.try_count_flops(trainer.model) + if flops is not None: + logging.info('FLOPs (multi-adds) in model: %f Billions.', + flops / 10.**9 / 2) + + if run_post_eval: + with distribution_strategy.scope(): + return trainer.model, trainer.evaluate( + tf.convert_to_tensor(params.trainer.validation_steps)) + else: + return trainer.model, {} diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/train_lib_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/train_lib_test.py new file mode 100644 index 0000000000000000000000000000000000000000..9c27054d539b009e43ba6f2e9b846d49e02357a1 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/train_lib_test.py @@ -0,0 +1,225 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for train_ctl_lib.""" +import json +import os + +from absl import flags +from absl.testing import flagsaver +from absl.testing import parameterized +import numpy as np +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.common import flags as tfm_flags +# pylint: disable=unused-import +from official.common import registry_imports +# pylint: enable=unused-import +from official.core import task_factory +from official.core import train_lib +from official.core import train_utils +from official.utils.testing import mock_task + +FLAGS = flags.FLAGS + +tfm_flags.define_flags() + + +class TrainTest(tf.test.TestCase, parameterized.TestCase): + + def setUp(self): + super(TrainTest, self).setUp() + self._test_config = { + 'trainer': { + 'checkpoint_interval': 10, + 'steps_per_loop': 10, + 'summary_interval': 10, + 'train_steps': 10, + 'validation_steps': 5, + 'validation_interval': 10, + 'continuous_eval_timeout': 1, + 'validation_summary_subdir': 'validation', + 'optimizer_config': { + 'optimizer': { + 'type': 'sgd', + }, + 'learning_rate': { + 'type': 'constant' + } + } + }, + } + + @combinations.generate( + combinations.combine( + distribution_strategy=[ + strategy_combinations.default_strategy, + strategy_combinations.cloud_tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ], + flag_mode=['train', 'eval', 'train_and_eval'], + run_post_eval=[True, False])) + def test_end_to_end(self, distribution_strategy, flag_mode, run_post_eval): + model_dir = self.get_temp_dir() + flags_dict = dict( + experiment='mock', + mode=flag_mode, + model_dir=model_dir, + params_override=json.dumps(self._test_config)) + with flagsaver.flagsaver(**flags_dict): + params = train_utils.parse_configuration(flags.FLAGS) + train_utils.serialize_config(params, model_dir) + with distribution_strategy.scope(): + task = task_factory.get_task(params.task, logging_dir=model_dir) + + _, logs = train_lib.run_experiment( + distribution_strategy=distribution_strategy, + task=task, + mode=flag_mode, + params=params, + model_dir=model_dir, + run_post_eval=run_post_eval) + + if 'eval' in flag_mode: + self.assertTrue( + tf.io.gfile.exists( + os.path.join(model_dir, + params.trainer.validation_summary_subdir))) + if run_post_eval: + self.assertNotEmpty(logs) + else: + self.assertEmpty(logs) + self.assertNotEmpty( + tf.io.gfile.glob(os.path.join(model_dir, 'params.yaml'))) + if flag_mode == 'eval': + return + self.assertNotEmpty( + tf.io.gfile.glob(os.path.join(model_dir, 'checkpoint'))) + # Tests continuous evaluation. + _, logs = train_lib.run_experiment( + distribution_strategy=distribution_strategy, + task=task, + mode='continuous_eval', + params=params, + model_dir=model_dir, + run_post_eval=run_post_eval) + + @combinations.generate( + combinations.combine( + distribution_strategy=[ + strategy_combinations.default_strategy, + strategy_combinations.cloud_tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ], + flag_mode=['train', 'train_and_eval'], + )) + def test_recovery_nan_error(self, distribution_strategy, flag_mode): + model_dir = self.get_temp_dir() + flags_dict = dict( + experiment='mock', + mode=flag_mode, + model_dir=model_dir, + params_override=json.dumps(self._test_config)) + with flagsaver.flagsaver(**flags_dict): + params = train_utils.parse_configuration(flags.FLAGS) + train_utils.serialize_config(params, model_dir) + with distribution_strategy.scope(): + # task = task_factory.get_task(params.task, logging_dir=model_dir) + task = mock_task.MockTask(params.task, logging_dir=model_dir) + + # Set the loss to NaN to trigger RunTimeError. + def build_losses(labels, model_outputs, aux_losses=None): + del labels, model_outputs + return tf.constant([np.nan], tf.float32) + aux_losses + + task.build_losses = build_losses + + with self.assertRaises(RuntimeError): + train_lib.run_experiment( + distribution_strategy=distribution_strategy, + task=task, + mode=flag_mode, + params=params, + model_dir=model_dir) + + @combinations.generate( + combinations.combine( + distribution_strategy=[ + strategy_combinations.default_strategy, + strategy_combinations.cloud_tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ], + flag_mode=['train'], + )) + def test_recovery(self, distribution_strategy, flag_mode): + loss_threshold = 1.0 + model_dir = self.get_temp_dir() + flags_dict = dict( + experiment='mock', + mode=flag_mode, + model_dir=model_dir, + params_override=json.dumps(self._test_config)) + with flagsaver.flagsaver(**flags_dict): + params = train_utils.parse_configuration(flags.FLAGS) + params.trainer.loss_upper_bound = loss_threshold + params.trainer.recovery_max_trials = 1 + train_utils.serialize_config(params, model_dir) + with distribution_strategy.scope(): + task = task_factory.get_task(params.task, logging_dir=model_dir) + + # Saves a checkpoint for reference. + model = task.build_model() + checkpoint = tf.train.Checkpoint(model=model) + checkpoint_manager = tf.train.CheckpointManager( + checkpoint, self.get_temp_dir(), max_to_keep=2) + checkpoint_manager.save() + before_weights = model.get_weights() + + def build_losses(labels, model_outputs, aux_losses=None): + del labels, model_outputs + return tf.constant([loss_threshold], tf.float32) + aux_losses + + task.build_losses = build_losses + + model, _ = train_lib.run_experiment( + distribution_strategy=distribution_strategy, + task=task, + mode=flag_mode, + params=params, + model_dir=model_dir) + after_weights = model.get_weights() + for left, right in zip(before_weights, after_weights): + self.assertAllEqual(left, right) + + def test_parse_configuration(self): + model_dir = self.get_temp_dir() + flags_dict = dict( + experiment='mock', + mode='train', + model_dir=model_dir, + params_override=json.dumps(self._test_config)) + with flagsaver.flagsaver(**flags_dict): + params = train_utils.parse_configuration(flags.FLAGS, lock_return=True) + with self.assertRaises(ValueError): + params.override({'task': {'init_checkpoint': 'Foo'}}) + + params = train_utils.parse_configuration(flags.FLAGS, lock_return=False) + params.override({'task': {'init_checkpoint': 'Bar'}}) + self.assertEqual(params.task.init_checkpoint, 'Bar') + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/train_utils.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/train_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6387b1c3df9d38a14695759f2cc36e4c19fb3e85 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/train_utils.py @@ -0,0 +1,478 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Training utils.""" +import copy +import json +import os +import pprint +from typing import Any, Callable, Dict, List, Optional, Union + +from absl import logging +import dataclasses +import gin +import orbit +import tensorflow as tf + +# pylint: disable=g-direct-tensorflow-import +from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2_as_graph +# pylint: enable=g-direct-tensorflow-import +from official.core import base_task +from official.core import base_trainer +from official.core import config_definitions +from official.core import exp_factory +from official.modeling import hyperparams + + +def get_leaf_nested_dict(d: Dict[str, Any], keys: List[str]) -> Dict[str, Any]: + """Get leaf from a dictionary with arbitrary depth with a list of keys. + + Args: + d: The dictionary to extract value from. + keys: The list of keys to extract values recursively. + + Returns: + The value of the leaf. + + Raises: + KeyError: If the value of keys extracted is a dictionary. + """ + leaf = d + for k in keys: + if not isinstance(leaf, dict) or k not in leaf: + raise KeyError( + 'Path not exist while traversing the dictionary: d with keys' + ': %s.' % keys) + leaf = leaf[k] + + if isinstance(leaf, dict): + raise KeyError('The value extracted with keys: %s is not a leaf of the ' + 'dictionary: %s.' % (keys, d)) + return leaf + + +def cast_leaf_nested_dict(d: Dict[str, Any], + cast_fn: Callable[[Any], Any]) -> Dict[str, Any]: + """Cast the leaves of a dictionary with arbitrary depth in place. + + Args: + d: The dictionary to extract value from. + cast_fn: The casting function. + + Returns: + A dictionray with the same structure as d. + """ + for key, value in d.items(): + if isinstance(value, dict): + d[key] = cast_leaf_nested_dict(value, cast_fn) + else: + d[key] = cast_fn(value) + return d + + +def maybe_create_best_ckpt_exporter(params: config_definitions.ExperimentConfig, + data_dir: str) -> Any: + """Maybe create a BestCheckpointExporter object, according to the config.""" + export_subdir = params.trainer.best_checkpoint_export_subdir + metric_name = params.trainer.best_checkpoint_eval_metric + metric_comp = params.trainer.best_checkpoint_metric_comp + if data_dir and export_subdir and metric_name: + best_ckpt_dir = os.path.join(data_dir, export_subdir) + best_ckpt_exporter = BestCheckpointExporter(best_ckpt_dir, metric_name, + metric_comp) + logging.info( + 'Created the best checkpoint exporter. ' + 'data_dir: %s, export_subdir: %s, metric_name: %s', data_dir, + export_subdir, metric_name) + else: + best_ckpt_exporter = None + + return best_ckpt_exporter + + +# TODO(b/180147589): Add tests for this module. +class BestCheckpointExporter: + """Keeps track of the best result, and saves its checkpoint. + + Orbit will support an API for checkpoint exporter. This class will be used + together with orbit once this functionality is ready. + """ + + def __init__(self, export_dir: str, metric_name: str, metric_comp: str): + """Initialization. + + Args: + export_dir: The directory that will contain exported checkpoints. + metric_name: Indicates which metric to look at, when determining which + result is better. If eval_logs being passed to maybe_export_checkpoint + is a nested dictionary, use `|` as a seperator for different layers. + metric_comp: Indicates how to compare results. Either `lower` or `higher`. + """ + self._export_dir = export_dir + self._metric_name = metric_name.split('|') + self._metric_comp = metric_comp + if self._metric_comp not in ('lower', 'higher'): + raise ValueError('best checkpoint metric comp must be one of ' + 'higher, lower. Got: {}'.format(self._metric_comp)) + tf.io.gfile.makedirs(os.path.dirname(self.best_ckpt_logs_path)) + self._best_ckpt_logs = self._maybe_load_best_eval_metric() + self._checkpoint_manager = None + + def _get_checkpoint_manager(self, checkpoint): + """Gets an existing checkpoint manager or creates a new one.""" + if self._checkpoint_manager is None or (self._checkpoint_manager.checkpoint + != checkpoint): + logging.info('Creates a new checkpoint manager.') + self._checkpoint_manager = tf.train.CheckpointManager( + checkpoint, + directory=self._export_dir, + max_to_keep=1, + checkpoint_name='best_ckpt') + + return self._checkpoint_manager + + def maybe_export_checkpoint( + self, checkpoint, eval_logs, global_step, write_logs=True) -> bool: + """Compare eval_logs with past eval_logs and export checkpoint if better.""" + logging.info('[BestCheckpointExporter] received eval_logs: %s, at step: %d', + eval_logs, global_step) + if self._best_ckpt_logs is None or self._new_metric_is_better( + self._best_ckpt_logs, eval_logs): + self._best_ckpt_logs = eval_logs + if write_logs: + self.export_best_eval_metric(self._best_ckpt_logs, global_step) + self._get_checkpoint_manager(checkpoint).save() + return True + return False + + def _maybe_load_best_eval_metric(self): + if not tf.io.gfile.exists(self.best_ckpt_logs_path): + return None + with tf.io.gfile.GFile(self.best_ckpt_logs_path, 'r') as reader: + return json.loads(reader.read()) + + def _new_metric_is_better(self, old_logs, new_logs): + """Check if the metric in new_logs is better than the metric in old_logs.""" + old_value = float( + orbit.utils.get_value( + get_leaf_nested_dict(old_logs, self._metric_name))) + new_value = float( + orbit.utils.get_value( + get_leaf_nested_dict(new_logs, self._metric_name))) + + logging.info('[BestCheckpointExporter] comparing results. old: %f, new: %f', + old_value, new_value) + if self._metric_comp == 'higher': + if new_value > old_value: + logging.info('[BestCheckpointExporter] ' + 'the new number is better since it is higher.') + return True + else: # self._metric_comp == 'lower': + if new_value < old_value: + logging.info('[BestCheckpointExporter] ' + 'the new number is better since it is lower.') + return True + return False + + def export_best_eval_metric(self, eval_logs, global_step): + """Export evaluation results of the best checkpoint into a json file.""" + eval_logs_ext = copy.copy(eval_logs) + eval_logs_ext['best_ckpt_global_step'] = global_step + eval_logs_ext = cast_leaf_nested_dict( + eval_logs_ext, lambda x: float(orbit.utils.get_value(x))) + # Saving json file is very fast. + with tf.io.gfile.GFile(self.best_ckpt_logs_path, 'w') as writer: + writer.write(json.dumps(eval_logs_ext, indent=4) + '\n') + + @property + def best_ckpt_logs(self): + return self._best_ckpt_logs + + @property + def best_ckpt_logs_path(self): + return os.path.join(self._export_dir, 'info.json') + + @property + def best_ckpt_path(self): + """Returns the best ckpt path or None if there is no ckpt yet.""" + return tf.train.latest_checkpoint(self._export_dir) + + +@gin.configurable +def create_trainer(params: config_definitions.ExperimentConfig, + task: base_task.Task, + train: bool, + evaluate: bool, + checkpoint_exporter: Optional[BestCheckpointExporter] = None, + trainer_cls=base_trainer.Trainer) -> base_trainer.Trainer: + """Create trainer.""" + logging.info('Running default trainer.') + model = task.build_model() + optimizer = task.create_optimizer(params.trainer.optimizer_config, + params.runtime) + return trainer_cls( + params, + task, + model=model, + optimizer=optimizer, + train=train, + evaluate=evaluate, + checkpoint_exporter=checkpoint_exporter) + + +@dataclasses.dataclass +class ParseConfigOptions: + """Use this dataclass instead of FLAGS to customize parse_configuration().""" + experiment: str + config_file: List[str] + tpu: str = '' + tf_data_service: str = '' + params_override: str = '' + + def __contains__(self, name): + return name in dataclasses.asdict(self) + + +def parse_configuration(flags_obj, lock_return=True, print_return=True): + """Parses ExperimentConfig from flags.""" + + if flags_obj.experiment is None: + raise ValueError('The flag --experiment must be specified.') + + # 1. Get the default config from the registered experiment. + params = exp_factory.get_exp_config(flags_obj.experiment) + + # 2. Get the first level of override from `--config_file`. + # `--config_file` is typically used as a template that specifies the common + # override for a particular experiment. + for config_file in flags_obj.config_file or []: + params = hyperparams.override_params_dict( + params, config_file, is_strict=True) + + # 3. Override the TPU address and tf.data service address. + params.override({ + 'runtime': { + 'tpu': flags_obj.tpu, + }, + }) + if ('tf_data_service' in flags_obj and flags_obj.tf_data_service and + isinstance(params.task, config_definitions.TaskConfig)): + params.override({ + 'task': { + 'train_data': { + 'tf_data_service_address': flags_obj.tf_data_service, + }, + 'validation_data': { + 'tf_data_service_address': flags_obj.tf_data_service, + } + } + }) + + # 4. Get the second level of override from `--params_override`. + # `--params_override` is typically used as a further override over the + # template. For example, one may define a particular template for training + # ResNet50 on ImageNet in a config file and pass it via `--config_file`, + # then define different learning rates and pass it via `--params_override`. + if flags_obj.params_override: + params = hyperparams.override_params_dict( + params, flags_obj.params_override, is_strict=True) + + params.validate() + if lock_return: + params.lock() + + if print_return: + pp = pprint.PrettyPrinter() + logging.info('Final experiment parameters:\n%s', + pp.pformat(params.as_dict())) + + return params + + +def serialize_config(params: config_definitions.ExperimentConfig, + model_dir: str): + """Serializes and saves the experiment config.""" + if model_dir is None: + raise ValueError('model_dir must be specified, but got None') + params_save_path = os.path.join(model_dir, 'params.yaml') + logging.info('Saving experiment configuration to %s', params_save_path) + tf.io.gfile.makedirs(model_dir) + hyperparams.save_params_dict_to_yaml(params, params_save_path) + + +def save_gin_config(filename_suffix: str, model_dir: str): + """Serializes and saves the experiment config.""" + gin_save_path = os.path.join( + model_dir, 'operative_config.{}.gin'.format(filename_suffix)) + logging.info('Saving gin configurations to %s', gin_save_path) + tf.io.gfile.makedirs(model_dir) + with tf.io.gfile.GFile(gin_save_path, 'w') as f: + f.write(gin.operative_config_str()) + + +def read_global_step_from_checkpoint(ckpt_file_path): + """Read global step from checkpoint, or get global step from its filename.""" + global_step = tf.Variable(-1, dtype=tf.int64) + ckpt = tf.train.Checkpoint(global_step=global_step) + try: + ckpt.restore(ckpt_file_path).expect_partial() + global_step_maybe_restored = global_step.numpy() + except tf.errors.InvalidArgumentError: + global_step_maybe_restored = -1 + + if global_step_maybe_restored == -1: + raise ValueError('global_step not found in checkpoint {}. ' + 'If you want to run finetune eval jobs, you need to ' + 'make sure that your pretrain model writes ' + 'global_step in its checkpoints.'.format(ckpt_file_path)) + global_step_restored = global_step.numpy() + logging.info('get global_step %d from checkpoint %s', global_step_restored, + ckpt_file_path) + return global_step_restored + + +def write_json_summary(log_dir, global_step, eval_metrics): + """Dump evaluation metrics to json file.""" + serializable_dict = {} + for name, value in eval_metrics.items(): + if hasattr(value, 'numpy'): + serializable_dict[name] = str(value.numpy()) + else: + serializable_dict[name] = str(value) + output_json = os.path.join(log_dir, 'metrics-{}.json'.format(global_step)) + logging.info('Evaluation results at pretrain step %d: %s', global_step, + serializable_dict) + with tf.io.gfile.GFile(output_json, 'w') as writer: + writer.write(json.dumps(serializable_dict, indent=4) + '\n') + + +def write_summary(summary_writer, global_step, eval_metrics): + """Write evaluation metrics to TF summary.""" + numeric_dict = {} + for name, value in eval_metrics.items(): + numeric_dict[name] = float(orbit.utils.get_value(value)) + with summary_writer.as_default(): + for name, value in numeric_dict.items(): + tf.summary.scalar(name, value, step=global_step) + summary_writer.flush() + + +def remove_ckpts(model_dir): + """Remove model checkpoints, so we can restart.""" + ckpts = os.path.join(model_dir, 'ckpt-*') + logging.info('removing checkpoint files %s', ckpts) + for file_to_remove in tf.io.gfile.glob(ckpts): + tf.io.gfile.rmtree(file_to_remove) + + file_to_remove = os.path.join(model_dir, 'checkpoint') + if tf.io.gfile.exists(file_to_remove): + tf.io.gfile.remove(file_to_remove) + + +def write_model_params(model: Union[tf.Module, tf.keras.Model], + output_path: str) -> None: + """Writes the model parameters and shapes to a file. + + Args: + model: A model instance. + output_path: Output file path. + """ + with tf.io.gfile.GFile(output_path, 'w') as f: + total_params = 0 + for var in model.variables: + shape = tf.shape(var) + total_params += tf.math.reduce_prod(shape).numpy() + f.write(f'{var.name} {shape.numpy().tolist()}\n') + f.write(f'\nTotal params: {total_params}\n') + + +def try_count_params( + model: Union[tf.Module, tf.keras.Model], + trainable_only: bool = False): + """Count the number of parameters if model is possible. + + Args: + model: Try to count the number of params in this model. + trainable_only: Whether to calculate trainable params only. This flag is + not used when the model has `count_params` attribute. + + Returns: + The number of parameters or None. + """ + if hasattr(model, 'count_params'): + try: + return model.count_params() + except ValueError: + logging.info('Number of trainable params unknown, because the build() ' + 'methods in keras layers were not called. This is probably ' + 'because the model was not feed any input, e.g., the max ' + 'train step already reached before this run.') + return None + else: + total_params = 0 + variables = model.trainable_variables if trainable_only else model.variables + for var in variables: + shape = tf.shape(var) + total_params += tf.math.reduce_prod(shape).numpy() + return total_params + + +def try_count_flops(model: Union[tf.Module, tf.keras.Model], + inputs_kwargs: Optional[Dict[str, Any]] = None, + output_path: Optional[str] = None): + """Counts and returns model FLOPs. + + Args: + model: A model instance. + inputs_kwargs: An optional dictionary of argument pairs specifying inputs' + shape specifications to getting corresponding concrete function. + output_path: A file path to write the profiling results to. + + Returns: + The model's FLOPs. + """ + if hasattr(model, 'inputs'): + try: + # Get input shape and set batch size to 1. + if model.inputs: + inputs = [ + tf.TensorSpec([1] + input.shape[1:], input.dtype) + for input in model.inputs + ] + concrete_func = tf.function(model).get_concrete_function(inputs) + # If model.inputs is invalid, try to use the input to get concrete + # function for model.call (subclass model). + else: + concrete_func = tf.function(model.call).get_concrete_function( + **inputs_kwargs) + frozen_func, _ = convert_variables_to_constants_v2_as_graph(concrete_func) + + # Calculate FLOPs. + run_meta = tf.compat.v1.RunMetadata() + opts = tf.compat.v1.profiler.ProfileOptionBuilder.float_operation() + if output_path is not None: + opts['output'] = f'file:outfile={output_path}' + else: + opts['output'] = 'none' + flops = tf.compat.v1.profiler.profile( + graph=frozen_func.graph, run_meta=run_meta, options=opts) + return flops.total_float_ops + except Exception as e: # pylint: disable=broad-except + logging.info( + 'Failed to count model FLOPs with error %s, because the build() ' + 'methods in keras layers were not called. This is probably because ' + 'the model was not feed any input, e.g., the max train step already ' + 'reached before this run.', e) + return None + return None diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/core/train_utils_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/train_utils_test.py new file mode 100644 index 0000000000000000000000000000000000000000..896de5dd827e3d20b223eb53350040743c9c829d --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/core/train_utils_test.py @@ -0,0 +1,98 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for official.core.train_utils.""" + +import os + +import numpy as np +import tensorflow as tf + +from official.core import test_utils +from official.core import train_utils + + +class TrainUtilsTest(tf.test.TestCase): + + def test_get_leaf_nested_dict(self): + d = {'a': {'i': {'x': 5}}} + self.assertEqual(train_utils.get_leaf_nested_dict(d, ['a', 'i', 'x']), 5) + + def test_get_leaf_nested_dict_not_leaf(self): + with self.assertRaisesRegex(KeyError, 'The value extracted with keys.*'): + d = {'a': {'i': {'x': 5}}} + train_utils.get_leaf_nested_dict(d, ['a', 'i']) + + def test_get_leaf_nested_dict_path_not_exist_missing_key(self): + with self.assertRaisesRegex(KeyError, 'Path not exist while traversing .*'): + d = {'a': {'i': {'x': 5}}} + train_utils.get_leaf_nested_dict(d, ['a', 'i', 'y']) + + def test_get_leaf_nested_dict_path_not_exist_out_of_range(self): + with self.assertRaisesRegex(KeyError, 'Path not exist while traversing .*'): + d = {'a': {'i': {'x': 5}}} + train_utils.get_leaf_nested_dict(d, ['a', 'i', 'z']) + + def test_get_leaf_nested_dict_path_not_exist_meets_leaf(self): + with self.assertRaisesRegex(KeyError, 'Path not exist while traversing .*'): + d = {'a': {'i': 5}} + train_utils.get_leaf_nested_dict(d, ['a', 'i', 'z']) + + def test_cast_leaf_nested_dict(self): + d = {'a': {'i': {'x': '123'}}, 'b': 456.5} + d = train_utils.cast_leaf_nested_dict(d, int) + self.assertEqual(d['a']['i']['x'], 123) + self.assertEqual(d['b'], 456) + + def test_write_model_params_keras_model(self): + inputs = np.zeros([2, 3]) + model = test_utils.FakeKerasModel() + model(inputs) # Must do forward pass to build the model. + + filepath = os.path.join(self.create_tempdir(), 'model_params.txt') + train_utils.write_model_params(model, filepath) + actual = tf.io.gfile.GFile(filepath, 'r').read().splitlines() + + expected = [ + 'fake_keras_model/dense/kernel:0 [3, 4]', + 'fake_keras_model/dense/bias:0 [4]', + 'fake_keras_model/dense_1/kernel:0 [4, 4]', + 'fake_keras_model/dense_1/bias:0 [4]', + '', + 'Total params: 36', + ] + self.assertEqual(actual, expected) + + def test_write_model_params_module(self): + inputs = np.zeros([2, 3], dtype=np.float32) + model = test_utils.FakeModule(3, name='fake_module') + model(inputs) # Must do forward pass to build the model. + + filepath = os.path.join(self.create_tempdir(), 'model_params.txt') + train_utils.write_model_params(model, filepath) + actual = tf.io.gfile.GFile(filepath, 'r').read().splitlines() + + expected = [ + 'fake_module/dense/b:0 [4]', + 'fake_module/dense/w:0 [3, 4]', + 'fake_module/dense_1/b:0 [4]', + 'fake_module/dense_1/w:0 [4, 4]', + '', + 'Total params: 36', + ] + self.assertEqual(actual, expected) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/__init__.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e419af524b5f349fe04abfa820c3cb51b777d422 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/__init__.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..086e1fb975f8517dcff3c020f5fd932f6e55edc7 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Activations package definition.""" +from official.modeling.activations.gelu import gelu +from official.modeling.activations.relu import relu6 +from official.modeling.activations.sigmoid import hard_sigmoid +from official.modeling.activations.swish import hard_swish +from official.modeling.activations.swish import identity +from official.modeling.activations.swish import simple_swish diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/gelu.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/gelu.py new file mode 100644 index 0000000000000000000000000000000000000000..a73294aa5493747af66d9bbbc2cc26914600d7cf --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/gelu.py @@ -0,0 +1,32 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gaussian error linear unit.""" + +import tensorflow as tf + + +@tf.keras.utils.register_keras_serializable(package='Text') +def gelu(x): + """Gaussian Error Linear Unit. + + This is a smoother version of the RELU. + Original paper: https://arxiv.org/abs/1606.08415 + Args: + x: float Tensor to perform activation. + + Returns: + `x` with the GELU activation applied. + """ + return tf.keras.activations.gelu(x, approximate=True) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/gelu_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/gelu_test.py new file mode 100644 index 0000000000000000000000000000000000000000..cfe1950d9f112c3c33421c410ecdd4ceedd6f1d7 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/gelu_test.py @@ -0,0 +1,34 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the Gaussian error linear unit.""" + +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.modeling import activations + + +@keras_parameterized.run_all_keras_modes +class GeluTest(keras_parameterized.TestCase): + + def test_gelu(self): + expected_data = [[0.14967535, 0., -0.10032465], + [-0.15880796, -0.04540223, 2.9963627]] + gelu_data = activations.gelu([[.25, 0, -.25], [-1, -2, 3]]) + self.assertAllClose(expected_data, gelu_data) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/relu.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/relu.py new file mode 100644 index 0000000000000000000000000000000000000000..b3941b2f3462fa6a3eea28e023a4450bcc070797 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/relu.py @@ -0,0 +1,31 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Customized Relu activation.""" + +import tensorflow as tf + + +@tf.keras.utils.register_keras_serializable(package='Text') +def relu6(features): + """Computes the Relu6 activation function. + + Args: + features: A `Tensor` representing preactivation values. + + Returns: + The activation value. + """ + features = tf.convert_to_tensor(features) + return tf.nn.relu6(features) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/relu_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/relu_test.py new file mode 100644 index 0000000000000000000000000000000000000000..215f189ea9a00ed93bf012d33429fd82b3dc7ca6 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/relu_test.py @@ -0,0 +1,35 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the customized Relu activation.""" + +import tensorflow as tf + +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.modeling import activations + + +@keras_parameterized.run_all_keras_modes +class CustomizedReluTest(keras_parameterized.TestCase): + + def test_relu6(self): + features = [[.25, 0, -.25], [-1, -2, 3]] + customized_relu6_data = activations.relu6(features) + relu6_data = tf.nn.relu6(features) + self.assertAllClose(customized_relu6_data, relu6_data) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/sigmoid.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/sigmoid.py new file mode 100644 index 0000000000000000000000000000000000000000..277463040e784325f2b47a5492c98d6e3283ad08 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/sigmoid.py @@ -0,0 +1,31 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Customized Sigmoid activation.""" + +import tensorflow as tf + + +@tf.keras.utils.register_keras_serializable(package='Text') +def hard_sigmoid(features): + """Computes the hard sigmoid activation function. + + Args: + features: A `Tensor` representing preactivation values. + + Returns: + The activation value. + """ + features = tf.convert_to_tensor(features) + return tf.nn.relu6(features + tf.cast(3., features.dtype)) * 0.16667 diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/sigmoid_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/sigmoid_test.py new file mode 100644 index 0000000000000000000000000000000000000000..6aad90ef3645b08708dbfde155654070c40d72ce --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/sigmoid_test.py @@ -0,0 +1,40 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the customized Sigmoid activation.""" + +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import \ + keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.modeling import activations + + +@keras_parameterized.run_all_keras_modes +class CustomizedSigmoidTest(keras_parameterized.TestCase): + + def _hard_sigmoid_nn(self, x): + x = np.float32(x) + return tf.nn.relu6(x + 3.) * 0.16667 + + def test_hard_sigmoid(self): + features = [[.25, 0, -.25], [-1, -2, 3]] + customized_hard_sigmoid_data = activations.hard_sigmoid(features) + sigmoid_data = self._hard_sigmoid_nn(features) + self.assertAllClose(customized_hard_sigmoid_data, sigmoid_data) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/swish.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/swish.py new file mode 100644 index 0000000000000000000000000000000000000000..ea79985e3006f1400350601d9b857e947287ace1 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/swish.py @@ -0,0 +1,72 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Customized Swish activation.""" + +import tensorflow as tf + + +@tf.keras.utils.register_keras_serializable(package='Text') +def simple_swish(features): + """Computes the Swish activation function. + + The tf.nn.swish operation uses a custom gradient to reduce memory usage. + Since saving custom gradients in SavedModel is currently not supported, and + one would not be able to use an exported TF-Hub module for fine-tuning, we + provide this wrapper that can allow to select whether to use the native + TensorFlow swish operation, or whether to use a customized operation that + has uses default TensorFlow gradient computation. + + Args: + features: A `Tensor` representing preactivation values. + + Returns: + The activation value. + """ + features = tf.convert_to_tensor(features) + return features * tf.nn.sigmoid(features) + + +@tf.keras.utils.register_keras_serializable(package='Text') +def hard_swish(features): + """Computes a hard version of the swish function. + + This operation can be used to reduce computational cost and improve + quantization for edge devices. + + Args: + features: A `Tensor` representing preactivation values. + + Returns: + The activation value. + """ + features = tf.convert_to_tensor(features) + fdtype = features.dtype + return features * tf.nn.relu6(features + tf.cast(3., fdtype)) * (1. / 6.) + + +@tf.keras.utils.register_keras_serializable(package='Text') +def identity(features): + """Computes the identity function. + + Useful for helping in quantization. + + Args: + features: A `Tensor` representing preactivation values. + + Returns: + The activation value. + """ + features = tf.convert_to_tensor(features) + return tf.identity(features) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/swish_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/swish_test.py new file mode 100644 index 0000000000000000000000000000000000000000..3cb9495d8d19a3b89e4a9b2db0679090ac1e3e9d --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/activations/swish_test.py @@ -0,0 +1,44 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the customized Swish activation.""" +import numpy as np +import tensorflow as tf + +from tensorflow.python.keras import keras_parameterized # pylint: disable=g-direct-tensorflow-import +from official.modeling import activations + + +@keras_parameterized.run_all_keras_modes +class CustomizedSwishTest(keras_parameterized.TestCase): + + def _hard_swish_np(self, x): + x = np.float32(x) + return x * np.clip(x + 3, 0, 6) / 6 + + def test_simple_swish(self): + features = [[.25, 0, -.25], [-1, -2, 3]] + customized_swish_data = activations.simple_swish(features) + swish_data = tf.nn.swish(features) + self.assertAllClose(customized_swish_data, swish_data) + + def test_hard_swish(self): + features = [[.25, 0, -.25], [-1, -2, 3]] + customized_swish_data = activations.hard_swish(features) + swish_data = self._hard_swish_np(features) + self.assertAllClose(customized_swish_data, swish_data) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/experimental/tf2_utils_2x_wide.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/experimental/tf2_utils_2x_wide.py new file mode 100644 index 0000000000000000000000000000000000000000..16940cffa153104ca9839d80bd0021acc8bdf2fe --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/experimental/tf2_utils_2x_wide.py @@ -0,0 +1,186 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Stacking model horizontally.""" + +from absl import logging +import numpy as np +import tensorflow as tf + + +def expand_vector(v: np.ndarray) -> np.ndarray: + """Expands a vector with batch dimensions. + + Equivalent to expand_1_axis(v, epsilon=0.0, axis=-1) + + Args: + v: A vector with shape [..., a]. + + Returns: + A vector with shape [..., 2 * a]. + """ + return np.repeat(v, 2, axis=-1) + + +def expand_1_axis(w: np.ndarray, + epsilon: float, + axis: int) -> np.ndarray: + """Expands either the first dimension or the last dimension of w. + + If `axis = 0`, the following constraint will be satisfied: + matmul(x, w) == + matmul(expand_vector(x), expand_1_axis(w, epsilon=0.1, axis=0)) + + If `axis = -1`, the following constraint will be satisfied if `epsilon = 0.0`: + expand_vector(matmul(x, w)) == + 2 * matmul(x, expand_1_axis(w, epsilon=0.0, axis=-1)) + + Args: + w: Numpy array of shape [a_0, a_1, ..., a_i-1, a_i]. + epsilon: Symmetric Noise added to expanded tensor. + axis: Must be either 0 or -1. + + Returns: + Expanded numpy array. + """ + assert axis in (0, -1), ( + "Only support expanding the first or the last dimension. " + "Got: {}".format(axis)) + + rank = len(w.shape) + + d_w = np.random.normal(np.zeros_like(w), np.fabs(w) * epsilon, w.shape) + d_w = np.repeat(d_w, 2, axis=axis) + + sign_flip = np.array([1, -1]) + for _ in range(rank - 1): + sign_flip = np.expand_dims(sign_flip, axis=-1 if axis == 0 else 0) + sign_flip = np.tile(sign_flip, + [w.shape[0]] + [1] * (rank - 2) + [w.shape[-1]]) + + d_w *= sign_flip + w_expand = (np.repeat(w, 2, axis=axis) + d_w) / 2 + return w_expand + + +def expand_2_axes(w: np.ndarray, + epsilon: float) -> np.ndarray: + """Expands the first dimension and the last dimension of w. + + The following constraint will be satisfied: + expand_vector(matmul(x, w)) == matmul(expand_vector(x), expand_2_axes(w)) + + Args: + w: Numpy array of shape [a_0, a_1, ..., a_i-1, a_i]. + epsilon: Symmetric Noise added to expanded tensor. + + Returns: + Expanded numpy array. + """ + rank = len(w.shape) + + d_w = np.random.normal(np.zeros_like(w), np.fabs(w) * epsilon, w.shape) + d_w = np.repeat(np.repeat(d_w, 2, axis=0), 2, axis=-1) + + sign_flip = np.array([1, -1]) + for _ in range(rank - 1): + sign_flip = np.expand_dims(sign_flip, axis=-1) + sign_flip = np.tile(sign_flip, + [w.shape[0]] + [1] * (rank - 2) + [w.shape[-1] * 2]) + d_w *= sign_flip + + w_expand = (np.repeat(np.repeat(w, 2, axis=0), 2, axis=-1) + d_w) / 2 + return w_expand + + +def var_to_var(var_from: tf.Variable, + var_to: tf.Variable, + epsilon: float): + """Expands a variable to another variable. + + Assume the shape of `var_from` is (a, b, ..., y, z), the shape of `var_to` + can be (a, ..., z * 2), (a * 2, ..., z * 2), (a * 2, ..., z) + + If the shape of `var_to` is (a, ..., 2 * z): + For any x, tf.matmul(x, var_to) ~= expand_vector(tf.matmul(x, var_from)) / 2 + Not that there will be noise added to the left hand side, if epsilon != 0. + If the shape of `var_to` is (2 * a, ..., z): + For any x, tf.matmul(expand_vector(x), var_to) == tf.matmul(x, var_from) + If the shape of `var_to` is (2 * a, ..., 2 * z): + For any x, tf.matmul(expand_vector(x), var_to) == + expand_vector(tf.matmul(expand_vector(x), var_from)) + + Args: + var_from: input variable to expand. + var_to: output variable. + epsilon: the noise ratio that will be added, when splitting `var_from`. + """ + shape_from = var_from.shape + shape_to = var_to.shape + + if shape_from == shape_to: + var_to.assign(var_from) + + elif len(shape_from) == 1 and len(shape_to) == 1: + var_to.assign(expand_vector(var_from.numpy())) + + elif shape_from[0] * 2 == shape_to[0] and shape_from[-1] == shape_to[-1]: + var_to.assign(expand_1_axis(var_from.numpy(), epsilon=epsilon, axis=0)) + + elif shape_from[0] == shape_to[0] and shape_from[-1] * 2 == shape_to[-1]: + var_to.assign(expand_1_axis(var_from.numpy(), epsilon=epsilon, axis=-1)) + + elif shape_from[0] * 2 == shape_to[0] and shape_from[-1] * 2 == shape_to[-1]: + var_to.assign(expand_2_axes(var_from.numpy(), epsilon=epsilon)) + + else: + raise ValueError("Shape not supported, {}, {}".format(shape_from, shape_to)) + + +def model_to_model_2x_wide(model_from: tf.Module, + model_to: tf.Module, + epsilon: float = 0.1): + """Expands a model to a wider version. + + Also makes sure that the output of the model is not changed after expanding. + For example: + ``` + model_narrow = tf.keras.Sequential() + model_narrow.add(tf.keras.Input(shape=(3,))) + model_narrow.add(tf.keras.layers.Dense(4)) + model_narrow.add(tf.keras.layers.Dense(1)) + + model_wide = tf.keras.Sequential() + model_wide.add(tf.keras.Input(shape=(6,))) + model_wide.add(tf.keras.layers.Dense(8)) + model_wide.add(tf.keras.layers.Dense(1)) + + model_to_model_2x_wide(model_narrow, model_wide) + assert model_narrow([[1, 2, 3]]) == model_wide([[1, 1, 2, 2, 3, 3]]) + ``` + + We assume that `model_from` and `model_to` has the same architecture and only + widths of them differ. + + Args: + model_from: input model to expand. + model_to: output model whose variables will be assigned expanded values + according to `model_from`. + epsilon: the noise ratio that will be added, when splitting `var_from`. + """ + for w_from, w_to in zip(model_from.trainable_variables, + model_to.trainable_variables): + logging.info("expanding %s %s to %s %s", + w_from.name, w_from.shape, w_to.name, w_to.shape) + var_to_var(w_from, w_to, epsilon=epsilon) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/experimental/tf2_utils_2x_wide_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/experimental/tf2_utils_2x_wide_test.py new file mode 100644 index 0000000000000000000000000000000000000000..25d6e7628d16dfcc83cc6608da73d5ef31834751 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/experimental/tf2_utils_2x_wide_test.py @@ -0,0 +1,101 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for tf2_utils_2x_wide.""" + +import numpy as np +import tensorflow as tf + +from official.modeling.fast_training.experimental import tf2_utils_2x_wide + + +class Tf2Utils2XWideTest(tf.test.TestCase): + + def test_expand_vector(self): + x = np.array([1, 2]) + self.assertAllClose(tf2_utils_2x_wide.expand_vector(x), + np.array([1, 1, 2, 2])) + + def test_expand_matrix(self): + x = np.array([[1, 2], [3, 4]]) + x = tf2_utils_2x_wide.expand_2_axes(x, epsilon=0.1) + self.assertAllClose(x[0, :] + x[1, :], np.array([1, 1, 2, 2])) + self.assertAllClose(x[2, :] + x[3, :], np.array([3, 3, 4, 4])) + + def test_expand_matrix_axis_0(self): + x = np.array([[1, 2], [3, 4]]) + x = tf2_utils_2x_wide.expand_1_axis(x, axis=0, epsilon=0.1) + self.assertAllClose(x[0, :] + x[1, :], np.array([1, 2])) + self.assertAllClose(x[2, :] + x[3, :], np.array([3, 4])) + + def test_expand_matrix_axis_1(self): + x = np.array([[1, 2], [3, 4]]) + x = tf2_utils_2x_wide.expand_1_axis(x, axis=-1, epsilon=0.1) + self.assertAllClose(x[:, 0] + x[:, 1], np.array([1, 3])) + self.assertAllClose(x[:, 2] + x[:, 3], np.array([2, 4])) + + def test_expand_3d_tensor(self): + x0 = np.array([10, 11]) + x1 = np.array([10, 10, 11, 11]) + w0 = np.random.rand(2, 2) + w1 = tf2_utils_2x_wide.expand_2_axes(w0, epsilon=0.1) + o0 = np.matmul(x0, w0) + o1 = np.matmul(x1, w1) + self.assertAllClose(np.repeat(o0, 2, axis=-1), o1) + + def test_expand_3d_tensor_axis_0(self): + x0 = np.array([10, 11]) + x1 = np.array([10, 10, 11, 11]) + w0 = np.random.rand(2, 2) + w1 = tf2_utils_2x_wide.expand_1_axis(w0, axis=0, epsilon=0.1) + o0 = np.matmul(x0, w0) + o1 = np.matmul(x1, w1) + self.assertAllClose(o0, o1) + + def test_expand_3d_tensor_axis_2(self): + x = np.array([10, 11]) + w0 = np.random.rand(2, 2) + w1 = tf2_utils_2x_wide.expand_1_axis(w0, axis=-1, epsilon=0.1) + o0 = np.matmul(x, w0) + o1 = np.matmul(x, w1) + self.assertAllClose(o0, np.sum(o1.reshape(2, 2), axis=-1)) + + def test_end_to_end(self): + """Covers expand_vector, expand_2_axes, and expand_1_axis.""" + model_narrow = tf.keras.Sequential() + model_narrow.add(tf.keras.Input(shape=(3,))) + model_narrow.add(tf.keras.layers.Dense(4)) + model_narrow.add(tf.keras.layers.Dense(4)) + model_narrow.add(tf.keras.layers.Dense(1)) + + model_wide = tf.keras.Sequential() + model_wide.add(tf.keras.Input(shape=(6,))) + model_wide.add(tf.keras.layers.Dense(8)) + model_wide.add(tf.keras.layers.Dense(8)) + model_wide.add(tf.keras.layers.Dense(1)) + + x0 = np.array([[1, 2, 3]]) + x1 = np.array([[1, 1, 2, 2, 3, 3]]) + + # Call model once to build variables first. + _, _ = model_narrow(x0), model_wide(x1) + tf2_utils_2x_wide.model_to_model_2x_wide( + model_narrow, model_wide, epsilon=0.2) + + self.assertAllClose(model_narrow(x0), model_wide(x1), + rtol=1e-05, atol=1e-05) + + +if __name__ == "__main__": + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/policies.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/policies.py new file mode 100644 index 0000000000000000000000000000000000000000..b4f7c3f018bbf45896ca3eb5b3a327dcd9b4dfb7 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/policies.py @@ -0,0 +1,178 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Base ProgressivePolicy definition for progressive training. + +To write a progressive model, subclass ProgressivePolicy and implement its +abstract methods to handle each training stage. +""" + +import abc +import dataclasses +from typing import Any, Mapping +from absl import logging +import six +import tensorflow as tf + +from official.common import streamz_counters +from official.modeling.fast_training.progressive import utils +from official.modeling.hyperparams import base_config + + +@dataclasses.dataclass +class ProgressiveConfig(base_config.Config): + pass + + +@six.add_metaclass(abc.ABCMeta) +class ProgressivePolicy: + """The APIs for handling progressive training stages. + + Attributes: + cur_model: The model for the current progressive training stage. + cur_train_dataset: The train dataset function for the current stage. + cur_eval_dataset: The eval dataset function for the current stage. + cur_optimizer: The optimizer for the current stage. + cur_checkpoint_items: Items to be saved in and restored from checkpoints, + for the progressive trainer. + is_last_stage: Whether it is currently in the last stage. + + Interfaces: + is_stage_advancing: Returns if progressive training is advancing to the + next stage. + update_pt_stage: Update progressive training stage. + """ + + def __init__(self): + """Initialize stage policy.""" + self._cur_train_dataset = None + self._cur_eval_dataset = None + self._volatiles = utils.VolatileTrackable(optimizer=None, model=None) + + stage_id = 0 + self._stage_id = tf.Variable( + stage_id, + trainable=False, + dtype=tf.int64, + aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA, + shape=[]) + self._volatiles.reassign_trackable( + optimizer=self.get_optimizer(stage_id), + model=self.get_model(stage_id, old_model=None)) # pytype: disable=wrong-arg-types # typed-keras + + streamz_counters.progressive_policy_creation_counter.get_cell( + ).increase_by(1) + + def compute_stage_id(self, global_step: int) -> int: + for stage_id in range(self.num_stages()): + global_step -= self.num_steps(stage_id) + if global_step < 0: + return stage_id + logging.error('Global step %d found no matching progressive stages. ' + 'Default to the last stage.', global_step) + return self.num_stages() - 1 + + @abc.abstractmethod + def num_stages(self) -> int: + """Return the total number of progressive stages.""" + pass + + @abc.abstractmethod + def num_steps(self, stage_id: int) -> int: + """Return the total number of steps in this stage.""" + pass + + @abc.abstractmethod + def get_model(self, + stage_id: int, + old_model: tf.keras.Model = None) -> tf.keras.Model: # pytype: disable=annotation-type-mismatch # typed-keras + """Return model for this stage. For initialization, `old_model` = None.""" + pass + + @abc.abstractmethod + def get_optimizer(self, stage_id: int) -> tf.keras.optimizers.Optimizer: + """Return optimizer for this stage.""" + pass + + @abc.abstractmethod + def get_train_dataset(self, stage_id: int) -> tf.data.Dataset: + """Return training Dataset for this stage.""" + pass + + @abc.abstractmethod + def get_eval_dataset(self, stage_id: int) -> tf.data.Dataset: + """Return evaluation Dataset for this stage.""" + pass + + @property + def cur_model(self) -> tf.keras.Model: + return self._volatiles.model + + @property + def cur_train_dataset(self) -> tf.data.Dataset: + if self._cur_train_dataset is None: + self._cur_train_dataset = self.get_train_dataset(self._stage_id.numpy()) + return self._cur_train_dataset + + @property + def cur_eval_dataset(self) -> tf.data.Dataset: + if self._cur_eval_dataset is None: + self._cur_eval_dataset = self.get_eval_dataset(self._stage_id.numpy()) + return self._cur_eval_dataset + + @property + def cur_optimizer(self) -> tf.keras.optimizers.Optimizer: + return self._volatiles.optimizer + + @property + def is_last_stage(self) -> bool: + stage_id = self._stage_id.numpy() + return stage_id >= self.num_stages() - 1 + + @property + def cur_checkpoint_items(self) -> Mapping[str, Any]: + return dict(stage_id=self._stage_id, volatiles=self._volatiles) + + def is_stage_advancing(self, global_step: int) -> bool: + old_stage_id = self._stage_id.numpy() + new_stage_id = self.compute_stage_id(global_step) + return old_stage_id != new_stage_id + + def update_pt_stage(self, global_step: int, pass_old_model=True) -> None: + """Update progressive training internal status. + + Call this after a training loop ends. + + Args: + global_step: an integer scalar of the current global step. + pass_old_model: whether to pass the old_model to get_model() function. + This is set to False if the old_model is irrelevant (e.g, just a default + model from stage 0). + """ + old_stage_id = self._stage_id.numpy() + new_stage_id = self.compute_stage_id(global_step) + logging.info('Switching stage from %d to %d', old_stage_id, new_stage_id) + + # Update stage id. + self._stage_id.assign(new_stage_id) + # Update dataset function. + self._cur_train_dataset = None + self._cur_eval_dataset = None + + # Update optimizer and model. + new_optimizer = self.get_optimizer(new_stage_id) + self._volatiles.reassign_trackable(optimizer=new_optimizer) + new_model = self.get_model( + new_stage_id, old_model=self.cur_model if pass_old_model else None) + self._volatiles.reassign_trackable(model=new_model) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/train.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/train.py new file mode 100644 index 0000000000000000000000000000000000000000..f547ac9a56b0843abce6f2cdeccd6c2cd9d55217 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/train.py @@ -0,0 +1,69 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TFM binary for the progressive trainer.""" + +from absl import app +from absl import flags +import gin + +from official.common import distribute_utils +# pylint: disable=unused-import +from official.common import registry_imports +# pylint: enable=unused-import +from official.common import flags as tfm_flags +from official.core import task_factory +from official.core import train_utils +from official.modeling import performance +from official.modeling.fast_training.progressive import train_lib + +FLAGS = flags.FLAGS + + +def main(_): + gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_params) + params = train_utils.parse_configuration(FLAGS) + model_dir = FLAGS.model_dir + if 'train' in FLAGS.mode: + # Pure eval modes do not output yaml files. Otherwise continuous eval job + # may race against the train job for writing the same file. + train_utils.serialize_config(params, model_dir) + + # Sets mixed_precision policy. Using 'mixed_float16' or 'mixed_bfloat16' + # can have significant impact on model speeds by utilizing float16 in case of + # GPUs, and bfloat16 in the case of TPUs. loss_scale takes effect only when + # dtype is float16 + if params.runtime.mixed_precision_dtype: + performance.set_mixed_precision_policy(params.runtime.mixed_precision_dtype) + distribution_strategy = distribute_utils.get_distribution_strategy( + distribution_strategy=params.runtime.distribution_strategy, + all_reduce_alg=params.runtime.all_reduce_alg, + num_gpus=params.runtime.num_gpus, + tpu_address=params.runtime.tpu, + **params.runtime.model_parallelism()) + with distribution_strategy.scope(): + task = task_factory.get_task(params.task, logging_dir=model_dir) + + train_lib.run_experiment( + distribution_strategy=distribution_strategy, + task=task, + mode=FLAGS.mode, + params=params, + model_dir=model_dir) + + train_utils.save_gin_config(FLAGS.mode, model_dir) + +if __name__ == '__main__': + tfm_flags.define_flags() + app.run(main) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/train_lib.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/train_lib.py new file mode 100644 index 0000000000000000000000000000000000000000..baa132e197bb621b276c7a6471d07fb402a804c0 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/train_lib.py @@ -0,0 +1,126 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TFM progressive training driver library. + +Compared to the common training driver, the only difference is that we use +prog_trainer_lib.ProgressiveTrainer instead of the base trainer. +""" + +# pytype: disable=attribute-error +import os +from typing import Any, Mapping, Tuple + +# Import libraries +from absl import logging +import orbit +import tensorflow as tf +from official.core import base_task +from official.core import config_definitions +from official.core import train_lib as base_train_lib +from official.modeling.fast_training.progressive import trainer as prog_trainer_lib + + +def run_experiment(distribution_strategy: tf.distribute.Strategy, + task: base_task.Task, + mode: str, + params: config_definitions.ExperimentConfig, + model_dir: str, + run_post_eval: bool = False, + save_summary: bool = True) \ +-> Tuple[tf.keras.Model, Mapping[str, Any]]: + """Runs train/eval configured by the experiment params. + + Args: + distribution_strategy: A distribution distribution_strategy. + task: A Task instance. + mode: A 'str', specifying the mode. Can be 'train', 'eval', 'train_and_eval' + or 'continuous_eval'. + params: ExperimentConfig instance. + model_dir: A 'str', a path to store model checkpoints and summaries. + run_post_eval: Whether to run post eval once after training, metrics logs + are returned. + save_summary: Whether to save train and validation summary. + + Returns: + A 2-tuple of (model, eval_logs). + model: `tf.keras.Model` instance. + eval_logs: returns eval metrics logs when run_post_eval is set to True, + otherwise, returns {}. + """ + + with distribution_strategy.scope(): + logging.info('Running progressive trainer.') + trainer = prog_trainer_lib.ProgressiveTrainer( + params, task, ckpt_dir=model_dir, + train='train' in mode, + evaluate=('eval' in mode) or run_post_eval, + checkpoint_exporter=base_train_lib.maybe_create_best_ckpt_exporter( + params, model_dir)) + + if trainer.checkpoint: + checkpoint_manager = tf.train.CheckpointManager( + trainer.checkpoint, + directory=model_dir, + max_to_keep=params.trainer.max_to_keep, + step_counter=trainer.global_step, + checkpoint_interval=params.trainer.checkpoint_interval, + init_fn=trainer.initialize) + else: + checkpoint_manager = None + + controller = orbit.Controller( + strategy=distribution_strategy, + trainer=trainer if 'train' in mode else None, + evaluator=trainer, + global_step=trainer.global_step, + steps_per_loop=params.trainer.steps_per_loop, + checkpoint_manager=checkpoint_manager, + summary_dir=os.path.join(model_dir, 'train') if (save_summary) else None, + eval_summary_dir=os.path.join(model_dir, 'validation') if + (save_summary) else None, + summary_interval=params.trainer.summary_interval if + (save_summary) else None) + + logging.info('Starts to execute mode: %s', mode) + with distribution_strategy.scope(): + if mode == 'train': + controller.train(steps=params.trainer.train_steps) + elif mode == 'train_and_eval': + controller.train_and_evaluate( + train_steps=params.trainer.train_steps, + eval_steps=params.trainer.validation_steps, + eval_interval=params.trainer.validation_interval) + elif mode == 'eval': + controller.evaluate(steps=params.trainer.validation_steps) + elif mode == 'continuous_eval': + + def timeout_fn(): + if trainer.global_step.numpy() >= params.trainer.train_steps: + return True + return False + + controller.evaluate_continuously( + steps=params.trainer.validation_steps, + timeout=params.trainer.continuous_eval_timeout, + timeout_fn=timeout_fn) + else: + raise NotImplementedError('The mode is not implemented: %s' % mode) + + if run_post_eval: + with distribution_strategy.scope(): + return trainer.model, trainer.evaluate( + tf.convert_to_tensor(params.trainer.validation_steps)) + else: + return trainer.model, {} diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/train_lib_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/train_lib_test.py new file mode 100644 index 0000000000000000000000000000000000000000..f91faf902ebad5a7af92907fd434f585a580bf3c --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/train_lib_test.py @@ -0,0 +1,183 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the progressive train_lib.""" +import os + +from absl import flags +from absl.testing import parameterized +import dataclasses +import orbit +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.common import flags as tfm_flags +# pylint: disable=unused-import +from official.common import registry_imports +# pylint: enable=unused-import +from official.core import config_definitions as cfg +from official.core import task_factory +from official.modeling import optimization +from official.modeling.hyperparams import params_dict +from official.modeling.fast_training.progressive import policies +from official.modeling.fast_training.progressive import train_lib +from official.modeling.fast_training.progressive import trainer as prog_trainer_lib +from official.utils.testing import mock_task + +FLAGS = flags.FLAGS + +tfm_flags.define_flags() + + +@dataclasses.dataclass +class ProgTaskConfig(cfg.TaskConfig): + pass + + +@task_factory.register_task_cls(ProgTaskConfig) +class ProgMockTask(policies.ProgressivePolicy, mock_task.MockTask): + """Progressive task for testing.""" + + def __init__(self, params: cfg.TaskConfig, logging_dir: str = None): + mock_task.MockTask.__init__( + self, params=params, logging_dir=logging_dir) + policies.ProgressivePolicy.__init__(self) + + def num_stages(self): + return 2 + + def num_steps(self, stage_id): + return 2 if stage_id == 0 else 4 + + def get_model(self, stage_id, old_model=None): + del stage_id, old_model + return self.build_model() + + def get_optimizer(self, stage_id): + """Build optimizer for each stage.""" + params = optimization.OptimizationConfig({ + 'optimizer': { + 'type': 'adamw', + }, + 'learning_rate': { + 'type': 'polynomial', + 'polynomial': { + 'initial_learning_rate': 0.01, + 'end_learning_rate': 0.0, + 'power': 1.0, + 'decay_steps': 10, + }, + }, + 'warmup': { + 'polynomial': { + 'power': 1, + 'warmup_steps': 2, + }, + 'type': 'polynomial', + } + }) + opt_factory = optimization.OptimizerFactory(params) + optimizer = opt_factory.build_optimizer(opt_factory.build_learning_rate()) + + return optimizer + + def get_train_dataset(self, stage_id): + del stage_id + strategy = tf.distribute.get_strategy() + return orbit.utils.make_distributed_dataset( + strategy, self.build_inputs, None) + + def get_eval_dataset(self, stage_id): + del stage_id + strategy = tf.distribute.get_strategy() + return orbit.utils.make_distributed_dataset( + strategy, self.build_inputs, None) + + +class TrainTest(tf.test.TestCase, parameterized.TestCase): + + def setUp(self): + super(TrainTest, self).setUp() + self._test_config = { + 'trainer': { + 'checkpoint_interval': 10, + 'steps_per_loop': 10, + 'summary_interval': 10, + 'train_steps': 10, + 'validation_steps': 5, + 'validation_interval': 10, + 'continuous_eval_timeout': 1, + 'optimizer_config': { + 'optimizer': { + 'type': 'sgd', + }, + 'learning_rate': { + 'type': 'constant' + } + } + }, + } + + @combinations.generate( + combinations.combine( + distribution_strategy=[ + strategy_combinations.default_strategy, + strategy_combinations.cloud_tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ], + flag_mode=['train', 'eval', 'train_and_eval'], + run_post_eval=[True, False])) + def test_end_to_end(self, distribution_strategy, flag_mode, run_post_eval): + model_dir = self.get_temp_dir() + experiment_config = cfg.ExperimentConfig( + trainer=prog_trainer_lib.ProgressiveTrainerConfig(), + task=ProgTaskConfig()) + experiment_config = params_dict.override_params_dict( + experiment_config, self._test_config, is_strict=False) + + with distribution_strategy.scope(): + task = task_factory.get_task(experiment_config.task, + logging_dir=model_dir) + + _, logs = train_lib.run_experiment( + distribution_strategy=distribution_strategy, + task=task, + mode=flag_mode, + params=experiment_config, + model_dir=model_dir, + run_post_eval=run_post_eval) + + if run_post_eval: + self.assertNotEmpty(logs) + else: + self.assertEmpty(logs) + + if flag_mode == 'eval': + return + self.assertNotEmpty( + tf.io.gfile.glob(os.path.join(model_dir, 'checkpoint'))) + # Tests continuous evaluation. + _, logs = train_lib.run_experiment( + distribution_strategy=distribution_strategy, + task=task, + mode='continuous_eval', + params=experiment_config, + model_dir=model_dir, + run_post_eval=run_post_eval) + print(logs) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/trainer.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..685ec395045469c1120be0d02f6575e1b65fc070 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/trainer.py @@ -0,0 +1,294 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Progressive Trainer implementation. + +The trainer implements the Orbit `StandardTrainable` and +`StandardEvaluable` interfaces. Trainers inside this project should be +interchangable and independent on model architectures and tasks. +""" + +import dataclasses +import os +from typing import Any, Optional + +# Import libraries +from absl import logging +import gin +import orbit +import tensorflow as tf +from official.core import base_task +from official.core import base_trainer as trainer_lib +from official.core import config_definitions +from official.modeling.fast_training.progressive import policies +from official.modeling.fast_training.progressive import utils + +ExperimentConfig = config_definitions.ExperimentConfig + + +@dataclasses.dataclass +class ProgressiveTrainerConfig(config_definitions.TrainerConfig): + """Configuration for progressive trainer. + + Attributes: + progressive: A task-specific config. Users can subclass ProgressiveConfig + and define any task-specific settings in their subclass. + export_checkpoint: A bool. Whether to export checkpoints in non-progressive + manner (without the volatiles wrapper) such that your down-stream tasks + can load checkpoints from a progressive trainer as if it is a regular + checkpoint. + export_checkpoint_interval: A bool. The number of steps between exporting + checkpoints. If None (by default), will use the same value as + TrainerConfig.checkpoint_interval. + export_max_to_keep: The maximum number of exported checkpoints to keep. + If None (by default), will use the same value as + TrainerConfig.max_to_keep. + export_only_final_stage_ckpt: A bool. Whether to just export checkpoints + during the final progressive training stage. In other words, whether to + not export small, partial models. In many cases, it is not meaningful to + finetune a small, partial model in down-stream tasks. + """ + progressive: Optional[policies.ProgressiveConfig] = None + export_checkpoint: bool = True + export_checkpoint_interval: Optional[int] = None + export_max_to_keep: Optional[int] = None + export_only_final_stage_ckpt: bool = True + + +@gin.configurable +class ProgressiveTrainer(trainer_lib.Trainer): + """Implements the progressive trainer shared for TensorFlow models.""" + + def __init__( + self, + config: ExperimentConfig, + prog_task: base_task.Task, # also implemented ProgressivePolicy. + ckpt_dir: str = '', + train: bool = True, + evaluate: bool = True, + checkpoint_exporter: Any = None): + """Initialize common trainer for TensorFlow models. + + Args: + config: An `ExperimentConfig` instance specifying experiment config. + prog_task: An instance both implemented policies.ProgressivePolicy and + base_task.Task. + ckpt_dir: Checkpoint directory. + train: bool, whether or not this trainer will be used for training. + default to True. + evaluate: bool, whether or not this trainer will be used for evaluation. + default to True. + checkpoint_exporter: an object that has the `maybe_export_checkpoint` + interface. + """ + # Gets the current distribution strategy. If not inside any strategy scope, + # it gets a single-replica no-op strategy. + self._strategy = tf.distribute.get_strategy() + self._config = config + self._runtime_options = trainer_lib.get_runtime_options(config) + self._task = prog_task + + # Directory for non-progressive checkpoint + self._export_ckpt_dir = os.path.join(ckpt_dir, 'exported_ckpts') + tf.io.gfile.makedirs(self._export_ckpt_dir) + self._export_ckpt_manager = None + + # Receive other checkpoint export, e.g, best checkpoint exporter. + # TODO(lehou): unify the checkpoint exporting logic, although the default + # setting does not use checkpoint_exporter. + self._checkpoint_exporter = checkpoint_exporter + + self._global_step = orbit.utils.create_global_step() + + self._checkpoint = utils.CheckpointWithHooks( + before_load_hook=self._update_pt_stage_from_ckpt, + global_step=self.global_step, + **self._task.cur_checkpoint_items) + + self._train_loss = tf.keras.metrics.Mean('training_loss', dtype=tf.float32) + self._validation_loss = tf.keras.metrics.Mean( + 'validation_loss', dtype=tf.float32) + self._train_metrics = self.task.build_metrics( + training=True) + self.model.metrics + self._validation_metrics = self.task.build_metrics( + training=False) + self.model.metrics + + if train: + orbit.StandardTrainer.__init__( + self, + None, # Manage train_dataset by ourselves, not by StandardTrainer. + options=orbit.StandardTrainerOptions( + use_tf_while_loop=config.trainer.train_tf_while_loop, + use_tf_function=config.trainer.train_tf_function)) + + if evaluate: + orbit.StandardEvaluator.__init__( + self, + None, # Manage train_dataset by ourselves, not by StandardEvaluator. + options=orbit.StandardEvaluatorOptions( + use_tf_function=config.trainer.eval_tf_function)) + + @property + def model(self): + return self._task.cur_model + + @property + def optimizer(self): + return self._task.cur_optimizer + + # override + @property + def train_dataset(self): + """Overriding StandardTrainer.train_dataset.""" + return self._task.cur_train_dataset + + # override + @train_dataset.setter + def train_dataset(self, _): + raise SyntaxError('Please do not set train_dataset. Progressive training ' + 'relies on progressive policy to manager train dataset.') + + # override + @property + def eval_dataset(self): + """Overriding StandardEvaluator.eval_dataset.""" + return self._task.cur_eval_dataset + + # override + @eval_dataset.setter + def eval_dataset(self, _): + raise SyntaxError('Please do not set eval_dataset. Progressive training ' + 'relies on progressive policy to manager eval dataset.') + + def train_loop_end(self): + """See base class.""" + logs = {} + for metric in self.train_metrics + [self.train_loss]: + logs[metric.name] = metric.result() + metric.reset_states() + if callable(self.optimizer.learning_rate): + logs['learning_rate'] = self.optimizer.learning_rate( + self.optimizer.iterations) + else: + logs['learning_rate'] = self.optimizer.learning_rate + + self._maybe_export_non_progressive_checkpoint(self._export_ckpt_dir) + if self._task.is_stage_advancing(self.global_step.numpy()): + old_train_dataset = self.train_dataset + + # Update progressive properties + self._task.update_pt_stage(self.global_step.numpy()) + + # Setting `self._train_loop_fn` and `self._eval_loop_fn` to None will + # rebuild the train and eval functions with the updated model. + self._train_loop_fn = None + self._eval_loop_fn = None + + if self.train_dataset != old_train_dataset: + # Setting `self._train_iter` to None will rebuild the dataset iterator. + self._train_iter = None + + # Setting `self._export_ckpt_manager` to None will rebuild the checkpoint + # for exporting. + self._export_ckpt_manager = None + + return logs + + def _update_pt_stage_from_ckpt(self, ckpt_file): + """Update stage properties based on the global_step variable in a ckpt file. + + Before loading variables from a checkpoint file, we need to go to the + correct stage and build corresponding model and optimizer, to make sure that + we retore variables of the right model and optimizer. + + Args: + ckpt_file: Checkpoint file that will be restored/read from. + """ + if not ckpt_file: + return + ckpt = tf.train.Checkpoint(global_step=self.global_step) + ckpt.read(ckpt_file).expect_partial().assert_existing_objects_matched() + + if self._task.is_stage_advancing(self.global_step.numpy()): + old_train_dataset = self.train_dataset + + # Update progressive properties + self._task.update_pt_stage(self.global_step.numpy(), pass_old_model=False) + + # Setting `self._train_loop_fn` and `self._eval_loop_fn` to None will + # rebuild the train and eval functions with the updated model. + self._train_loop_fn = None + self._eval_loop_fn = None + + if self.train_dataset != old_train_dataset: + # Setting `self._train_iter` to None will rebuild the dataset iterator. + self._train_iter = None + + # Setting `self._export_ckpt_manager` to None will rebuild the checkpoint + # for exporting. + self._export_ckpt_manager = None + + def _maybe_export_non_progressive_checkpoint(self, export_ckpt_dir): + """Export checkpoints in non-progressive format. + + This basically removes the wrapping of self._task.cur_checkpoint_items + -- just save the model, optimizer, etc., directly. + The purpose is to let your down-stream tasks to use these checkpoints. + + Args: + export_ckpt_dir: A str. folder of exported checkpoints. + """ + if not self.config.trainer.export_checkpoint: + logging.info('Not exporting checkpoints.') + return + if not self._task.is_last_stage and ( + self.config.trainer.export_only_final_stage_ckpt): + logging.info('Not exporting checkpoints until the last stage.') + return + + if self._export_ckpt_manager is None: + # Create a checkpoint object just now, to make sure we use + # progressive_policy.cur_model and progressive_policy.cur_optimizer of the + # current stage. + if hasattr(self.model, 'checkpoint_items'): + checkpoint_items = self.model.checkpoint_items + else: + checkpoint_items = {} + checkpoint = tf.train.Checkpoint( + global_step=self.global_step, + model=self.model, + optimizer=self.optimizer, + **checkpoint_items) + + max_to_keep = self.config.trainer.export_max_to_keep or ( + self.config.trainer.max_to_keep) + checkpoint_interval = self.config.trainer.export_checkpoint_interval or ( + self.config.trainer.checkpoint_interval) + self._export_ckpt_manager = tf.train.CheckpointManager( + checkpoint, + directory=export_ckpt_dir, + checkpoint_name='ckpt', + step_counter=self.global_step, + max_to_keep=max_to_keep, + checkpoint_interval=checkpoint_interval, + ) + + # Make sure we export the last checkpoint. + last_checkpoint = ( + self.global_step.numpy() == self._config.trainer.train_steps) + checkpoint_path = self._export_ckpt_manager.save( + checkpoint_number=self.global_step.numpy(), + check_interval=not last_checkpoint) + if checkpoint_path: + logging.info('Checkpoints exported: %s.', checkpoint_path) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/trainer_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/trainer_test.py new file mode 100644 index 0000000000000000000000000000000000000000..a0c5d82a55dc94fc5c6f16dfe94f047b56ebf05f --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/trainer_test.py @@ -0,0 +1,238 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the progressive trainer.""" +# pylint: disable=g-direct-tensorflow-import +import os + +from absl.testing import parameterized +import orbit +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.core import config_definitions as cfg +from official.modeling import optimization +from official.modeling.fast_training.progressive import policies +from official.modeling.fast_training.progressive import trainer as trainer_lib +from official.nlp.configs import bert +from official.utils.testing import mock_task + + +def all_strategy_combinations(): + return combinations.combine( + distribution=[ + strategy_combinations.default_strategy, + strategy_combinations.cloud_tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ],) + + +def get_exp_config(): + return cfg.ExperimentConfig( + task=cfg.TaskConfig( + model=bert.PretrainerConfig()), + trainer=trainer_lib.ProgressiveTrainerConfig( + export_checkpoint=True, + export_checkpoint_interval=1, + export_only_final_stage_ckpt=False)) + + +class TestPolicy(policies.ProgressivePolicy, mock_task.MockTask): + """Just for testing purposes.""" + + def __init__(self, strategy, task_config, change_train_dataset=True): + self._strategy = strategy + self._change_train_dataset = change_train_dataset + self._my_train_dataset = None + mock_task.MockTask.__init__(self, params=task_config, logging_dir=None) + policies.ProgressivePolicy.__init__(self) + + def num_stages(self) -> int: + return 2 + + def num_steps(self, stage_id: int) -> int: + return 2 if stage_id == 0 else 4 + + def get_model(self, + stage_id: int, + old_model: tf.keras.Model) -> tf.keras.Model: + del stage_id, old_model + return self.build_model() + + def get_optimizer(self, stage_id: int) -> tf.keras.optimizers.Optimizer: + optimizer_type = 'sgd' if stage_id == 0 else 'adamw' + optimizer_config = cfg.OptimizationConfig({ + 'optimizer': {'type': optimizer_type}, + 'learning_rate': {'type': 'constant'}}) + opt_factory = optimization.OptimizerFactory(optimizer_config) + return opt_factory.build_optimizer(opt_factory.build_learning_rate()) + + def get_train_dataset(self, stage_id: int) -> tf.data.Dataset: + if not self._change_train_dataset and self._my_train_dataset: + return self._my_train_dataset + if self._strategy: + self._my_train_dataset = orbit.utils.make_distributed_dataset( + self._strategy, + self._build_inputs, + stage_id) + else: + self._my_train_dataset = self._build_inputs(stage_id) + return self._my_train_dataset + + def get_eval_dataset(self, stage_id: int) -> tf.data.Dataset: + if self._strategy: + return orbit.utils.make_distributed_dataset( + self._strategy, + self._build_inputs, + stage_id) + return self._build_inputs(stage_id) + + def _build_inputs(self, stage_id): + def dummy_data(_): + batch_size = 2 if stage_id == 0 else 1 + x = tf.zeros(shape=(batch_size, 2), dtype=tf.float32) + label = tf.zeros(shape=(batch_size, 1), dtype=tf.float32) + return x, label + dataset = tf.data.Dataset.range(1) + dataset = dataset.repeat() + return dataset.map( + dummy_data, num_parallel_calls=tf.data.experimental.AUTOTUNE) + + +class TrainerTest(tf.test.TestCase, parameterized.TestCase): + + def setUp(self): + super(TrainerTest, self).setUp() + self._config = get_exp_config() + + def create_test_trainer(self, distribution, model_dir, change_train_dataset): + trainer = trainer_lib.ProgressiveTrainer( + self._config, + prog_task=TestPolicy( + distribution, self._config.task, change_train_dataset), + ckpt_dir=model_dir) + return trainer + + @combinations.generate(all_strategy_combinations()) + def test_checkpointing(self, distribution): + model_dir = self.get_temp_dir() + ckpt_file = os.path.join(model_dir, 'ckpt') + with distribution.scope(): + trainer = self.create_test_trainer(distribution, model_dir, True) + self.assertFalse(trainer._task.is_last_stage) + trainer.train(tf.convert_to_tensor(4, dtype=tf.int32)) + self.assertTrue(trainer._task.is_last_stage) + trainer.checkpoint.save(ckpt_file) + + trainer = self.create_test_trainer(distribution, model_dir, True) + self.assertFalse(trainer._task.is_last_stage) + trainer.checkpoint.restore(ckpt_file + '-1') + self.assertTrue(trainer._task.is_last_stage) + + @combinations.generate(all_strategy_combinations()) + def test_train_dataset(self, distribution): + model_dir = self.get_temp_dir() + with distribution.scope(): + trainer = self.create_test_trainer(distribution, model_dir, True) + # Using dataset of stage == 0 + train_iter = tf.nest.map_structure(iter, trainer.train_dataset) + train_data = train_iter.next()[0] + if distribution.num_replicas_in_sync > 1: + train_data = train_data.values[0] + self.assertEqual(train_data.shape[0], 2) + + trainer.train(tf.convert_to_tensor(4, dtype=tf.int32)) + # Using dataset of stage == 1 + train_iter = tf.nest.map_structure(iter, trainer.train_dataset) + train_data = train_iter.next()[0] + if distribution.num_replicas_in_sync > 1: + train_data = train_data.values[0] + self.assertEqual(train_data.shape[0], 1) + + with self.assertRaises(SyntaxError): + trainer.train_dataset = None + + @combinations.generate(all_strategy_combinations()) + def test_train_dataset_no_switch(self, distribution): + model_dir = self.get_temp_dir() + with distribution.scope(): + trainer = self.create_test_trainer(distribution, model_dir, False) + trainer.train(tf.convert_to_tensor(2, dtype=tf.int32)) + # _train_iter is not reset since the dataset is not changed. + self.assertIsNotNone(trainer._train_iter) + with distribution.scope(): + trainer = self.create_test_trainer(distribution, model_dir, True) + trainer.train(tf.convert_to_tensor(2, dtype=tf.int32)) + # _train_iter is reset since the dataset changed. + self.assertIsNone(trainer._train_iter) + + +class TrainerWithMaskedLMTaskTest(tf.test.TestCase, parameterized.TestCase): + + def setUp(self): + super(TrainerWithMaskedLMTaskTest, self).setUp() + self._config = get_exp_config() + + def create_test_trainer(self, distribution): + trainer = trainer_lib.ProgressiveTrainer( + self._config, + prog_task=TestPolicy(distribution, self._config.task), + ckpt_dir=self.get_temp_dir()) + return trainer + + @combinations.generate(all_strategy_combinations()) + def test_trainer_train(self, distribution): + with distribution.scope(): + trainer = self.create_test_trainer(distribution) + logs = trainer.train(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertIn('training_loss', logs) + self.assertIn('learning_rate', logs) + + @combinations.generate(all_strategy_combinations()) + def test_trainer_validate(self, distribution): + with distribution.scope(): + trainer = self.create_test_trainer(distribution) + logs = trainer.evaluate(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertIn('validation_loss', logs) + self.assertEqual(logs['counter'], 5. * distribution.num_replicas_in_sync) + + @combinations.generate( + combinations.combine( + mixed_precision_dtype=['float32', 'bfloat16', 'float16'], + loss_scale=[None, 'dynamic', 128, 256], + )) + def test_configure_optimizer(self, mixed_precision_dtype, loss_scale): + config = cfg.ExperimentConfig( + task=cfg.TaskConfig( + model=bert.PretrainerConfig()), + runtime=cfg.RuntimeConfig( + mixed_precision_dtype=mixed_precision_dtype, loss_scale=loss_scale), + trainer=trainer_lib.ProgressiveTrainerConfig( + export_checkpoint=True, + export_checkpoint_interval=1, + export_only_final_stage_ckpt=False)) + task = TestPolicy(None, config.task) + trainer = trainer_lib.ProgressiveTrainer(config, task, self.get_temp_dir()) + if mixed_precision_dtype != 'float16': + self.assertIsInstance(trainer.optimizer, tf.keras.optimizers.SGD) + elif mixed_precision_dtype == 'float16' and loss_scale is None: + self.assertIsInstance(trainer.optimizer, tf.keras.optimizers.SGD) + + metrics = trainer.train(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertIn('training_loss', metrics) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/utils.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..192170cb87825de6972ab4a85a6b556ee40600c4 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/fast_training/progressive/utils.py @@ -0,0 +1,56 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Util classes and functions.""" + +from absl import logging +import tensorflow as tf + +# pylint: disable=g-direct-tensorflow-import +from tensorflow.python.training.tracking import tracking + + +class VolatileTrackable(tracking.AutoTrackable): + """A util class to keep Trackables that might change instances.""" + + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + + def reassign_trackable(self, **kwargs): + for k, v in kwargs.items(): + delattr(self, k) # untrack this object + setattr(self, k, v) # track the new object + + +class CheckpointWithHooks(tf.train.Checkpoint): + """Same as tf.train.Checkpoint but supports hooks. + + In progressive training, use this class instead of tf.train.Checkpoint. + + Since the network architecture changes during progressive training, we need to + prepare something (like switch to the correct architecture) before loading the + checkpoint. This class supports a hook that will be executed before checkpoint + loading. + """ + + def __init__(self, before_load_hook, **kwargs): + self._before_load_hook = before_load_hook + super(CheckpointWithHooks, self).__init__(**kwargs) + + # override + def read(self, save_path, options=None): + self._before_load_hook(save_path) + logging.info('Ran before_load_hook.') + super(CheckpointWithHooks, self).read(save_path=save_path, options=options) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/grad_utils.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/grad_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1113d39d5e6f19c9c8fba9e8d8b5c3f99e4e6fba --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/grad_utils.py @@ -0,0 +1,151 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Some gradient util functions to help users writing custom training loop.""" + +from absl import logging + +import tensorflow as tf + + +def _filter_grads(grads_and_vars): + """Filter out iterable with grad equal to None.""" + grads_and_vars = tuple(grads_and_vars) + if not grads_and_vars: + return grads_and_vars + filtered = [] + vars_with_empty_grads = [] + for grad, var in grads_and_vars: + if grad is None: + vars_with_empty_grads.append(var) + else: + filtered.append((grad, var)) + filtered = tuple(filtered) + if not filtered: + raise ValueError("No gradients provided for any variable: %s." % + ([v.name for _, v in grads_and_vars],)) + if vars_with_empty_grads: + logging.warning( + ("Gradients do not exist for variables %s when minimizing the loss."), + ([v.name for v in vars_with_empty_grads])) + return filtered + + +def _filter_and_allreduce_gradients(grads_and_vars, + allreduce_precision="float32", + bytes_per_pack=0): + """Filter None grads and then allreduce gradients in specified precision. + + This utils function is used when users intent to explicitly allreduce + gradients and customize gradients operations before and after allreduce. + The allreduced gradients are then passed to optimizer.apply_gradients( + experimental_aggregate_gradients=False). + + Args: + grads_and_vars: gradients and variables pairs. + allreduce_precision: Whether to allreduce gradients in float32 or float16. + bytes_per_pack: A non-negative integer. Breaks collective operations into + packs of certain size. If it's zero, all gradients are in one pack. + + Returns: + pairs of allreduced non-None gradients and variables. + """ + filtered_grads_and_vars = _filter_grads(grads_and_vars) + (grads, variables) = zip(*filtered_grads_and_vars) + if allreduce_precision == "float16": + grads = [tf.cast(grad, "float16") for grad in grads] + hints = tf.distribute.experimental.CommunicationOptions( + bytes_per_pack=bytes_per_pack) + allreduced_grads = tf.distribute.get_strategy( # pylint: disable=protected-access + ).extended._replica_ctx_all_reduce(tf.distribute.ReduceOp.SUM, grads, hints) + if allreduce_precision == "float16": + allreduced_grads = [tf.cast(grad, "float32") for grad in allreduced_grads] + return allreduced_grads, variables + + +def _run_callbacks(callbacks, grads_and_vars): + for callback in callbacks: + grads_and_vars = callback(grads_and_vars) + return grads_and_vars + + +def minimize_using_explicit_allreduce(tape, + optimizer, + loss, + trainable_variables, + pre_allreduce_callbacks=None, + post_allreduce_callbacks=None, + allreduce_bytes_per_pack=0): + """Minimizes loss for one step by updating `trainable_variables`. + + Minimizes loss for one step by updating `trainable_variables`. + This explicitly performs gradient allreduce, instead of relying on implicit + allreduce in optimizer.apply_gradients(). If training using FP16 mixed + precision, explicit allreduce will aggregate gradients in FP16 format. + For TPU and GPU training using FP32, explicit allreduce will aggregate + gradients in FP32 format. + + Args: + tape: An instance of `tf.GradientTape`. + optimizer: An instance of `tf.keras.optimizers.Optimizer`. + loss: the loss tensor. + trainable_variables: A list of model Variables. + pre_allreduce_callbacks: A list of callback functions that takes gradients + and model variables pairs as input, manipulate them, and returns a new + gradients and model variables pairs. The callback functions will be + invoked in the list order and before gradients are allreduced. With + mixed precision training, the pre_allreduce_allbacks will be applied on + scaled_gradients. Default is no callbacks. + post_allreduce_callbacks: A list of callback functions that takes + gradients and model variables pairs as input, manipulate them, and + returns a new gradients and model variables paris. The callback + functions will be invoked in the list order and right before gradients + are applied to variables for updates. Default is no callbacks. + allreduce_bytes_per_pack: A non-negative integer. Breaks collective + operations into packs of certain size. If it's zero, all gradients are + in one pack. + """ + if isinstance(optimizer, + tf.keras.mixed_precision.LossScaleOptimizer): + # FP16 GPU code path + with tape: + scaled_loss = optimizer.get_scaled_loss(loss) + scaled_grads = tape.gradient(scaled_loss, trainable_variables) + grads_and_vars = zip(scaled_grads, trainable_variables) + if pre_allreduce_callbacks: + grads_and_vars = _run_callbacks(pre_allreduce_callbacks, grads_and_vars) + (allreduced_scaled_grads, + filtered_training_vars) = _filter_and_allreduce_gradients( + grads_and_vars, + allreduce_precision="float16", + bytes_per_pack=allreduce_bytes_per_pack) + allreduced_unscaled_grads = optimizer.get_unscaled_gradients( + allreduced_scaled_grads) + grads_and_vars = zip(allreduced_unscaled_grads, filtered_training_vars) + else: + # TPU or FP32 GPU code path + grads = tape.gradient(loss, trainable_variables) + grads_and_vars = zip(grads, trainable_variables) + if pre_allreduce_callbacks: + grads_and_vars = _run_callbacks(pre_allreduce_callbacks, grads_and_vars) + (allreduced_grads, + filtered_training_vars) = _filter_and_allreduce_gradients( + grads_and_vars, + allreduce_precision="float32", + bytes_per_pack=allreduce_bytes_per_pack) + grads_and_vars = zip(allreduced_grads, filtered_training_vars) + if post_allreduce_callbacks: + grads_and_vars = _run_callbacks(post_allreduce_callbacks, grads_and_vars) + optimizer.apply_gradients( + grads_and_vars, experimental_aggregate_gradients=False) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/hyperparams/__init__.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/hyperparams/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bcbc0aedd3d6013c14c641d9e61a0a717f188ec5 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/hyperparams/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Hyperparams package definition.""" +# pylint: disable=g-multiple-import +from official.modeling.hyperparams.base_config import * +from official.modeling.hyperparams.oneof import * +from official.modeling.hyperparams.params_dict import * + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/hyperparams/base_config.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/hyperparams/base_config.py new file mode 100644 index 0000000000000000000000000000000000000000..f0afca0909eb0a63d54ae83b4d5fc44515a30c1e --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/hyperparams/base_config.py @@ -0,0 +1,306 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Base configurations to standardize experiments.""" +import copy +import dataclasses +import functools +import inspect +from typing import Any, List, Mapping, Optional, Type + +from absl import logging +import tensorflow as tf +import yaml + +from official.modeling.hyperparams import params_dict + +_BOUND = set() + + +def bind(config_cls): + """Bind a class to config cls.""" + if not inspect.isclass(config_cls): + raise ValueError('The bind decorator is supposed to apply on the class ' + f'attribute. Received {config_cls}, not a class.') + + def decorator(builder): + if config_cls in _BOUND: + raise ValueError('Inside a program, we should not bind the config with a' + ' class twice.') + if inspect.isclass(builder): + config_cls._BUILDER = builder # pylint: disable=protected-access + elif inspect.isfunction(builder): + + def _wrapper(self, *args, **kwargs): # pylint: disable=unused-argument + return builder(*args, **kwargs) + + config_cls._BUILDER = _wrapper # pylint: disable=protected-access + else: + raise ValueError(f'The `BUILDER` type is not supported: {builder}') + _BOUND.add(config_cls) + return builder + + return decorator + + +@dataclasses.dataclass +class Config(params_dict.ParamsDict): + """The base configuration class that supports YAML/JSON based overrides. + + Because of YAML/JSON serialization limitations, some semantics of dataclass + are not supported: + * It recursively enforces a allowlist of basic types and container types, so + it avoids surprises with copy and reuse caused by unanticipated types. + * Warning: it converts Dict to `Config` even within sequences, + e.g. for config = Config({'key': [([{'a': 42}],)]), + type(config.key[0][0][0]) is Config rather than dict. + If you define/annotate some field as Dict, the field will convert to a + `Config` instance and lose the dictionary type. + """ + # The class or method to bind with the params class. + _BUILDER = None + # It's safe to add bytes and other immutable types here. + IMMUTABLE_TYPES = (str, int, float, bool, type(None)) + # It's safe to add set, frozenset and other collections here. + SEQUENCE_TYPES = (list, tuple) + + default_params: dataclasses.InitVar[Optional[Mapping[str, Any]]] = None + restrictions: dataclasses.InitVar[Optional[List[str]]] = None + + def __post_init__(self, default_params, restrictions): + super().__init__( + default_params=default_params, + restrictions=restrictions) + + @property + def BUILDER(self): + return self._BUILDER + + @classmethod + def _isvalidsequence(cls, v): + """Check if the input values are valid sequences. + + Args: + v: Input sequence. + + Returns: + True if the sequence is valid. Valid sequence includes the sequence + type in cls.SEQUENCE_TYPES and element type is in cls.IMMUTABLE_TYPES or + is dict or ParamsDict. + """ + if not isinstance(v, cls.SEQUENCE_TYPES): + return False + return (all(isinstance(e, cls.IMMUTABLE_TYPES) for e in v) or + all(isinstance(e, dict) for e in v) or + all(isinstance(e, params_dict.ParamsDict) for e in v)) + + @classmethod + def _import_config(cls, v, subconfig_type): + """Returns v with dicts converted to Configs, recursively.""" + if not issubclass(subconfig_type, params_dict.ParamsDict): + raise TypeError( + 'Subconfig_type should be subclass of ParamsDict, found {!r}'.format( + subconfig_type)) + if isinstance(v, cls.IMMUTABLE_TYPES): + return v + elif isinstance(v, cls.SEQUENCE_TYPES): + # Only support one layer of sequence. + if not cls._isvalidsequence(v): + raise TypeError( + 'Invalid sequence: only supports single level {!r} of {!r} or ' + 'dict or ParamsDict found: {!r}'.format(cls.SEQUENCE_TYPES, + cls.IMMUTABLE_TYPES, v)) + import_fn = functools.partial( + cls._import_config, subconfig_type=subconfig_type) + return type(v)(map(import_fn, v)) + elif isinstance(v, params_dict.ParamsDict): + # Deepcopy here is a temporary solution for preserving type in nested + # Config object. + return copy.deepcopy(v) + elif isinstance(v, dict): + return subconfig_type(v) + else: + raise TypeError('Unknown type: {!r}'.format(type(v))) + + @classmethod + def _export_config(cls, v): + """Returns v with Configs converted to dicts, recursively.""" + if isinstance(v, cls.IMMUTABLE_TYPES): + return v + elif isinstance(v, cls.SEQUENCE_TYPES): + return type(v)(map(cls._export_config, v)) + elif isinstance(v, params_dict.ParamsDict): + return v.as_dict() + elif isinstance(v, dict): + raise TypeError('dict value not supported in converting.') + else: + raise TypeError('Unknown type: {!r}'.format(type(v))) + + @classmethod + def _get_subconfig_type(cls, k) -> Type[params_dict.ParamsDict]: + """Get element type by the field name. + + Args: + k: the key/name of the field. + + Returns: + Config as default. If a type annotation is found for `k`, + 1) returns the type of the annotation if it is subtype of ParamsDict; + 2) returns the element type if the annotation of `k` is List[SubType] + or Tuple[SubType]. + """ + subconfig_type = Config + if k in cls.__annotations__: + # Directly Config subtype. + type_annotation = cls.__annotations__[k] # pytype: disable=invalid-annotation + if (isinstance(type_annotation, type) and + issubclass(type_annotation, Config)): + subconfig_type = cls.__annotations__[k] # pytype: disable=invalid-annotation + else: + # Check if the field is a sequence of subtypes. + field_type = getattr(type_annotation, '__origin__', type(None)) + if (isinstance(field_type, type) and + issubclass(field_type, cls.SEQUENCE_TYPES)): + element_type = getattr(type_annotation, '__args__', [type(None)])[0] + subconfig_type = ( + element_type if issubclass(element_type, params_dict.ParamsDict) + else subconfig_type) + return subconfig_type + + def _set(self, k, v): + """Overrides same method in ParamsDict. + + Also called by ParamsDict methods. + + Args: + k: key to set. + v: value. + + Raises: + RuntimeError + """ + subconfig_type = self._get_subconfig_type(k) + + def is_null(k): + if k not in self.__dict__ or not self.__dict__[k]: + return True + return False + + if isinstance(v, dict): + if is_null(k): + # If the key not exist or the value is None, a new Config-family object + # sould be created for the key. + self.__dict__[k] = subconfig_type(v) + else: + self.__dict__[k].override(v) + elif not is_null(k) and isinstance(v, self.SEQUENCE_TYPES) and all( + [not isinstance(e, self.IMMUTABLE_TYPES) for e in v]): + if len(self.__dict__[k]) == len(v): + for i in range(len(v)): + self.__dict__[k][i].override(v[i]) + elif not all([isinstance(e, self.IMMUTABLE_TYPES) for e in v]): + logging.warning( + "The list/tuple don't match the value dictionaries provided. Thus, " + 'the list/tuple is determined by the type annotation and ' + 'values provided. This is error-prone.') + self.__dict__[k] = self._import_config(v, subconfig_type) + else: + self.__dict__[k] = self._import_config(v, subconfig_type) + else: + self.__dict__[k] = self._import_config(v, subconfig_type) + + def __setattr__(self, k, v): + if k == 'BUILDER' or k == '_BUILDER': + raise AttributeError('`BUILDER` is a property and `_BUILDER` is the ' + 'reserved class attribute. We should only assign ' + '`_BUILDER` at the class level.') + + if k not in self.RESERVED_ATTR: + if getattr(self, '_locked', False): + raise ValueError('The Config has been locked. ' 'No change is allowed.') + self._set(k, v) + + def _override(self, override_dict, is_strict=True): + """Overrides same method in ParamsDict. + + Also called by ParamsDict methods. + + Args: + override_dict: dictionary to write to . + is_strict: If True, not allows to add new keys. + + Raises: + KeyError: overriding reserved keys or keys not exist (is_strict=True). + """ + for k, v in sorted(override_dict.items()): + if k in self.RESERVED_ATTR: + raise KeyError('The key {!r} is internally reserved. ' + 'Can not be overridden.'.format(k)) + if k not in self.__dict__: + if is_strict: + raise KeyError('The key {!r} does not exist in {!r}. ' + 'To extend the existing keys, use ' + '`override` with `is_strict` = False.'.format( + k, type(self))) + else: + self._set(k, v) + else: + if isinstance(v, dict) and self.__dict__[k]: + self.__dict__[k]._override(v, is_strict) # pylint: disable=protected-access + elif isinstance(v, params_dict.ParamsDict) and self.__dict__[k]: + self.__dict__[k]._override(v.as_dict(), is_strict) # pylint: disable=protected-access + else: + self._set(k, v) + + def as_dict(self): + """Returns a dict representation of params_dict.ParamsDict. + + For the nested params_dict.ParamsDict, a nested dict will be returned. + """ + return { + k: self._export_config(v) + for k, v in self.__dict__.items() + if k not in self.RESERVED_ATTR + } + + def replace(self, **kwargs): + """Overrides/returns a unlocked copy with the current config unchanged.""" + # pylint: disable=protected-access + params = copy.deepcopy(self) + params._locked = False + params._override(kwargs, is_strict=True) + # pylint: enable=protected-access + return params + + @classmethod + def from_yaml(cls, file_path: str): + # Note: This only works if the Config has all default values. + with tf.io.gfile.GFile(file_path, 'r') as f: + loaded = yaml.load(f, Loader=yaml.FullLoader) + config = cls() + config.override(loaded) + return config + + @classmethod + def from_json(cls, file_path: str): + """Wrapper for `from_yaml`.""" + return cls.from_yaml(file_path) + + @classmethod + def from_args(cls, *args, **kwargs): + """Builds a config from the given list of arguments.""" + attributes = list(cls.__annotations__.keys()) + default_params = {a: p for a, p in zip(attributes, args)} + default_params.update(kwargs) + return cls(default_params=default_params) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/hyperparams/base_config_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/hyperparams/base_config_test.py new file mode 100644 index 0000000000000000000000000000000000000000..21d0aaa1c3ee56884505e4ab5f72bc0212ceb74d --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/hyperparams/base_config_test.py @@ -0,0 +1,385 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pprint +from typing import List, Tuple + +from absl.testing import parameterized +import dataclasses +import tensorflow as tf +from official.modeling.hyperparams import base_config + + +@dataclasses.dataclass +class DumpConfig1(base_config.Config): + a: int = 1 + b: str = 'text' + + +@dataclasses.dataclass +class DumpConfig2(base_config.Config): + c: int = 2 + d: str = 'text' + e: DumpConfig1 = DumpConfig1() + + +@dataclasses.dataclass +class DumpConfig3(DumpConfig2): + f: int = 2 + g: str = 'text' + h: List[DumpConfig1] = dataclasses.field( + default_factory=lambda: [DumpConfig1(), DumpConfig1()]) + g: Tuple[DumpConfig1, ...] = (DumpConfig1(),) + + +@dataclasses.dataclass +class DumpConfig4(DumpConfig2): + x: int = 3 + + +@dataclasses.dataclass +class DummyConfig5(base_config.Config): + y: Tuple[DumpConfig2, ...] = (DumpConfig2(), DumpConfig4()) + z: Tuple[str] = ('a',) + + +class BaseConfigTest(parameterized.TestCase, tf.test.TestCase): + + def assertHasSameTypes(self, c, d, msg=''): + """Checks if a Config has the same structure as a given dict. + + Args: + c: the Config object to be check. + d: the reference dict object. + msg: The error message to show when type mismatched. + """ + # Make sure d is not a Config. Assume d is either + # dictionary or primitive type and c is the Config or primitive types. + self.assertNotIsInstance(d, base_config.Config) + if isinstance(d, base_config.Config.IMMUTABLE_TYPES): + self.assertEqual(pprint.pformat(c), pprint.pformat(d), msg=msg) + elif isinstance(d, base_config.Config.SEQUENCE_TYPES): + self.assertEqual(type(c), type(d), msg=msg) + for i, v in enumerate(d): + self.assertHasSameTypes(c[i], v, msg='{}[{!r}]'.format(msg, i)) + elif isinstance(d, dict): + self.assertIsInstance(c, base_config.Config, msg=msg) + for k, v in sorted(d.items()): + self.assertHasSameTypes(getattr(c, k), v, msg='{}[{!r}]'.format(msg, k)) + else: + raise TypeError('Unknown type: %r' % type(d)) + + def assertImportExport(self, v): + config = base_config.Config({'key': v}) + back = config.as_dict()['key'] + self.assertEqual(pprint.pformat(back), pprint.pformat(v)) + self.assertHasSameTypes(config.key, v, msg='=%s v' % pprint.pformat(v)) + + def test_invalid_keys(self): + params = base_config.Config() + with self.assertRaises(AttributeError): + _ = params.a + + def test_cls(self): + params = base_config.Config() + with self.assertRaisesRegex( + AttributeError, + '`BUILDER` is a property and `_BUILDER` is the reserved'): + params.BUILDER = DumpConfig2 + with self.assertRaisesRegex( + AttributeError, + '`BUILDER` is a property and `_BUILDER` is the reserved'): + params._BUILDER = DumpConfig2 + + base_config.bind(DumpConfig1)(DumpConfig2) + params = DumpConfig1() + self.assertEqual(params.BUILDER, DumpConfig2) + with self.assertRaisesRegex(ValueError, + 'Inside a program, we should not bind'): + base_config.bind(DumpConfig1)(DumpConfig2) + + def _test(): + return 'test' + + base_config.bind(DumpConfig2)(_test) + params = DumpConfig2() + self.assertEqual(params.BUILDER(), 'test') + + def test_nested_config_types(self): + config = DumpConfig3() + self.assertIsInstance(config.e, DumpConfig1) + self.assertIsInstance(config.h[0], DumpConfig1) + self.assertIsInstance(config.h[1], DumpConfig1) + self.assertIsInstance(config.g[0], DumpConfig1) + + config.override({'e': {'a': 2, 'b': 'new text'}}) + self.assertIsInstance(config.e, DumpConfig1) + self.assertEqual(config.e.a, 2) + self.assertEqual(config.e.b, 'new text') + + config.override({'h': [{'a': 3, 'b': 'new text 2'}]}) + self.assertIsInstance(config.h[0], DumpConfig1) + self.assertLen(config.h, 1) + self.assertEqual(config.h[0].a, 3) + self.assertEqual(config.h[0].b, 'new text 2') + + config.override({'g': [{'a': 4, 'b': 'new text 3'}]}) + self.assertIsInstance(config.g[0], DumpConfig1) + self.assertLen(config.g, 1) + self.assertEqual(config.g[0].a, 4) + self.assertEqual(config.g[0].b, 'new text 3') + + def test_replace(self): + config = DumpConfig2() + new_config = config.replace(e={'a': 2}) + self.assertEqual(new_config.e.a, 2) + self.assertIsInstance(new_config.e, DumpConfig1) + + config = DumpConfig2(e=DumpConfig2()) + new_config = config.replace(e={'c': 4}) + self.assertEqual(new_config.e.c, 4) + self.assertIsInstance(new_config.e, DumpConfig2) + + config = DumpConfig3() + new_config = config.replace(g=[{'a': 4, 'b': 'new text 3'}]) + self.assertIsInstance(new_config.g[0], DumpConfig1) + self.assertEqual(new_config.g[0].a, 4) + + @parameterized.parameters( + ('_locked', "The key '_locked' is internally reserved."), + ('_restrictions', "The key '_restrictions' is internally reserved."), + ('aa', "The key 'aa' does not exist."), + ) + def test_key_error(self, key, msg): + params = base_config.Config() + with self.assertRaisesRegex(KeyError, msg): + params.override({key: True}) + + @parameterized.parameters( + ('str data',), + (123,), + (1.23,), + (None,), + (['str', 1, 2.3, None],), + (('str', 1, 2.3, None),), + ) + def test_import_export_immutable_types(self, v): + self.assertImportExport(v) + out = base_config.Config({'key': v}) + self.assertEqual(pprint.pformat(v), pprint.pformat(out.key)) + + def test_override_is_strict_true(self): + params = base_config.Config({ + 'a': 'aa', + 'b': 2, + 'c': { + 'c1': 'cc', + 'c2': 20 + } + }) + params.override({'a': 2, 'c': {'c1': 'ccc'}}, is_strict=True) + self.assertEqual(params.a, 2) + self.assertEqual(params.c.c1, 'ccc') + with self.assertRaises(KeyError): + params.override({'d': 'ddd'}, is_strict=True) + with self.assertRaises(KeyError): + params.override({'c': {'c3': 30}}, is_strict=True) + + config = base_config.Config({'key': [{'a': 42}]}) + with self.assertRaisesRegex(KeyError, "The key 'b' does not exist"): + config.override({'key': [{'b': 43}]}) + + @parameterized.parameters( + (lambda x: x, 'Unknown type'), + (object(), 'Unknown type'), + (set(), 'Unknown type'), + (frozenset(), 'Unknown type'), + ) + def test_import_unsupport_types(self, v, msg): + with self.assertRaisesRegex(TypeError, msg): + _ = base_config.Config({'key': v}) + + @parameterized.parameters( + ({ + 'a': [{ + 'b': 2, + }, { + 'c': 3, + }] + },), + ({ + 'c': [{ + 'f': 1.1, + }, { + 'h': [1, 2], + }] + },), + (({ + 'a': 'aa', + 'b': 2, + 'c': { + 'c1': 10, + 'c2': 20, + } + },),), + ) + def test_import_export_nested_structure(self, d): + self.assertImportExport(d) + + @parameterized.parameters( + ([{ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + }],), + (({ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + },),), + ) + def test_import_export_nested_sequences(self, v): + self.assertImportExport(v) + + @parameterized.parameters( + ([([{}],)],), + ([['str', 1, 2.3, None]],), + ((('str', 1, 2.3, None),),), + ([ + ('str', 1, 2.3, None), + ],), + ([ + ('str', 1, 2.3, None), + ],), + ([[{ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + }]],), + ([[[{ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + }]]],), + ((({ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + },),),), + (((({ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + },),),),), + ([({ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + },)],), + (([{ + 'a': 42, + 'b': 'hello', + 'c': 1.2 + }],),), + ) + def test_import_export_unsupport_sequence(self, v): + with self.assertRaisesRegex(TypeError, + 'Invalid sequence: only supports single level'): + _ = base_config.Config({'key': v}) + + def test_construct_subtype(self): + pass + + def test_import_config(self): + params = base_config.Config({'a': [{'b': 2}, {'c': {'d': 3}}]}) + self.assertLen(params.a, 2) + self.assertEqual(params.a[0].b, 2) + self.assertEqual(type(params.a[0]), base_config.Config) + self.assertEqual(pprint.pformat(params.a[0].b), '2') + self.assertEqual(type(params.a[1]), base_config.Config) + self.assertEqual(type(params.a[1].c), base_config.Config) + self.assertEqual(pprint.pformat(params.a[1].c.d), '3') + + def test_override(self): + params = base_config.Config({'a': [{'b': 2}, {'c': {'d': 3}}]}) + params.override({'a': [{'b': 4}, {'c': {'d': 5}}]}, is_strict=False) + self.assertEqual(type(params.a), list) + self.assertEqual(type(params.a[0]), base_config.Config) + self.assertEqual(pprint.pformat(params.a[0].b), '4') + self.assertEqual(type(params.a[1]), base_config.Config) + self.assertEqual(type(params.a[1].c), base_config.Config) + self.assertEqual(pprint.pformat(params.a[1].c.d), '5') + + @parameterized.parameters( + ([{}],), + (({},),), + ) + def test_config_vs_params_dict(self, v): + d = {'key': v} + self.assertEqual(type(base_config.Config(d).key[0]), base_config.Config) + self.assertEqual(type(base_config.params_dict.ParamsDict(d).key[0]), dict) + + def test_ppformat(self): + self.assertEqual( + pprint.pformat([ + 's', 1, 1.0, True, None, {}, [], (), { + (2,): (3, [4], { + 6: 7, + }), + 8: 9, + } + ]), + "['s', 1, 1.0, True, None, {}, [], (), {8: 9, (2,): (3, [4], {6: 7})}]") + + def test_with_restrictions(self): + restrictions = ['e.a[a-zA-Z][\w\.]*) # variable name: "var" or "x" + \s*=\s* + ((?P\'(.*?)\' # single quote + | + \"(.*?)\" # double quote + | + [^,\[]* # single value + | + \[[^\]]*\])) # list of values + ($|,\s*)""", re.VERBOSE) + +_CONST_VALUE_RE = re.compile(r'(\d.*|-\d.*|None)') + +# Yaml loader with an implicit resolver to parse float decimal and exponential +# format. The regular experission parse the following cases: +# 1- Decimal number with an optional exponential term. +# 2- Integer number with an exponential term. +# 3- Decimal number with an optional exponential term. +# 4- Decimal number. + +LOADER = yaml.SafeLoader +LOADER.add_implicit_resolver( + 'tag:yaml.org,2002:float', + re.compile(r''' + ^(?:[-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)? + | + [-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+) + | + \\.[0-9_]+(?:[eE][-+][0-9]+)? + | + [-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*)$''', re.X), + list('-+0123456789.')) + + +class ParamsDict(object): + """A hyperparameter container class.""" + + RESERVED_ATTR = ['_locked', '_restrictions'] + + def __init__(self, default_params=None, restrictions=None): + """Instantiate a ParamsDict. + + Instantiate a ParamsDict given a set of default parameters and a list of + restrictions. Upon initialization, it validates itself by checking all the + defined restrictions, and raise error if it finds inconsistency. + + Args: + default_params: a Python dict or another ParamsDict object including the + default parameters to initialize. + restrictions: a list of strings, which define a list of restrictions to + ensure the consistency of different parameters internally. Each + restriction string is defined as a binary relation with a set of + operators, including {'==', '!=', '<', '<=', '>', '>='}. + """ + self._locked = False + self._restrictions = [] + if restrictions: + self._restrictions = restrictions + if default_params is None: + default_params = {} + self.override(default_params, is_strict=False) + + def _set(self, k, v): + if isinstance(v, dict): + self.__dict__[k] = ParamsDict(v) + else: + self.__dict__[k] = copy.deepcopy(v) + + def __setattr__(self, k, v): + """Sets the value of the existing key. + + Note that this does not allow directly defining a new key. Use the + `override` method with `is_strict=False` instead. + + Args: + k: the key string. + v: the value to be used to set the key `k`. + + Raises: + KeyError: if k is not defined in the ParamsDict. + """ + if k not in ParamsDict.RESERVED_ATTR: + if k not in self.__dict__.keys(): + raise KeyError('The key `%{}` does not exist. ' + 'To extend the existing keys, use ' + '`override` with `is_strict` = True.'.format(k)) + if self._locked: + raise ValueError('The ParamsDict has been locked. ' + 'No change is allowed.') + self._set(k, v) + + def __getattr__(self, k): + """Gets the value of the existing key. + + Args: + k: the key string. + + Returns: + the value of the key. + + Raises: + AttributeError: if k is not defined in the ParamsDict. + """ + if k not in self.__dict__.keys(): + raise AttributeError('The key `{}` does not exist. '.format(k)) + return self.__dict__[k] + + def __contains__(self, key): + """Implements the membership test operator.""" + return key in self.__dict__ + + def get(self, key, value=None): + """Accesses through built-in dictionary get method.""" + return self.__dict__.get(key, value) + + def __delattr__(self, k): + """Deletes the key and removes its values. + + Args: + k: the key string. + + Raises: + AttributeError: if k is reserverd or not defined in the ParamsDict. + ValueError: if the ParamsDict instance has been locked. + """ + if k in ParamsDict.RESERVED_ATTR: + raise AttributeError( + 'The key `{}` is reserved. No change is allowes. '.format(k)) + if k not in self.__dict__.keys(): + raise AttributeError('The key `{}` does not exist. '.format(k)) + if self._locked: + raise ValueError('The ParamsDict has been locked. No change is allowed.') + del self.__dict__[k] + + def override(self, override_params, is_strict=True): + """Override the ParamsDict with a set of given params. + + Args: + override_params: a dict or a ParamsDict specifying the parameters to be + overridden. + is_strict: a boolean specifying whether override is strict or not. If + True, keys in `override_params` must be present in the ParamsDict. If + False, keys in `override_params` can be different from what is currently + defined in the ParamsDict. In this case, the ParamsDict will be extended + to include the new keys. + """ + if self._locked: + raise ValueError('The ParamsDict has been locked. No change is allowed.') + if isinstance(override_params, ParamsDict): + override_params = override_params.as_dict() + self._override(override_params, is_strict) # pylint: disable=protected-access + + def _override(self, override_dict, is_strict=True): + """The implementation of `override`.""" + for k, v in six.iteritems(override_dict): + if k in ParamsDict.RESERVED_ATTR: + raise KeyError('The key `%{}` is internally reserved. ' + 'Can not be overridden.') + if k not in self.__dict__.keys(): + if is_strict: + raise KeyError('The key `{}` does not exist. ' + 'To extend the existing keys, use ' + '`override` with `is_strict` = False.'.format(k)) + else: + self._set(k, v) + else: + if isinstance(v, dict): + self.__dict__[k]._override(v, is_strict) # pylint: disable=protected-access + elif isinstance(v, ParamsDict): + self.__dict__[k]._override(v.as_dict(), is_strict) # pylint: disable=protected-access + else: + self.__dict__[k] = copy.deepcopy(v) + + def lock(self): + """Makes the ParamsDict immutable.""" + self._locked = True + + def as_dict(self): + """Returns a dict representation of ParamsDict. + + For the nested ParamsDict, a nested dict will be returned. + """ + params_dict = {} + for k, v in six.iteritems(self.__dict__): + if k not in ParamsDict.RESERVED_ATTR: + if isinstance(v, ParamsDict): + params_dict[k] = v.as_dict() + else: + params_dict[k] = copy.deepcopy(v) + return params_dict + + def validate(self): + """Validate the parameters consistency based on the restrictions. + + This method validates the internal consistency using the pre-defined list of + restrictions. A restriction is defined as a string which specfiies a binary + operation. The supported binary operations are {'==', '!=', '<', '<=', '>', + '>='}. Note that the meaning of these operators are consistent with the + underlying Python immplementation. Users should make sure the define + restrictions on their type make sense. + + For example, for a ParamsDict like the following + ``` + a: + a1: 1 + a2: 2 + b: + bb: + bb1: 10 + bb2: 20 + ccc: + a1: 1 + a3: 3 + ``` + one can define two restrictions like this + ['a.a1 == b.ccc.a1', 'a.a2 <= b.bb.bb2'] + + What it enforces are: + - a.a1 = 1 == b.ccc.a1 = 1 + - a.a2 = 2 <= b.bb.bb2 = 20 + + Raises: + KeyError: if any of the following happens + (1) any of parameters in any of restrictions is not defined in + ParamsDict, + (2) any inconsistency violating the restriction is found. + ValueError: if the restriction defined in the string is not supported. + """ + + def _get_kv(dotted_string, params_dict): + """Get keys and values indicated by dotted_string.""" + if _CONST_VALUE_RE.match(dotted_string) is not None: + const_str = dotted_string + if const_str == 'None': + constant = None + else: + constant = float(const_str) + return None, constant + else: + tokenized_params = dotted_string.split('.') + v = params_dict + for t in tokenized_params: + v = v[t] + return tokenized_params[-1], v + + def _get_kvs(tokens, params_dict): + if len(tokens) != 2: + raise ValueError('Only support binary relation in restriction.') + stripped_tokens = [t.strip() for t in tokens] + left_k, left_v = _get_kv(stripped_tokens[0], params_dict) + right_k, right_v = _get_kv(stripped_tokens[1], params_dict) + return left_k, left_v, right_k, right_v + + params_dict = self.as_dict() + for restriction in self._restrictions: + if '==' in restriction: + tokens = restriction.split('==') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v != right_v: + raise KeyError( + 'Found inconsistncy between key `{}` and key `{}`.'.format( + tokens[0], tokens[1])) + elif '!=' in restriction: + tokens = restriction.split('!=') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v == right_v: + raise KeyError( + 'Found inconsistncy between key `{}` and key `{}`.'.format( + tokens[0], tokens[1])) + elif '<' in restriction: + tokens = restriction.split('<') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v >= right_v: + raise KeyError( + 'Found inconsistncy between key `{}` and key `{}`.'.format( + tokens[0], tokens[1])) + elif '<=' in restriction: + tokens = restriction.split('<=') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v > right_v: + raise KeyError( + 'Found inconsistncy between key `{}` and key `{}`.'.format( + tokens[0], tokens[1])) + elif '>' in restriction: + tokens = restriction.split('>') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v <= right_v: + raise KeyError( + 'Found inconsistncy between key `{}` and key `{}`.'.format( + tokens[0], tokens[1])) + elif '>=' in restriction: + tokens = restriction.split('>=') + _, left_v, _, right_v = _get_kvs(tokens, params_dict) + if left_v < right_v: + raise KeyError( + 'Found inconsistncy between key `{}` and key `{}`.'.format( + tokens[0], tokens[1])) + else: + raise ValueError('Unsupported relation in restriction.') + + +def read_yaml_to_params_dict(file_path: str): + """Reads a YAML file to a ParamsDict.""" + with tf.io.gfile.GFile(file_path, 'r') as f: + params_dict = yaml.load(f, Loader=LOADER) + return ParamsDict(params_dict) + + +def save_params_dict_to_yaml(params, file_path): + """Saves the input ParamsDict to a YAML file.""" + with tf.io.gfile.GFile(file_path, 'w') as f: + + def _my_list_rep(dumper, data): + # u'tag:yaml.org,2002:seq' is the YAML internal tag for sequence. + return dumper.represent_sequence( + u'tag:yaml.org,2002:seq', data, flow_style=True) + + yaml.add_representer(list, _my_list_rep) + yaml.dump(params.as_dict(), f, default_flow_style=False) + + +def nested_csv_str_to_json_str(csv_str): + """Converts a nested (using '.') comma-separated k=v string to a JSON string. + + Converts a comma-separated string of key/value pairs that supports + nesting of keys to a JSON string. Nesting is implemented using + '.' between levels for a given key. + + Spacing between commas and = is supported (e.g. there is no difference between + "a=1,b=2", "a = 1, b = 2", or "a=1, b=2") but there should be no spaces before + keys or after values (e.g. " a=1,b=2" and "a=1,b=2 " are not supported). + + Note that this will only support values supported by CSV, meaning + values such as nested lists (e.g. "a=[[1,2,3],[4,5,6]]") are not + supported. Strings are supported as well, e.g. "a='hello'". + + An example conversion would be: + + "a=1, b=2, c.a=2, c.b=3, d.a.a=5" + + to + + "{ a: 1, b : 2, c: {a : 2, b : 3}, d: {a: {a : 5}}}" + + Args: + csv_str: the comma separated string. + + Returns: + the converted JSON string. + + Raises: + ValueError: If csv_str is not in a comma separated string or + if the string is formatted incorrectly. + """ + if not csv_str: + return '' + + formatted_entries = [] + nested_map = collections.defaultdict(list) + pos = 0 + while pos < len(csv_str): + m = _PARAM_RE.match(csv_str, pos) + if not m: + raise ValueError('Malformed hyperparameter value while parsing ' + 'CSV string: %s' % csv_str[pos:]) + pos = m.end() + # Parse the values. + m_dict = m.groupdict() + name = m_dict['name'] + v = m_dict['val'] + + # If a GCS path (e.g. gs://...) is provided, wrap this in quotes + # as yaml.load would otherwise throw an exception + if re.match(r'(?=[^\"\'])(?=[gs://])', v): + v = '\'{}\''.format(v) + + name_nested = name.split('.') + if len(name_nested) > 1: + grouping = name_nested[0] + value = '.'.join(name_nested[1:]) + '=' + v + nested_map[grouping].append(value) + else: + formatted_entries.append('%s : %s' % (name, v)) + + for grouping, value in nested_map.items(): + value = ','.join(value) + value = nested_csv_str_to_json_str(value) + formatted_entries.append('%s : %s' % (grouping, value)) + return '{' + ', '.join(formatted_entries) + '}' + + +def override_params_dict(params, dict_or_string_or_yaml_file, is_strict): + """Override a given ParamsDict using a dict, JSON/YAML/CSV string or YAML file. + + The logic of the function is outlined below: + 1. Test that the input is a dict. If not, proceed to 2. + 2. Tests that the input is a string. If not, raise unknown ValueError + 2.1. Test if the string is in a CSV format. If so, parse. + If not, proceed to 2.2. + 2.2. Try loading the string as a YAML/JSON. If successful, parse to + dict and use it to override. If not, proceed to 2.3. + 2.3. Try using the string as a file path and load the YAML file. + + Args: + params: a ParamsDict object to be overridden. + dict_or_string_or_yaml_file: a Python dict, JSON/YAML/CSV string or path to + a YAML file specifying the parameters to be overridden. + is_strict: a boolean specifying whether override is strict or not. + + Returns: + params: the overridden ParamsDict object. + + Raises: + ValueError: if failed to override the parameters. + """ + if not dict_or_string_or_yaml_file: + return params + if isinstance(dict_or_string_or_yaml_file, dict): + params.override(dict_or_string_or_yaml_file, is_strict) + elif isinstance(dict_or_string_or_yaml_file, six.string_types): + try: + dict_or_string_or_yaml_file = ( + nested_csv_str_to_json_str(dict_or_string_or_yaml_file)) + except ValueError: + pass + params_dict = yaml.load(dict_or_string_or_yaml_file, Loader=LOADER) + if isinstance(params_dict, dict): + params.override(params_dict, is_strict) + else: + with tf.io.gfile.GFile(dict_or_string_or_yaml_file) as f: + params.override(yaml.load(f, Loader=yaml.FullLoader), is_strict) + else: + raise ValueError('Unknown input type to parse.') + return params diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/hyperparams/params_dict_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/hyperparams/params_dict_test.py new file mode 100644 index 0000000000000000000000000000000000000000..248a81652a496266fb9656d40f77e665e8606f10 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/hyperparams/params_dict_test.py @@ -0,0 +1,429 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for params_dict.py.""" + +import os + +import tensorflow as tf +import yaml + +from official.modeling.hyperparams import params_dict + + +class ParamsDictTest(tf.test.TestCase): + + def test_init_from_an_empty_dict(self): + params = params_dict.ParamsDict() + with self.assertRaises(AttributeError): + _ = params.a + + with self.assertRaises(KeyError): + params.a = 'aa' + + def test_init_from_a_dict(self): + params = params_dict.ParamsDict({'a': 'aa', 'b': 2}) + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + + def test_init_from_a_param_dict(self): + params_init = params_dict.ParamsDict({'a': 'aa', 'b': 2}) + params = params_dict.ParamsDict(params_init) + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + + def test_lock(self): + params = params_dict.ParamsDict({'a': 1, 'b': 2, 'c': 3}) + params.lock() + with self.assertRaises(ValueError): + params.a = 10 + with self.assertRaises(ValueError): + params.override({'b': 20}) + with self.assertRaises(ValueError): + del params.c + + def test_setattr(self): + params = params_dict.ParamsDict() + params.override({'a': 'aa', 'b': 2, 'c': None}, is_strict=False) + params.c = 'ccc' + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + self.assertEqual(params.c, 'ccc') + + def test_getattr(self): + params = params_dict.ParamsDict() + params.override({'a': 'aa', 'b': 2, 'c': None}, is_strict=False) + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + self.assertEqual(params.c, None) + + def test_delattr(self): + params = params_dict.ParamsDict() + params.override({ + 'a': 'aa', + 'b': 2, + 'c': None, + 'd': { + 'd1': 1, + 'd2': 10 + } + }, + is_strict=False) + del params.c + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + with self.assertRaises(AttributeError): + _ = params.c + del params.d + with self.assertRaises(AttributeError): + _ = params.d.d1 + + def test_contains(self): + params = params_dict.ParamsDict() + params.override({'a': 'aa'}, is_strict=False) + self.assertIn('a', params) + self.assertNotIn('b', params) + + def test_get(self): + params = params_dict.ParamsDict() + params.override({'a': 'aa'}, is_strict=False) + self.assertEqual(params.get('a'), 'aa') + self.assertEqual(params.get('b', 2), 2) + self.assertEqual(params.get('b'), None) + + def test_override_is_strict_true(self): + params = params_dict.ParamsDict({ + 'a': 'aa', + 'b': 2, + 'c': { + 'c1': 'cc', + 'c2': 20 + } + }) + params.override({'a': 2, 'c': {'c1': 'ccc'}}, is_strict=True) + self.assertEqual(params.a, 2) + self.assertEqual(params.c.c1, 'ccc') + with self.assertRaises(KeyError): + params.override({'d': 'ddd'}, is_strict=True) + with self.assertRaises(KeyError): + params.override({'c': {'c3': 30}}, is_strict=True) + + def test_override_is_strict_false(self): + params = params_dict.ParamsDict({ + 'a': 'aa', + 'b': 2, + 'c': { + 'c1': 10, + 'c2': 20 + } + }) + params.override({'a': 2, 'c': {'c3': 3000}}, is_strict=False) + self.assertEqual(params.a, 2) + self.assertEqual(params.c.c3, 3000) + params.override({'d': 'ddd'}, is_strict=False) + self.assertEqual(params.d, 'ddd') + params.override({'c': {'c4': 4444}}, is_strict=False) + self.assertEqual(params.c.c4, 4444) + + def test_as_dict(self): + params = params_dict.ParamsDict({ + 'a': 'aa', + 'b': 2, + 'c': { + 'c1': 10, + 'c2': 20 + } + }) + params_d = params.as_dict() + self.assertEqual(params_d['a'], 'aa') + self.assertEqual(params_d['b'], 2) + self.assertEqual(params_d['c']['c1'], 10) + self.assertEqual(params_d['c']['c2'], 20) + + def test_validate(self): + # Raise error due to the unknown parameter. + with self.assertRaises(KeyError): + params = params_dict.ParamsDict({'a': 1, 'b': {'a': 11}}, ['a == c']) + params.validate() + + # OK to check equality of two nested dicts. + params = params_dict.ParamsDict({ + 'a': 1, + 'b': { + 'a': 10 + }, + 'c': { + 'a': 10 + } + }, ['b == c']) + + # Raise error due to inconsistency + with self.assertRaises(KeyError): + params = params_dict.ParamsDict({'a': 1, 'c': {'a': 10}}, ['a == c.a']) + params.validate() + + # Valid rule. + params = params_dict.ParamsDict({'a': 1, 'c': {'a': 1}}, ['a == c.a']) + + # Overridding violates the existing rule, raise error upon validate. + params.override({'a': 11}) + with self.assertRaises(KeyError): + params.validate() + + # Valid restrictions with constant. + params = params_dict.ParamsDict({ + 'a': None, + 'c': { + 'a': 1 + } + }, ['a == None', 'c.a == 1']) + params.validate() + with self.assertRaises(KeyError): + params = params_dict.ParamsDict({ + 'a': 4, + 'c': { + 'a': 1 + } + }, ['a == None', 'c.a == 1']) + params.validate() + + +class ParamsDictIOTest(tf.test.TestCase): + + def write_temp_file(self, filename, text): + temp_file = os.path.join(self.get_temp_dir(), filename) + with tf.io.gfile.GFile(temp_file, 'w') as writer: + writer.write(text) + return temp_file + + def test_save_params_dict_to_yaml(self): + params = params_dict.ParamsDict({ + 'a': 'aa', + 'b': 2, + 'c': { + 'c1': 10, + 'c2': 20 + } + }) + output_yaml_file = os.path.join(self.get_temp_dir(), 'params.yaml') + params_dict.save_params_dict_to_yaml(params, output_yaml_file) + + with tf.io.gfile.GFile(output_yaml_file, 'r') as f: + params_d = yaml.load(f) + self.assertEqual(params.a, params_d['a']) + self.assertEqual(params.b, params_d['b']) + self.assertEqual(params.c.c1, params_d['c']['c1']) + self.assertEqual(params.c.c2, params_d['c']['c2']) + + def test_read_yaml_to_params_dict(self): + input_yaml_file = self.write_temp_file( + 'params.yaml', r""" + a: 'aa' + b: 2 + c: + c1: 10 + c2: 20 + """) + params = params_dict.read_yaml_to_params_dict(input_yaml_file) + + self.assertEqual(params.a, 'aa') + self.assertEqual(params.b, 2) + self.assertEqual(params.c.c1, 10) + self.assertEqual(params.c.c2, 20) + + def test_override_params_dict_using_dict(self): + params = params_dict.ParamsDict({ + 'a': 1, + 'b': 2.5, + 'c': [3, 4], + 'd': 'hello', + 'e': False + }) + override_dict = {'b': 5.2, 'c': [30, 40]} + params = params_dict.override_params_dict( + params, override_dict, is_strict=True) + self.assertEqual(1, params.a) + self.assertEqual(5.2, params.b) + self.assertEqual([30, 40], params.c) + self.assertEqual('hello', params.d) + self.assertEqual(False, params.e) + + def test_override_params_dict_using_yaml_string(self): + params = params_dict.ParamsDict({ + 'a': 1, + 'b': 2.5, + 'c': [3, 4], + 'd': 'hello', + 'e': False + }) + override_yaml_string = "'b': 5.2\n'c': [30, 40]" + params = params_dict.override_params_dict( + params, override_yaml_string, is_strict=True) + self.assertEqual(1, params.a) + self.assertEqual(5.2, params.b) + self.assertEqual([30, 40], params.c) + self.assertEqual('hello', params.d) + self.assertEqual(False, params.e) + + def test_override_params_dict_using_json_string(self): + params = params_dict.ParamsDict({ + 'a': 1, + 'b': { + 'b1': 2, + 'b2': [2, 3], + }, + 'd': { + 'd1': { + 'd2': 'hello' + } + }, + 'e': False + }) + override_json_string = "{ b: { b2: [3, 4] }, d: { d1: { d2: 'hi' } } }" + params = params_dict.override_params_dict( + params, override_json_string, is_strict=True) + self.assertEqual(1, params.a) + self.assertEqual(2, params.b.b1) + self.assertEqual([3, 4], params.b.b2) + self.assertEqual('hi', params.d.d1.d2) + self.assertEqual(False, params.e) + + def test_override_params_dict_using_csv_string(self): + params = params_dict.ParamsDict({ + 'a': 1, + 'b': { + 'b1': 2, + 'b2': [2, 3], + }, + 'd': { + 'd1': { + 'd2': 'hello' + } + }, + 'e': False + }) + override_csv_string = "b.b2=[3,4], d.d1.d2='hi, world', e=gs://test" + params = params_dict.override_params_dict( + params, override_csv_string, is_strict=True) + self.assertEqual(1, params.a) + self.assertEqual(2, params.b.b1) + self.assertEqual([3, 4], params.b.b2) + self.assertEqual('hi, world', params.d.d1.d2) + self.assertEqual('gs://test', params.e) + # Test different float formats + override_csv_string = 'b.b2=-1.e-3, d.d1.d2=+0.001, e=1e+3, a=-1.5E-3' + params = params_dict.override_params_dict( + params, override_csv_string, is_strict=True) + self.assertEqual(-1e-3, params.b.b2) + self.assertEqual(0.001, params.d.d1.d2) + self.assertEqual(1e3, params.e) + self.assertEqual(-1.5e-3, params.a) + + def test_override_params_dict_using_yaml_file(self): + params = params_dict.ParamsDict({ + 'a': 1, + 'b': 2.5, + 'c': [3, 4], + 'd': 'hello', + 'e': False + }) + override_yaml_file = self.write_temp_file( + 'params.yaml', r""" + b: 5.2 + c: [30, 40] + """) + params = params_dict.override_params_dict( + params, override_yaml_file, is_strict=True) + self.assertEqual(1, params.a) + self.assertEqual(5.2, params.b) + self.assertEqual([30, 40], params.c) + self.assertEqual('hello', params.d) + self.assertEqual(False, params.e) + + +class IOTest(tf.test.TestCase): + + def test_basic_csv_str_to_json_str(self): + csv_str = 'a=1,b=2,c=3' + json_str = '{a : 1, b : 2, c : 3}' + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + self.assertEqual(converted_csv_str, json_str) + + def test_basic_csv_str_load(self): + csv_str = 'a=1,b=2,c=3' + expected_output = {'a': 1, 'b': 2, 'c': 3} + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + converted_dict = yaml.load(converted_csv_str) + self.assertDictEqual(converted_dict, expected_output) + + def test_basic_nested_csv_str_to_json_str(self): + csv_str = 'a=1,b.b1=2' + json_str = '{a : 1, b : {b1 : 2}}' + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + self.assertEqual(converted_csv_str, json_str) + + def test_basic_nested_csv_str_load(self): + csv_str = 'a=1,b.b1=2,c.c1=3' + expected_output = {'a': 1, 'b': {'b1': 2}, 'c': {'c1': 3}} + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + converted_dict = yaml.load(converted_csv_str) + self.assertDictEqual(converted_dict, expected_output) + + def test_complex_nested_csv_str_to_json_str(self): + csv_str = 'a.aa.aaa.aaaaa.a=1' + json_str = '{a : {aa : {aaa : {aaaaa : {a : 1}}}}}' + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + self.assertEqual(converted_csv_str, json_str) + + def test_complex_nested_csv_str_load(self): + csv_str = 'a.aa.aaa.aaaaa.a=1,a.a=2' + expected_output = {'a': {'aa': {'aaa': {'aaaaa': {'a': 1}}}, 'a': 2}} + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + converted_dict = yaml.load(converted_csv_str) + self.assertDictEqual(converted_dict, expected_output) + + def test_csv_str_load_supported_datatypes(self): + csv_str = 'a=1,b=2.,c=[1,2,3],d=\'hello, there\',e=\"Hi.\"' + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + converted_dict = yaml.load(converted_csv_str) + self.assertEqual(converted_dict['a'], 1) + self.assertEqual(converted_dict['b'], 2.) + self.assertEqual(converted_dict['c'], [1, 2, 3]) + self.assertEqual(converted_dict['d'], 'hello, there') + self.assertEqual(converted_dict['e'], 'Hi.') + + def test_csv_str_load_unsupported_datatypes(self): + csv_str = 'a=[[1,2,3],[4,5,6]]' + self.assertRaises(ValueError, params_dict.nested_csv_str_to_json_str, + csv_str) + + def test_csv_str_to_json_str_spacing(self): + csv_str1 = 'a=1,b=2,c=3' + csv_str2 = 'a = 1, b = 2, c = 3' + json_str = '{a : 1, b : 2, c : 3}' + converted_csv_str1 = params_dict.nested_csv_str_to_json_str(csv_str1) + converted_csv_str2 = params_dict.nested_csv_str_to_json_str(csv_str2) + self.assertEqual(converted_csv_str1, converted_csv_str2) + self.assertEqual(converted_csv_str1, json_str) + self.assertEqual(converted_csv_str2, json_str) + + def test_gcs_added_quotes(self): + csv_str = 'a=gs://abc, b=gs://def' + expected_output = '{a : \'gs://abc\', b : \'gs://def\'}' + converted_csv_str = params_dict.nested_csv_str_to_json_str(csv_str) + self.assertEqual(converted_csv_str, expected_output) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/__init__.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e419af524b5f349fe04abfa820c3cb51b777d422 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/base_model.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/base_model.py new file mode 100644 index 0000000000000000000000000000000000000000..835d7e3443dd8991c32eb12c570479640b58487a --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/base_model.py @@ -0,0 +1,45 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Abstraction of multi-task model.""" +from typing import Text, Dict + +import tensorflow as tf + + +class MultiTaskBaseModel(tf.Module): + """Base class that holds multi-task model computation.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._sub_tasks = self._instantiate_sub_tasks() + + def _instantiate_sub_tasks(self) -> Dict[Text, tf.keras.Model]: + """Abstract function that sets up the computation for each sub-task. + + Returns: + A map from task name (as string) to a tf.keras.Model object that + represents the sub-task in the multi-task pool. + """ + raise NotImplementedError( + "_instantiate_sub_task_models() is not implemented.") + + @property + def sub_tasks(self): + """Fetch a map of task name (string) to task model (tf.keras.Model).""" + return self._sub_tasks + + def initialize(self): + """Optional function that loads a pre-train checkpoint.""" + return diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/base_trainer.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/base_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..45cdb6cdde32866c31f23804df8b1efac521eee8 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/base_trainer.py @@ -0,0 +1,170 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Multitask base trainer implementation. + +The trainer derives from the Orbit `StandardTrainer` class. +""" +from typing import Union + +import gin +import orbit +import tensorflow as tf + +from official.modeling import optimization +from official.modeling.multitask import base_model +from official.modeling.multitask import multitask + + +@gin.configurable +class MultiTaskBaseTrainer(orbit.StandardTrainer): + """Multitask base trainer.""" + + def __init__(self, + multi_task: multitask.MultiTask, + multi_task_model: Union[tf.keras.Model, + base_model.MultiTaskBaseModel], + optimizer: tf.optimizers.Optimizer, + trainer_options=None, + train_datasets=None): + self._strategy = tf.distribute.get_strategy() + self._multi_task = multi_task + self._multi_task_model = multi_task_model + self._optimizer = optimizer + + self._training_losses = None + self._training_metrics = None + self._global_step = orbit.utils.create_global_step() + + # Creates a shadow copy of the weights to store weights moving average. + if isinstance(self._optimizer, optimization.ExponentialMovingAverage + ) and not self._optimizer.has_shadow_copy: + self._optimizer.shadow_copy(multi_task_model) + + if hasattr(self.multi_task_model, "checkpoint_items"): + checkpoint_items = self.multi_task_model.checkpoint_items + else: + checkpoint_items = {} + + self._checkpoint = tf.train.Checkpoint( + model=self.multi_task_model, + optimizer=self.optimizer, + global_step=self.global_step, + **checkpoint_items) + + if train_datasets is None: + train_datasets = {} + for name, task in self.multi_task.tasks.items(): + train_datasets[name] = orbit.utils.make_distributed_dataset( + self.strategy, task.build_inputs, task.task_config.train_data) + + super().__init__( + train_dataset=train_datasets, + options=trainer_options or orbit.StandardTrainerOptions()) + + def train_loop_begin(self): + """Clean up states that hold losses and metrics.""" + for _, train_loss_metric in self.training_losses.items(): + train_loss_metric.reset_states() + + for _, metrics in self.training_metrics.items(): + for metric in metrics: + metric.reset_states() + + def train_loop_end(self): + """Record loss and metric values per task.""" + result = {} + for task_name, loss in self.training_losses.items(): + result[task_name] = {loss.name: loss.result()} + for task_name, task_metrics in self.training_metrics.items(): + result[task_name].update( + {metric.name: metric.result() for metric in task_metrics}) + # Note that, the learning rate schedule is managed by the keras optimizer + # internally, which respects the number of backward pass as `iterations`. + # The learning rate schedule does not follow the trainer logical global + # step of multiple tasks. + if callable(self.optimizer.learning_rate): + result["learning_rate"] = self.optimizer.learning_rate( + self.optimizer.iterations) + else: + result["learning_rate"] = self.optimizer.learning_rate + return result + + @property + def checkpoint(self): + """Accesses the training checkpoint.""" + return self._checkpoint + + @property + def training_losses(self): + """Access training loss metric objects for all tasks.""" + if self._training_losses is None: + # Builds the per-task metrics and losses. + # This the total summed training loss of tasks in the joint training. + self._training_losses = dict( + total_loss=tf.keras.metrics.Mean("training_loss", dtype=tf.float32)) + for name in self.multi_task.tasks: + self._training_losses[name] = tf.keras.metrics.Mean( + "training_loss", dtype=tf.float32) + return self._training_losses + + @property + def training_metrics(self): + """Access training metric metric objects for all tasks.""" + if self._training_metrics is None: + # Builds the per-task metrics and losses. + self._training_metrics = {} + for name, task in self.multi_task.tasks.items(): + self._training_metrics[name] = task.build_metrics(training=True) + return self._training_metrics + + @property + def strategy(self): + return self._strategy + + @property + def multi_task(self): + return self._multi_task + + @property + def multi_task_model(self): + return self._multi_task_model + + @property + def optimizer(self): + return self._optimizer + + @property + def global_step(self): + return self._global_step + + def train_step(self, iterator_map): + """The default train step calling the multi-task train step. + + Args: + iterator_map: a dictionary of task names and per-task dataset iterators. + """ + + def step_fn(inputs): + losses = self.multi_task.joint_train_step( + inputs, + multi_task_model=self.multi_task_model, + optimizer=self.optimizer, + task_metrics=self.training_metrics) + for key, loss in losses.items(): + self.training_losses[key].update_state(loss) + + self.strategy.run( + step_fn, args=(tf.nest.map_structure(next, iterator_map),)) + self.global_step.assign_add(1) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/base_trainer_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/base_trainer_test.py new file mode 100644 index 0000000000000000000000000000000000000000..2427ff85f2af4c79fb3f7f3cc40c9fc82c0a7e61 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/base_trainer_test.py @@ -0,0 +1,90 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for multitask.base_trainer.""" +from absl.testing import parameterized +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.modeling.multitask import base_trainer +from official.modeling.multitask import configs +from official.modeling.multitask import multitask +from official.modeling.multitask import test_utils + + +def all_strategy_combinations(): + return combinations.combine( + distribution=[ + strategy_combinations.default_strategy, + strategy_combinations.cloud_tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ], + mode="eager", + ) + + +class BaseTrainerTest(tf.test.TestCase, parameterized.TestCase): + + @combinations.generate(all_strategy_combinations()) + def test_multitask_joint_trainer(self, distribution): + with distribution.scope(): + tasks = [ + test_utils.MockFooTask(params=test_utils.FooConfig(), name="foo"), + test_utils.MockBarTask(params=test_utils.BarConfig(), name="bar") + ] + task_weights = {"foo": 1.0, "bar": 1.0} + test_multitask = multitask.MultiTask( + tasks=tasks, task_weights=task_weights) + test_optimizer = tf.keras.optimizers.SGD(0.1) + model = test_utils.MockMultiTaskModel() + test_trainer = base_trainer.MultiTaskBaseTrainer( + multi_task=test_multitask, + multi_task_model=model, + optimizer=test_optimizer) + results = test_trainer.train(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertContainsSubset(["training_loss", "bar_acc"], + results["bar"].keys()) + self.assertContainsSubset(["training_loss", "foo_acc"], + results["foo"].keys()) + + def test_trainer_with_configs(self): + config = configs.MultiTaskConfig( + task_routines=(configs.TaskRoutine( + task_name="foo", + task_config=test_utils.FooConfig(), + task_weight=0.5), + configs.TaskRoutine( + task_name="bar", + task_config=test_utils.BarConfig(), + task_weight=0.5))) + test_multitask = multitask.MultiTask.from_config(config) + test_optimizer = tf.keras.optimizers.SGD(0.1) + model = test_utils.MockMultiTaskModel() + test_trainer = base_trainer.MultiTaskBaseTrainer( + multi_task=test_multitask, + multi_task_model=model, + optimizer=test_optimizer) + results = test_trainer.train(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertContainsSubset(["training_loss", "bar_acc"], + results["bar"].keys()) + self.assertContainsSubset(["training_loss", "foo_acc"], + results["foo"].keys()) + self.assertEqual(test_multitask.task_weight("foo"), 0.5) + self.assertEqual(test_trainer.global_step.numpy(), 5) + self.assertIn("learning_rate", results) + + +if __name__ == "__main__": + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/configs.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/configs.py new file mode 100644 index 0000000000000000000000000000000000000000..453db3475072086606f0a979758524c5f789d454 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/configs.py @@ -0,0 +1,80 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Configuration definitions for multi-task training.""" +from typing import Optional, Tuple + +import dataclasses + +from official.core import config_definitions as cfg +from official.modeling import hyperparams + + +@dataclasses.dataclass +class TaskRoutine(hyperparams.Config): + # TODO(hongkuny): deprecate the task_name once we migrated client code. + task_name: str = "" + task_config: cfg.TaskConfig = None + eval_steps: Optional[int] = None + task_weight: Optional[float] = 1.0 + + +@dataclasses.dataclass +class MultiTaskConfig(hyperparams.Config): + init_checkpoint: str = "" + model: hyperparams.Config = None + task_routines: Tuple[TaskRoutine, ...] = () + + +@dataclasses.dataclass +class ProportionalSampleConfig(hyperparams.Config): + alpha: float = 1.0 + + +@dataclasses.dataclass +class AnnealingSampleConfig(hyperparams.Config): + steps_per_epoch: int = 5 + total_steps: int = 20 + + +@dataclasses.dataclass +class TaskSamplingConfig(hyperparams.OneOfConfig): + type: str = "" + uniform: hyperparams.Config = hyperparams.Config() + proportional: ProportionalSampleConfig = ProportionalSampleConfig() + annealing: AnnealingSampleConfig = AnnealingSampleConfig() + + +@dataclasses.dataclass +class MultiTaskTrainerConfig(cfg.TrainerConfig): + trainer_type: str = "interleaving" + task_sampler: TaskSamplingConfig = TaskSamplingConfig(type="proportional") + + +@dataclasses.dataclass +class MultiTaskExperimentConfig(hyperparams.Config): + """An experiment config for multi-task training and multi-task evaluation.""" + task: MultiTaskConfig = MultiTaskConfig() + trainer: MultiTaskTrainerConfig = MultiTaskTrainerConfig() + runtime: cfg.RuntimeConfig = cfg.RuntimeConfig() + + +@dataclasses.dataclass +class MultiEvalExperimentConfig(cfg.ExperimentConfig): + """An experiment config for single-task training and multi-task evaluation. + + Attributes: + eval_tasks: individual evaluation tasks. + """ + eval_tasks: Tuple[TaskRoutine, ...] = () diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/evaluator.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..c896e2c8811c828c2eb0199ae5ade8103ce65184 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/evaluator.py @@ -0,0 +1,180 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Multitask Evaluator implementation. + +The evaluator implements the Orbit `AbstractEvaluator` interface. +""" +from typing import Dict, List, Optional, Union +import gin +import orbit +import tensorflow as tf + +from official.core import base_task +from official.core import train_utils +from official.modeling.multitask import base_model + + +@gin.configurable +class MultiTaskEvaluator(orbit.AbstractEvaluator): + """Implements the common trainer shared for TensorFlow models.""" + + def __init__( + self, + eval_tasks: List[base_task.Task], + model: Union[tf.keras.Model, base_model.MultiTaskBaseModel], + global_step: Optional[tf.Variable] = None, + eval_steps: Optional[Dict[str, int]] = None, + checkpoint_exporter: Optional[train_utils.BestCheckpointExporter] = None): + """Initialize common trainer for TensorFlow models. + + Args: + eval_tasks: A list of tasks to evaluate. + model: tf.keras.Model instance. + global_step: the global step variable. + eval_steps: a dictionary of steps to run eval keyed by task names. + checkpoint_exporter: an object that has the `maybe_export_checkpoint` + interface. + """ + # Gets the current distribution strategy. If not inside any strategy scope, + # it gets a single-replica no-op strategy. + self._strategy = tf.distribute.get_strategy() + self._tasks = eval_tasks + self._model = model + self._global_step = global_step or orbit.utils.create_global_step() + self._checkpoint_exporter = checkpoint_exporter + if hasattr(self.model, "checkpoint_items"): + checkpoint_items = self.model.checkpoint_items + else: + checkpoint_items = {} + + self._checkpoint = tf.train.Checkpoint( + model=self.model, + global_step=self.global_step, + **checkpoint_items) + + self._validation_losses = None + self._validation_metrics = None + + # Builds per-task datasets. + self.eval_datasets = {} + self.eval_steps = eval_steps or {} + for task in self.tasks: + self.eval_datasets[task.name] = orbit.utils.make_distributed_dataset( + self.strategy, task.build_inputs, task.task_config.validation_data) + + # Builds per-task validation loops. + def get_function(task_name, task): + + task_metrics = self.validation_metrics[task_name] + task_loss = self.validation_losses[task_name] + if isinstance(self.model, base_model.MultiTaskBaseModel): + model = self.model.sub_tasks[task_name] + else: + model = self.model + + def step_fn(inputs): + logs = task.validation_step(inputs, model=model, metrics=task_metrics) + task_loss.update_state(logs[task.loss]) + return logs + + @tf.function + def eval_step_fn(iterator): + distributed_outputs = self.strategy.run(step_fn, args=(next(iterator),)) + return tf.nest.map_structure(self.strategy.experimental_local_results, + distributed_outputs) + + return orbit.utils.create_loop_fn(eval_step_fn) + + self.task_fns = { + task.name: get_function(task.name, task) for task in self.tasks + } + + @property + def strategy(self): + return self._strategy + + @property + def tasks(self): + return self._tasks + + @property + def model(self): + return self._model + + @property + def global_step(self): + return self._global_step + + @property + def validation_losses(self): + """Accesses the validation loss metric object.""" + if self._validation_losses is None: + # Builds the per-task metrics and losses. + self._validation_losses = {} + for task in self.tasks: + self._validation_losses[task.name] = tf.keras.metrics.Mean( + "validation_loss", dtype=tf.float32) + return self._validation_losses + + @property + def validation_metrics(self): + """Accesses all validation metric metric objects.""" + if self._validation_metrics is None: + # Builds the per-task metrics and losses. + self._validation_metrics = {} + for task in self.tasks: + self._validation_metrics[task.name] = task.build_metrics(training=False) + return self._validation_metrics + + @property + def checkpoint(self): + """Accesses the training checkpoint.""" + return self._checkpoint + + def evaluate(self, num_steps: tf.Tensor): + """Performs evaluation for each `EvalTask`.""" + for metric in self.validation_losses.values(): + metric.reset_states() + for metrics in self.validation_metrics.values(): + for metric in metrics: + metric.reset_states() + results = {} + eval_iters = tf.nest.map_structure(iter, self.eval_datasets) + + for task in self.tasks: + outputs = None + name = task.name + eval_iter = eval_iters[name] + task_eval_steps = self.eval_steps.get(name, None) or num_steps + outputs = self.task_fns[name]( + eval_iter, + task_eval_steps, + state=outputs, + reduce_fn=task.aggregate_logs) + task_metrics = self.validation_metrics[name] + task_loss = self.validation_losses[name] + logs = {} + for metric in task_metrics + [task_loss]: + logs[metric.name] = metric.result() + if outputs: + metrics = task.reduce_aggregated_logs( + outputs, global_step=self.global_step) + logs.update(metrics) + results[name] = logs + + if self._checkpoint_exporter: + self._checkpoint_exporter.maybe_export_checkpoint( + self.checkpoint, results, self.global_step.numpy()) + return results diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/evaluator_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/evaluator_test.py new file mode 100644 index 0000000000000000000000000000000000000000..4725e63e5f996fa3432557906dd8548f08e99c53 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/evaluator_test.py @@ -0,0 +1,133 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for multitask.evaluator.""" +from absl.testing import parameterized +import numpy as np +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.core import base_task +from official.core import config_definitions as cfg +from official.modeling.multitask import evaluator + + +def all_strategy_combinations(): + return combinations.combine( + distribution=[ + strategy_combinations.default_strategy, + strategy_combinations.cloud_tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ], + mode="eager", + ) + + +class MockModel(tf.keras.Model): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.dense = tf.keras.layers.Dense(1) + + def call(self, inputs): + print(inputs, type(inputs)) + if "y" in inputs: + self.add_loss(tf.zeros((1,), dtype=tf.float32)) + else: + self.add_loss(tf.ones((1,), dtype=tf.float32)) + return self.dense(inputs["x"]) + + +class MockTask(base_task.Task): + """Mock task object for testing.""" + + def build_metrics(self, training: bool = True): + del training + return [tf.keras.metrics.Accuracy(name="acc")] + + def build_inputs(self, params): + + def generate_data(_): + x = tf.zeros(shape=(2,), dtype=tf.float32) + label = tf.zeros([1], dtype=tf.int32) + if self.name == "bar": + return dict(x=x, y=x), label + else: + return dict(x=x), label + + dataset = tf.data.Dataset.range(1) + dataset = dataset.repeat() + dataset = dataset.map( + generate_data, num_parallel_calls=tf.data.experimental.AUTOTUNE) + return dataset.prefetch(buffer_size=1).batch(2, drop_remainder=True) + + def validation_step(self, inputs, model: tf.keras.Model, metrics=None): + logs = super().validation_step(inputs, model, metrics) + logs["counter"] = tf.ones((1,), dtype=tf.float32) + return logs + + def aggregate_logs(self, state, step_outputs): + if state is None: + state = {} + for key, value in step_outputs.items(): + if key not in state: + state[key] = [] + state[key].append( + np.concatenate([np.expand_dims(v.numpy(), axis=0) for v in value])) + return state + + def reduce_aggregated_logs(self, aggregated_logs, global_step=None): + for k, v in aggregated_logs.items(): + aggregated_logs[k] = np.sum(np.stack(v, axis=0)) + return aggregated_logs + + +class EvaluatorTest(tf.test.TestCase, parameterized.TestCase): + + @combinations.generate(all_strategy_combinations()) + def test_multitask_evaluator(self, distribution): + with distribution.scope(): + tasks = [ + MockTask(params=cfg.TaskConfig(), name="bar"), + MockTask(params=cfg.TaskConfig(), name="foo") + ] + model = MockModel() + test_evaluator = evaluator.MultiTaskEvaluator( + eval_tasks=tasks, model=model) + results = test_evaluator.evaluate(tf.convert_to_tensor(1, dtype=tf.int32)) + self.assertContainsSubset(["validation_loss", "acc"], results["bar"].keys()) + self.assertContainsSubset(["validation_loss", "acc"], results["foo"].keys()) + self.assertEqual(results["bar"]["validation_loss"], 0.0) + self.assertEqual(results["foo"]["validation_loss"], 1.0) + + @combinations.generate(all_strategy_combinations()) + def test_multitask_evaluator_numpy_metrics(self, distribution): + with distribution.scope(): + tasks = [ + MockTask(params=cfg.TaskConfig(), name="bar"), + MockTask(params=cfg.TaskConfig(), name="foo") + ] + model = MockModel() + test_evaluator = evaluator.MultiTaskEvaluator( + eval_tasks=tasks, model=model) + results = test_evaluator.evaluate(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertEqual(results["bar"]["counter"], + 5. * distribution.num_replicas_in_sync) + self.assertEqual(results["foo"]["counter"], + 5. * distribution.num_replicas_in_sync) + + +if __name__ == "__main__": + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/interleaving_trainer.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/interleaving_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..1bc943dfb99696fbdcc3ec3517a9bbf7aea51e34 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/interleaving_trainer.py @@ -0,0 +1,102 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Multitask trainer that interleaves each task's train step.""" +from typing import Union +import gin +import orbit +import tensorflow as tf +from official.modeling.multitask import base_model +from official.modeling.multitask import base_trainer +from official.modeling.multitask import multitask +from official.modeling.multitask import task_sampler as sampler + + +@gin.configurable +class MultiTaskInterleavingTrainer(base_trainer.MultiTaskBaseTrainer): + """MultiTask trainer that interleaves task update.""" + + def __init__(self, + multi_task: multitask.MultiTask, + multi_task_model: Union[tf.keras.Model, + base_model.MultiTaskBaseModel], + optimizer: tf.optimizers.Optimizer, + task_sampler: sampler.TaskSampler, + trainer_options=None): + super().__init__( + multi_task=multi_task, + multi_task_model=multi_task_model, + optimizer=optimizer, + trainer_options=trainer_options) + self._task_sampler = task_sampler + + # Build per task train step. + def _get_task_step(task_name, task): + + def step_fn(inputs): + if isinstance(self.multi_task_model, base_model.MultiTaskBaseModel): + task_model = self.multi_task_model.sub_tasks[task_name] + else: + task_model = self.multi_task_model + task_logs = task.train_step( + inputs, + model=task_model, + optimizer=self.optimizer, + metrics=self.training_metrics[task_name]) + self.training_losses[task_name].update_state(task_logs[task.loss]) + + return step_fn + + self._task_train_step_map = { + name: _get_task_step(name, task) + for name, task in self.multi_task.tasks.items() + } + + # TODO(haozhangthu): Add taskwise step counter to train_loop_end for logging + # on TensorBoard. + self._task_step_counters = { + name: orbit.utils.create_global_step() for name in self.multi_task.tasks + } + + def task_step_counter(self, name): + return self._task_step_counters[name] + + def train_step(self, iterator_map): + # Sample one task to train according to a multinomial distribution + rn = tf.random.stateless_uniform(shape=[], seed=(0, self.global_step)) + cumulative_sample_distribution = self._task_sampler.task_cumulative_distribution( + self.global_step) + # Prepend a [0.0] for indexing convenience. + cumulative_sample_distribution = tf.concat( + [tf.constant([0.0], dtype=tf.float32), cumulative_sample_distribution], + axis=0) + + for idx, (name, _) in enumerate(self.multi_task.tasks.items()): + begin = cumulative_sample_distribution[idx] + end = cumulative_sample_distribution[idx + 1] + if rn >= begin and rn < end: + self._strategy.run( + self._task_train_step_map[name], args=(next(iterator_map[name]),)) + self.global_step.assign_add(1) + self.task_step_counter(name).assign_add(1) + + def train_loop_end(self): + """Record loss and metric values per task.""" + result = super().train_loop_end() + # Interleaving training does not have a good semantic for `total_loss`. In + # fact, it is always zero. To avoid confusion, we filter the `total_loss` + # from the result logs. + if 'total_loss' in result: + result.pop('total_loss') + return result diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/interleaving_trainer_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/interleaving_trainer_test.py new file mode 100644 index 0000000000000000000000000000000000000000..a2b1da1b60d983817b029128737ce11275dfb549 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/interleaving_trainer_test.py @@ -0,0 +1,102 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for multitask.interleaving_trainer.""" +from absl.testing import parameterized +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.modeling.multitask import configs +from official.modeling.multitask import interleaving_trainer +from official.modeling.multitask import multitask +from official.modeling.multitask import task_sampler +from official.modeling.multitask import test_utils + + +def all_strategy_combinations(): + return combinations.combine( + distribution=[ + strategy_combinations.default_strategy, + strategy_combinations.cloud_tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ], + mode="eager", + ) + + +class InterleavingTrainerTest(tf.test.TestCase, parameterized.TestCase): + + @combinations.generate(all_strategy_combinations()) + def test_multitask_interleaving_trainer(self, distribution): + with distribution.scope(): + tasks = [ + test_utils.MockFooTask(params=test_utils.FooConfig(), name="foo"), + test_utils.MockBarTask(params=test_utils.BarConfig(), name="bar") + ] + test_multitask = multitask.MultiTask(tasks=tasks) + test_optimizer = tf.keras.optimizers.SGD(0.1) + model = test_utils.MockMultiTaskModel() + sampler = task_sampler.UniformTaskSampler( + task_weights=test_multitask.task_weights) + test_trainer = interleaving_trainer.MultiTaskInterleavingTrainer( + multi_task=test_multitask, + multi_task_model=model, + optimizer=test_optimizer, + task_sampler=sampler) + results = test_trainer.train(tf.convert_to_tensor(5, dtype=tf.int32)) + self.assertContainsSubset(["training_loss", "bar_acc"], + results["bar"].keys()) + self.assertContainsSubset(["training_loss", "foo_acc"], + results["foo"].keys()) + self.assertNotIn("total_loss", results) + + @combinations.generate(all_strategy_combinations()) + def test_trainer_with_configs(self, distribution): + config = configs.MultiTaskConfig( + task_routines=(configs.TaskRoutine( + task_name="foo", + task_config=test_utils.FooConfig(), + task_weight=3.0), + configs.TaskRoutine( + task_name="bar", + task_config=test_utils.BarConfig(), + task_weight=1.0))) + with distribution.scope(): + test_multitask = multitask.MultiTask.from_config(config) + test_optimizer = tf.keras.optimizers.SGD(0.1) + model = test_utils.MockMultiTaskModel() + num_step = 1000 + sampler = task_sampler.AnnealingTaskSampler( + task_weights=test_multitask.task_weights, + steps_per_epoch=num_step/5, + total_steps=num_step) + test_trainer = interleaving_trainer.MultiTaskInterleavingTrainer( + multi_task=test_multitask, + multi_task_model=model, + optimizer=test_optimizer, + task_sampler=sampler) + results = test_trainer.train(tf.convert_to_tensor(num_step, dtype=tf.int32)) + self.assertContainsSubset(["training_loss", "bar_acc"], + results["bar"].keys()) + self.assertContainsSubset(["training_loss", "foo_acc"], + results["foo"].keys()) + self.assertEqual(test_trainer.global_step.numpy(), num_step) + bar_sampled_step = test_trainer.task_step_counter("bar").numpy() + foo_sampled_step = test_trainer.task_step_counter("foo").numpy() + self.assertEqual(bar_sampled_step + foo_sampled_step, num_step) + + +if __name__ == "__main__": + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/multitask.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/multitask.py new file mode 100644 index 0000000000000000000000000000000000000000..85a345382a33871bd587767166f73faef8454595 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/multitask.py @@ -0,0 +1,145 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Experimental MultiTask base class for multi-task training/evaluation.""" +import abc +from typing import Dict, List, Optional, Text, Union + +import tensorflow as tf +from official.core import base_task +from official.core import config_definitions +from official.core import task_factory +from official.modeling import optimization +from official.modeling.multitask import base_model +from official.modeling.multitask import configs + +OptimizationConfig = optimization.OptimizationConfig +RuntimeConfig = config_definitions.RuntimeConfig + + +class MultiTask(tf.Module, metaclass=abc.ABCMeta): + """A multi-task class to manage multiple tasks.""" + + def __init__(self, + tasks: Union[Dict[Text, base_task.Task], List[base_task.Task]], + task_weights: Optional[Dict[str, Union[float, int]]] = None, + task_eval_steps: Optional[Dict[str, int]] = None, + name: Optional[str] = None): + """MultiTask initialization. + + Args: + tasks: a list or a flat dict of Task. + task_weights: a dict of (task, task weight), task weight can be applied + directly during loss summation in a joint backward step, or it can be + used to sample task among interleaved backward step. + task_eval_steps: a dict of (task, eval steps). + name: the instance name of a MultiTask object. + """ + super().__init__(name=name) + if isinstance(tasks, list): + self._tasks = {} + for task in tasks: + if task.name in self._tasks: + raise ValueError("Duplicated tasks found, task.name is %s" % + task.name) + self._tasks[task.name] = task + elif isinstance(tasks, dict): + self._tasks = tasks + else: + raise ValueError("The tasks argument has an invalid type: %s" % + type(tasks)) + self.task_eval_steps = task_eval_steps or {} + self._task_weights = task_weights or {} + self._task_weights = dict([ + (name, self._task_weights.get(name, 1.0)) for name in self.tasks + ]) + + @classmethod + def from_config(cls, config: configs.MultiTaskConfig, logging_dir=None): + tasks = {} + task_eval_steps = {} + task_weights = {} + for task_routine in config.task_routines: + task_name = task_routine.task_name or task_routine.task_config.name + tasks[task_name] = task_factory.get_task( + task_routine.task_config, logging_dir=logging_dir, name=task_name) + task_eval_steps[task_name] = task_routine.eval_steps + task_weights[task_name] = task_routine.task_weight + return cls( + tasks, task_eval_steps=task_eval_steps, task_weights=task_weights) + + @property + def tasks(self): + return self._tasks + + def task_weight(self, task_name): + return self._task_weights[task_name] + + @property + def task_weights(self): + return self._task_weights + + @classmethod + def create_optimizer(cls, + optimizer_config: OptimizationConfig, + runtime_config: Optional[RuntimeConfig] = None): + return base_task.Task.create_optimizer( + optimizer_config=optimizer_config, runtime_config=runtime_config) + + def joint_train_step(self, task_inputs, + multi_task_model: base_model.MultiTaskBaseModel, + optimizer: tf.keras.optimizers.Optimizer, task_metrics, + **kwargs): + """The joint train step. + + Args: + task_inputs: a dictionary of task names and per-task features. + multi_task_model: a MultiTaskBaseModel instance. + optimizer: a tf.optimizers.Optimizer. + task_metrics: a dictionary of task names and per-task metrics. + **kwargs: other arguments to pass through. + + Returns: + A dictionary of losses, inculding per-task losses and their weighted sum. + """ + losses = {} + with tf.GradientTape() as tape: + total_loss = 0.0 + for name, model in multi_task_model.sub_tasks.items(): + inputs = task_inputs[name] + if isinstance(inputs, tuple) and len(inputs) == 2: + features, labels = inputs + elif isinstance(inputs, dict): + features, labels = inputs, inputs + else: + raise ValueError("The iterator output is neither a tuple nor a " + "dictionary. It is not implemented to support " + "such outputs.") + outputs = model(features, training=True) + task_loss = self.tasks[name].build_losses(labels, outputs) + task_weight = self.task_weight(name) + total_loss += task_weight * task_loss + losses[name] = task_loss + self.tasks[name].process_metrics(task_metrics[name], labels, outputs, + **kwargs) + + # Scales loss as the default gradients allreduce performs sum inside + # the optimizer. + scaled_loss = total_loss / tf.distribute.get_strategy( + ).num_replicas_in_sync + tvars = multi_task_model.trainable_variables + grads = tape.gradient(scaled_loss, tvars) + optimizer.apply_gradients(list(zip(grads, tvars))) + losses["total_loss"] = total_loss + return losses diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/task_sampler.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/task_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..1c365a9df09866636f3a6bfa4ef78be8dd8ff624 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/task_sampler.py @@ -0,0 +1,128 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utils to sample tasks for interleaved optimization.""" +import abc +from typing import Union, Dict, Text +import tensorflow as tf + +from official.modeling.multitask import configs + + +class TaskSampler(tf.Module, metaclass=abc.ABCMeta): + """An abstract class defining task sampling API for interleaving trainer.""" + + def __init__(self, task_weights: Dict[Text, Union[float, int]]): + self._task_weights = task_weights + + @property + def task_weights(self): + return self._task_weights + + @abc.abstractmethod + def task_cumulative_distribution(self, global_step: tf.Tensor) -> tf.Tensor: + """Compute cumulative distribution to sample tasks. + + It calculates the cumulative distribution of the multinomial task + distribution with respect to which to be sampled against. + + Args: + global_step: A tensor indicating current progess of training. + + Returns: + A float tensor with shape (#(task), 1) that represents the cumulative + sampling distribution. + """ + pass + + +class UniformTaskSampler(TaskSampler): + """Sample all tasks uniformly.""" + + def __init__(self, task_weights: Dict[Text, Union[float, int]]): + super(UniformTaskSampler, self).__init__(task_weights=task_weights) + self._uniform_cumulative = tf.math.cumsum( + tf.constant( + [1.0 / len(self._task_weights)] * len(self._task_weights), + dtype=tf.float32)) + + def task_cumulative_distribution(self, global_step: tf.Tensor) -> tf.Tensor: + del global_step + return self._uniform_cumulative + + +class ProportionalTaskSampler(TaskSampler): + """Sample tasks proportional to task weights.""" + + def __init__(self, + task_weights: Dict[Text, Union[float, int]], + alpha: float = 1.0): + super(ProportionalTaskSampler, self).__init__(task_weights=task_weights) + self._alpha = tf.cast(alpha, dtype=tf.float32) + task_weight_dict_ordered_list = tf.constant( + [weight for _, weight in self._task_weights.items()], dtype=tf.float32) + task_sizes = tf.math.pow(task_weight_dict_ordered_list, self._alpha) + task_distribution = task_sizes / tf.reduce_sum(task_sizes) + self._porportional_cumulative = tf.math.cumsum(task_distribution) + + def task_cumulative_distribution(self, global_step: tf.Tensor) -> tf.Tensor: + del global_step + return self._porportional_cumulative + + +class AnnealingTaskSampler(TaskSampler): + """Sample tasks according to task weights as well as training progress. + + See http://proceedings.mlr.press/v97/stickland19a/stickland19a.pdf + """ + + def __init__(self, + task_weights: Dict[Text, Union[float, int]], + steps_per_epoch: int, + total_steps: int): + super(AnnealingTaskSampler, self).__init__(task_weights=task_weights) + self._steps_per_epoch = tf.cast(steps_per_epoch, dtype=tf.float32) + self._total_epochs = tf.cast( + total_steps / self._steps_per_epoch, dtype=tf.float32) + + def task_cumulative_distribution(self, global_step: tf.Tensor) -> tf.Tensor: + cur_epoch = tf.math.floor( + tf.cast(global_step, dtype=tf.float32) / self._steps_per_epoch) + alpha = 1.0 - 0.8 * (cur_epoch - 1) / (self._total_epochs - 1 + 1e-10) + task_weight_dict_ordered_list = [ + weight for _, weight in self._task_weights.items() + ] + task_sizes = tf.math.pow( + tf.constant(task_weight_dict_ordered_list, dtype=tf.float32), + tf.cast(alpha, dtype=tf.float32)) + dynamic_task_distribution = task_sizes / tf.reduce_sum(task_sizes) + return tf.math.cumsum(dynamic_task_distribution) + + +def get_task_sampler(config: configs.TaskSamplingConfig, + task_weights: Dict[Text, float]) -> TaskSampler: + """Utils to create task sampler with configuration and task weights.""" + oneof_config = config.get() + if config.type == 'uniform': + return UniformTaskSampler(task_weights=task_weights) + elif config.type == 'proportional': + return ProportionalTaskSampler( + task_weights=task_weights, alpha=oneof_config.alpha) + elif config.type == 'annealing': + return AnnealingTaskSampler( + task_weights=task_weights, + steps_per_epoch=oneof_config.steps_per_epoch, + total_steps=oneof_config.total_steps) + else: + raise RuntimeError('Task sampler type not supported') diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/task_sampler_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/task_sampler_test.py new file mode 100644 index 0000000000000000000000000000000000000000..5b4695049952dab250f9fdac3d6bfd134e2c644d --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/task_sampler_test.py @@ -0,0 +1,75 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for multitask.task_sampler.""" +import tensorflow as tf + +from official.modeling.multitask import configs +from official.modeling.multitask import task_sampler as sampler + + +class TaskSamplerTest(tf.test.TestCase): + + def setUp(self): + super(TaskSamplerTest, self).setUp() + self._task_weights = {'A': 1.0, 'B': 2.0, 'C': 3.0} + + def test_uniform_sample_distribution(self): + uniform_sampler = sampler.get_task_sampler( + configs.TaskSamplingConfig(type='uniform'), self._task_weights) + for step in range(5): + cumulative_distribution = uniform_sampler.task_cumulative_distribution( + tf.constant(step, dtype=tf.int64)) + self.assertAllClose([0.333333, 0.666666, 1.0], + cumulative_distribution.numpy()) + + def test_proportional_sample_distribution(self): + prop_sampler = sampler.get_task_sampler( + configs.TaskSamplingConfig( + type='proportional', + proportional=configs.ProportionalSampleConfig(alpha=2.0)), + self._task_weights) + # CucmulativeOf(Normalize([1.0^2, 2.0^2, 3.0^2])) + for step in range(5): + cumulative_distribution = prop_sampler.task_cumulative_distribution( + tf.constant(step, dtype=tf.int64)) + self.assertAllClose([0.07142857, 0.35714286, 1.0], + cumulative_distribution.numpy()) + + def test_annealing_sample_distribution(self): + num_epoch = 3 + step_per_epoch = 6 + annel_sampler = sampler.get_task_sampler( + configs.TaskSamplingConfig( + type='annealing', + annealing=configs.AnnealingSampleConfig( + steps_per_epoch=step_per_epoch, + total_steps=step_per_epoch * num_epoch)), self._task_weights) + + global_step = tf.Variable( + 0, dtype=tf.int64, name='global_step', trainable=False) + expected_cumulative_epochs = [[0.12056106, 0.4387236, 1.0], + [0.16666667, 0.5, 1.0], + [0.22477472, 0.5654695, 1.0]] + for epoch in range(num_epoch): + for _ in range(step_per_epoch): + cumulative_distribution = annel_sampler.task_cumulative_distribution( + tf.constant(global_step, dtype=tf.int64)) + global_step.assign_add(1) + self.assertAllClose(expected_cumulative_epochs[epoch], + cumulative_distribution.numpy()) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/test_utils.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/test_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..aa831223817b4968615f5aa87c1e3fbc39021218 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/test_utils.py @@ -0,0 +1,125 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Testing utils for mock models and tasks.""" +from typing import Dict, Text +import tensorflow as tf +from official.core import base_task +from official.core import config_definitions as cfg +from official.core import task_factory +from official.modeling.multitask import base_model + + +class MockFooModel(tf.keras.Model): + """A mock model can consume 'foo' and 'bar' inputs.""" + + def __init__(self, shared_layer, *args, **kwargs): + super().__init__(*args, **kwargs) + self._share_layer = shared_layer + self._foo_specific_layer = tf.keras.layers.Dense(1) + + def call(self, inputs): + self.add_loss(tf.zeros((1,), dtype=tf.float32)) + if "foo" in inputs: + input_tensor = inputs["foo"] + else: + input_tensor = inputs["bar"] + return self._foo_specific_layer(self._share_layer(input_tensor)) + + +class MockBarModel(tf.keras.Model): + + def __init__(self, shared_layer, *args, **kwargs): + super().__init__(*args, **kwargs) + self._share_layer = shared_layer + self._bar_specific_layer = tf.keras.layers.Dense(1) + + def call(self, inputs): + self.add_loss(tf.zeros((2,), dtype=tf.float32)) + return self._bar_specific_layer(self._share_layer(inputs["bar"])) + + +class MockMultiTaskModel(base_model.MultiTaskBaseModel): + + def __init__(self, *args, **kwargs): + self._shared_dense = tf.keras.layers.Dense(1) + super().__init__(*args, **kwargs) + + def _instantiate_sub_tasks(self) -> Dict[Text, tf.keras.Model]: + return { + "foo": MockFooModel(self._shared_dense), + "bar": MockBarModel(self._shared_dense) + } + + +def mock_data(feature_name): + """Mock dataset function.""" + + def _generate_data(_): + x = tf.zeros(shape=(2,), dtype=tf.float32) + label = tf.zeros([1], dtype=tf.int32) + return {feature_name: x}, label + + dataset = tf.data.Dataset.range(1) + dataset = dataset.repeat() + dataset = dataset.map( + _generate_data, num_parallel_calls=tf.data.experimental.AUTOTUNE) + return dataset.prefetch(buffer_size=1).batch(2, drop_remainder=True) + + +class FooConfig(cfg.TaskConfig): + pass + + +class BarConfig(cfg.TaskConfig): + pass + + +@task_factory.register_task_cls(FooConfig) +class MockFooTask(base_task.Task): + """Mock foo task object for testing.""" + + def build_metrics(self, training: bool = True): + del training + return [tf.keras.metrics.Accuracy(name="foo_acc")] + + def build_inputs(self, params): + return mock_data("foo") + + def build_model(self) -> tf.keras.Model: + return MockFooModel(shared_layer=tf.keras.layers.Dense(1)) + + def build_losses(self, labels, model_outputs, aux_losses=None) -> tf.Tensor: + loss = tf.keras.losses.mean_squared_error(labels, model_outputs) + if aux_losses: + loss += tf.add_n(aux_losses) + return tf.reduce_mean(loss) + + +@task_factory.register_task_cls(BarConfig) +class MockBarTask(base_task.Task): + """Mock bar task object for testing.""" + + def build_metrics(self, training: bool = True): + del training + return [tf.keras.metrics.Accuracy(name="bar_acc")] + + def build_inputs(self, params): + return mock_data("bar") + + def build_losses(self, labels, model_outputs, aux_losses=None) -> tf.Tensor: + loss = tf.keras.losses.mean_squared_error(labels, model_outputs) + if aux_losses: + loss += tf.add_n(aux_losses) + return tf.reduce_mean(loss) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/train_lib.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/train_lib.py new file mode 100644 index 0000000000000000000000000000000000000000..62b022030937660720aed2d4417355f86a6fd7c8 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/train_lib.py @@ -0,0 +1,265 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Multitask training driver library.""" +# pytype: disable=attribute-error +import os +from typing import Any, List, Optional, Tuple +from absl import logging +import orbit +import tensorflow as tf +from official.core import base_task +from official.core import base_trainer as core_lib +from official.core import train_utils +from official.modeling.multitask import base_model +from official.modeling.multitask import base_trainer +from official.modeling.multitask import configs +from official.modeling.multitask import evaluator as evaluator_lib +from official.modeling.multitask import interleaving_trainer +from official.modeling.multitask import multitask +from official.modeling.multitask import task_sampler + +TRAINERS = { + 'interleaving': interleaving_trainer.MultiTaskInterleavingTrainer, + 'joint': base_trainer.MultiTaskBaseTrainer +} + + +def run_experiment( + *, + distribution_strategy: tf.distribute.Strategy, + task: multitask.MultiTask, + model: base_model.MultiTaskBaseModel, + mode: str, + params: configs.MultiTaskExperimentConfig, + model_dir: str, + trainer: base_trainer.MultiTaskBaseTrainer = None +) -> base_model.MultiTaskBaseModel: + """Runs train/eval configured by the experiment params. + + Args: + distribution_strategy: A distribution distribution_strategy. + task: A MultiTaskTask instance. + model: A MultiTaskBaseModel instance. + mode: A 'str', specifying the mode. Can be 'train', 'eval', 'train_and_eval' + or 'continuous_eval'. + params: ExperimentConfig instance. + model_dir: A 'str', a path to store model checkpoints and summaries. + trainer: (optional) A multi-task trainer to use. If none is provided, a + default one will be created based on `params`. + + Returns: + model: `base_model.MultiTaskBaseModel` instance. + """ + + is_training = 'train' in mode + is_eval = 'eval' in mode + with distribution_strategy.scope(): + optimizer = task.create_optimizer(params.trainer.optimizer_config, + params.runtime) + kwargs = dict(multi_task=task, multi_task_model=model, optimizer=optimizer) + if params.trainer.trainer_type == 'interleaving': + sampler = task_sampler.get_task_sampler(params.trainer.task_sampler, + task.task_weights) + kwargs.update(dict(task_sampler=sampler)) + if trainer is None: + trainer = TRAINERS[params.trainer.trainer_type]( + **kwargs) if is_training else None + if is_eval: + eval_steps = task.task_eval_steps + evaluator = evaluator_lib.MultiTaskEvaluator( + eval_tasks=task.tasks.values(), + model=model, + eval_steps=eval_steps, + global_step=trainer.global_step if is_training else None, + checkpoint_exporter=train_utils.maybe_create_best_ckpt_exporter( + params, model_dir)) + else: + evaluator = None + + if trainer: + checkpoint = trainer.checkpoint + global_step = trainer.global_step + else: + checkpoint = evaluator.checkpoint + global_step = evaluator.global_step + + # TODO(hongkuny,haozhangthu): Revisit initialization method. + checkpoint_manager = tf.train.CheckpointManager( + checkpoint, + directory=model_dir, + max_to_keep=params.trainer.max_to_keep, + step_counter=global_step, + checkpoint_interval=params.trainer.checkpoint_interval, + init_fn=model.initialize) + + controller = orbit.Controller( + strategy=distribution_strategy, + trainer=trainer, + evaluator=evaluator, + global_step=global_step, + steps_per_loop=params.trainer.steps_per_loop, + checkpoint_manager=checkpoint_manager, + summary_dir=os.path.join(model_dir, 'train'), + eval_summary_dir=os.path.join(model_dir, 'validation'), + summary_interval=params.trainer.summary_interval) + + logging.info('Starts to execute mode: %s', mode) + with distribution_strategy.scope(): + if mode == 'train': + controller.train(steps=params.trainer.train_steps) + elif mode == 'train_and_eval': + controller.train_and_evaluate( + train_steps=params.trainer.train_steps, + eval_steps=params.trainer.validation_steps, + eval_interval=params.trainer.validation_interval) + elif mode == 'eval': + controller.evaluate(steps=params.trainer.validation_steps) + elif mode == 'continuous_eval': + + def timeout_fn(): + if evaluator.global_step.numpy() >= params.trainer.train_steps: + return True + return False + + controller.evaluate_continuously( + steps=params.trainer.validation_steps, + timeout=params.trainer.continuous_eval_timeout, + timeout_fn=timeout_fn) + else: + raise NotImplementedError('The mode is not implemented: %s' % mode) + + return model + + +def run_experiment_with_multitask_eval( + *, + distribution_strategy: tf.distribute.Strategy, + train_task: base_task.Task, + eval_tasks: List[base_task.Task], + mode: str, + params: configs.MultiEvalExperimentConfig, + model_dir: str, + run_post_eval: bool = False, + save_summary: bool = True, + trainer: Optional[core_lib.Trainer] = None) -> Tuple[Any, Any]: + """Runs train/eval configured by the experiment params. + + Args: + distribution_strategy: A distribution distribution_strategy. + train_task: A base_task.Task instance. + eval_tasks: A list of evaluation tasks. + mode: A 'str', specifying the mode. Can be 'train', 'eval', 'train_and_eval' + or 'continuous_eval'. + params: MultiEvalExperimentConfig instance. + model_dir: A 'str', a path to store model checkpoints and summaries. + run_post_eval: Whether to run post eval once after training, metrics logs + are returned. + save_summary: Whether to save train and validation summary. + trainer: the core_lib.Trainer instance. It should be created within the + strategy.scope(). If not provided, an instance will be created by default + if `mode` contains 'train'. + + Returns: + model: `tf.keras.Model` instance. + """ + + is_training = 'train' in mode + is_eval = 'eval' in mode + with distribution_strategy.scope(): + if is_training: + trainer = trainer or core_lib.Trainer( + config=params, + task=train_task, + model=train_task.build_model(), + optimizer=train_task.create_optimizer(params.trainer.optimizer_config, + params.runtime), + train=True, + evaluate=False) + else: + trainer = None + model = trainer.model if trainer else train_task.build_model() + + if is_eval: + eval_steps = dict([(task_routine.task_config.name, + task_routine.eval_steps) + for task_routine in params.eval_tasks]) + evaluator = evaluator_lib.MultiTaskEvaluator( + eval_tasks=eval_tasks, + model=model, + global_step=trainer.global_step if is_training else None, + eval_steps=eval_steps, + checkpoint_exporter=train_utils.maybe_create_best_ckpt_exporter( + params, model_dir)) + else: + evaluator = None + + if trainer: + checkpoint = trainer.checkpoint + global_step = trainer.global_step + else: + checkpoint = evaluator.checkpoint + global_step = evaluator.global_step + + checkpoint_manager = tf.train.CheckpointManager( + checkpoint, + directory=model_dir, + max_to_keep=params.trainer.max_to_keep, + step_counter=global_step, + checkpoint_interval=params.trainer.checkpoint_interval, + init_fn=trainer.initialize if trainer else None) + + controller = orbit.Controller( + strategy=distribution_strategy, + trainer=trainer, + evaluator=evaluator, + global_step=global_step, + steps_per_loop=params.trainer.steps_per_loop, + checkpoint_manager=checkpoint_manager, + summary_dir=os.path.join(model_dir, 'train') if save_summary else None, + eval_summary_dir=os.path.join(model_dir, 'validation') if + (save_summary) else None, + summary_interval=params.trainer.summary_interval if + (save_summary) else None) + + logging.info('Starts to execute mode: %s', mode) + with distribution_strategy.scope(): + if mode == 'train': + controller.train(steps=params.trainer.train_steps) + elif mode == 'train_and_eval': + controller.train_and_evaluate( + train_steps=params.trainer.train_steps, + eval_steps=params.trainer.validation_steps, + eval_interval=params.trainer.validation_interval) + elif mode == 'eval': + controller.evaluate(steps=params.trainer.validation_steps) + elif mode == 'continuous_eval': + + def timeout_fn(): + if evaluator.global_step.numpy() >= params.trainer.train_steps: + return True + return False + + controller.evaluate_continuously( + steps=params.trainer.validation_steps, + timeout=params.trainer.continuous_eval_timeout, + timeout_fn=timeout_fn) + else: + raise NotImplementedError('The mode is not implemented: %s' % mode) + + if run_post_eval: + return model, evaluator.evaluate( + tf.convert_to_tensor(params.trainer.validation_steps)) # pytype: disable=bad-return-type # typed-keras + else: + return model, {} # pytype: disable=bad-return-type # typed-keras diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/train_lib_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/train_lib_test.py new file mode 100644 index 0000000000000000000000000000000000000000..6f90a47f3dca42381bf0024fc8c22a835d3dfd52 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/multitask/train_lib_test.py @@ -0,0 +1,121 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for multitask.train_lib.""" +from absl.testing import parameterized +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.core import task_factory +from official.modeling.hyperparams import params_dict +from official.modeling.multitask import configs +from official.modeling.multitask import multitask +from official.modeling.multitask import test_utils +from official.modeling.multitask import train_lib + + +class TrainLibTest(tf.test.TestCase, parameterized.TestCase): + + def setUp(self): + super().setUp() + self._test_config = { + 'trainer': { + 'checkpoint_interval': 10, + 'steps_per_loop': 10, + 'summary_interval': 10, + 'train_steps': 10, + 'validation_steps': 5, + 'validation_interval': 10, + 'continuous_eval_timeout': 1, + 'optimizer_config': { + 'optimizer': { + 'type': 'sgd', + }, + 'learning_rate': { + 'type': 'constant' + } + } + }, + } + + @combinations.generate( + combinations.combine( + distribution_strategy=[ + strategy_combinations.default_strategy, + strategy_combinations.cloud_tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ], + mode='eager', + flag_mode=['train', 'eval', 'train_and_eval'])) + def test_end_to_end(self, distribution_strategy, flag_mode): + model_dir = self.get_temp_dir() + experiment_config = configs.MultiTaskExperimentConfig( + task=configs.MultiTaskConfig( + task_routines=( + configs.TaskRoutine( + task_name='foo', task_config=test_utils.FooConfig()), + configs.TaskRoutine( + task_name='bar', task_config=test_utils.BarConfig())))) + experiment_config = params_dict.override_params_dict( + experiment_config, self._test_config, is_strict=False) + with distribution_strategy.scope(): + test_multitask = multitask.MultiTask.from_config(experiment_config.task) + model = test_utils.MockMultiTaskModel() + train_lib.run_experiment( + distribution_strategy=distribution_strategy, + task=test_multitask, + model=model, + mode=flag_mode, + params=experiment_config, + model_dir=model_dir) + + @combinations.generate( + combinations.combine( + distribution_strategy=[ + strategy_combinations.default_strategy, + strategy_combinations.cloud_tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + ], + mode='eager', + flag_mode=['train', 'eval', 'train_and_eval'])) + def test_end_to_end_multi_eval(self, distribution_strategy, flag_mode): + model_dir = self.get_temp_dir() + experiment_config = configs.MultiEvalExperimentConfig( + task=test_utils.FooConfig(), + eval_tasks=(configs.TaskRoutine( + task_name='foo', task_config=test_utils.FooConfig(), eval_steps=2), + configs.TaskRoutine( + task_name='bar', + task_config=test_utils.BarConfig(), + eval_steps=3))) + experiment_config = params_dict.override_params_dict( + experiment_config, self._test_config, is_strict=False) + with distribution_strategy.scope(): + train_task = task_factory.get_task(experiment_config.task) + eval_tasks = [ + task_factory.get_task(config.task_config, name=config.task_name) + for config in experiment_config.eval_tasks + ] + train_lib.run_experiment_with_multitask_eval( + distribution_strategy=distribution_strategy, + train_task=train_task, + eval_tasks=eval_tasks, + mode=flag_mode, + params=experiment_config, + model_dir=model_dir) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/__init__.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ee2b99603b0caf5338c0ecd1b78ef0b1577b64c1 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Optimization package definition.""" + +# pylint: disable=wildcard-import +from official.modeling.optimization.configs.learning_rate_config import * +from official.modeling.optimization.configs.optimization_config import * +from official.modeling.optimization.configs.optimizer_config import * +from official.modeling.optimization.ema_optimizer import ExponentialMovingAverage +from official.modeling.optimization.lr_schedule import * +from official.modeling.optimization.optimizer_factory import OptimizerFactory +from official.modeling.optimization.optimizer_factory import register_optimizer_cls diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/adafactor_optimizer.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/adafactor_optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..cea09bda415a7375172d781df3b7f84b3a9da322 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/adafactor_optimizer.py @@ -0,0 +1,20 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Adafactor optimizer. + +A new optimizer that will be open sourced soon. +""" +# pylint: disable=invalid-name, represents an unimplemented class definition. +Adafactor = "Unimplemented" diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/configs/__init__.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/configs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e419af524b5f349fe04abfa820c3cb51b777d422 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/configs/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/configs/learning_rate_config.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/configs/learning_rate_config.py new file mode 100644 index 0000000000000000000000000000000000000000..3904b53dacb83fcb7b85793271f63ee304ad32c0 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/configs/learning_rate_config.py @@ -0,0 +1,288 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dataclasses for learning rate schedule config.""" +from typing import List, Optional + +import dataclasses +from official.modeling.hyperparams import base_config + + +@dataclasses.dataclass +class ConstantLrConfig(base_config.Config): + """Configuration for constant learning rate. + + This class is a containers for the constant learning rate decay configs. + + Attributes: + name: The name of the learning rate schedule. Defaults to Constant. + learning_rate: A float. The learning rate. Defaults to 0.1. + """ + name: str = 'Constant' + learning_rate: float = 0.1 + + +@dataclasses.dataclass +class StepwiseLrConfig(base_config.Config): + """Configuration for stepwise learning rate decay. + + This class is a container for the piecewise constant learning rate scheduling + configs. It will configure an instance of PiecewiseConstantDecay keras + learning rate schedule. + + An example (from keras docs): use a learning rate that's 1.0 for the first + 100001 steps, 0.5 for the next 10000 steps, and 0.1 for any additional steps. + ```python + boundaries: [100000, 110000] + values: [1.0, 0.5, 0.1] + + Attributes: + name: The name of the learning rate schedule. Defaults to PiecewiseConstant. + boundaries: A list of ints of strictly increasing entries. Defaults to None. + values: A list of floats that specifies the values for the intervals defined + by `boundaries`. It should have one more element than `boundaries`. + The learning rate is computed as follows: [0, boundaries[0]] -> + values[0] [boundaries[0], boundaries[1]] -> values[1] + [boundaries[n-1], boundaries[n]] -> values[n] [boundaries[n], + end] -> values[n+1] Defaults to None. + offset: An int. The offset applied to steps. Defaults to 0. + """ + name: str = 'PiecewiseConstantDecay' + boundaries: Optional[List[int]] = None + values: Optional[List[float]] = None + offset: int = 0 + + +@dataclasses.dataclass +class ExponentialLrConfig(base_config.Config): + """Configuration for exponential learning rate decay. + + This class is a containers for the exponential learning rate decay configs. + + Attributes: + name: The name of the learning rate schedule. Defaults to ExponentialDecay. + initial_learning_rate: A float. The initial learning rate. Defaults to None. + decay_steps: A positive integer that is used for decay computation. Defaults + to None. + decay_rate: A float. Defaults to None. + staircase: A boolean, if true, learning rate is decreased at discreate + intervals. Defaults to False. + offset: An int. The offset applied to steps. Defaults to 0. + """ + name: str = 'ExponentialDecay' + initial_learning_rate: Optional[float] = None + decay_steps: Optional[int] = None + decay_rate: Optional[float] = None + staircase: Optional[bool] = None + offset: int = 0 + + +@dataclasses.dataclass +class PolynomialLrConfig(base_config.Config): + """Configuration for polynomial learning rate decay. + + This class is a containers for the polynomial learning rate decay configs. + + Attributes: + name: The name of the learning rate schedule. Defaults to PolynomialDecay. + initial_learning_rate: A float. The initial learning rate. Defaults to None. + decay_steps: A positive integer that is used for decay computation. Defaults + to None. + end_learning_rate: A float. The minimal end learning rate. + power: A float. The power of the polynomial. Defaults to linear, 1.0. + cycle: A boolean, whether or not it should cycle beyond decay_steps. + Defaults to False. + offset: An int. The offset applied to steps. Defaults to 0. + """ + name: str = 'PolynomialDecay' + initial_learning_rate: Optional[float] = None + decay_steps: Optional[int] = None + end_learning_rate: float = 0.0001 + power: float = 1.0 + cycle: bool = False + offset: int = 0 + + +@dataclasses.dataclass +class CosineLrConfig(base_config.Config): + """Configuration for Cosine learning rate decay. + + This class is a containers for the cosine learning rate decay configs, + tf.keras.experimental.CosineDecay. + + Attributes: + name: The name of the learning rate schedule. Defaults to CosineDecay. + initial_learning_rate: A float. The initial learning rate. Defaults to None. + decay_steps: A positive integer that is used for decay computation. Defaults + to None. + alpha: A float. Minimum learning rate value as a fraction of + initial_learning_rate. + offset: An int. The offset applied to steps. Defaults to 0. + """ + name: str = 'CosineDecay' + initial_learning_rate: Optional[float] = None + decay_steps: Optional[int] = None + alpha: float = 0.0 + offset: int = 0 + + +@dataclasses.dataclass +class DirectPowerLrConfig(base_config.Config): + """Configuration for DirectPower learning rate decay. + + This class configures a schedule following follows lr * (step)^power. + + Attributes: + name: The name of the learning rate schedule. Defaults to DirectPowerDecay. + initial_learning_rate: A float. The initial learning rate. Defaults to None. + power: A float. Defaults to -0.5, for sqrt decay. + """ + name: str = 'DirectPowerDecay' + initial_learning_rate: Optional[float] = None + power: float = -0.5 + + +@dataclasses.dataclass +class PowerAndLinearDecayLrConfig(base_config.Config): + """Configuration for DirectPower learning rate decay. + + The schedule has the following behavoir. + Let offset_step = step - offset. + 1) offset_step < 0, the actual learning rate equals initial_learning_rate. + 2) offset_step <= total_decay_steps * (1 - linear_decay_fraction), the + actual learning rate equals lr * offset_step^power. + 3) total_decay_steps * (1 - linear_decay_fraction) <= offset_step < + total_decay_steps, the actual learning rate equals lr * offset_step^power * + (total_decay_steps - offset_step) / (total_decay_steps * + linear_decay_fraction). + 4) offset_step >= total_decay_steps, the actual learning rate equals zero. + + Attributes: + name: The name of the learning rate schedule. Defaults to + PowerAndLinearDecay. + initial_learning_rate: A float. The initial learning rate. Defaults to None. + total_decay_steps: An int. The total number of steps for power + linear + decay. Defaults to None. + power: A float. The order of the polynomial. Defaults to -0.5, for sqrt + decay. + linear_decay_fraction: A float. In the last `linear_decay_fraction` steps, + the learning rate will be multiplied by a linear decay. Defaults to 0.1. + offset: An int. The offset applied to steps. Defaults to 0. + """ + name: str = 'PowerAndLinearDecay' + initial_learning_rate: Optional[float] = None + total_decay_steps: Optional[int] = None + power: float = -0.5 + linear_decay_fraction: float = 0.1 + offset: int = 0 + + +@dataclasses.dataclass +class PowerDecayWithOffsetLrConfig(base_config.Config): + """Configuration for power learning rate decay with step offset. + + Learning rate equals to `pre_offset_learning_rate` if `step` < `offset`. + Otherwise, learning rate equals to lr * (step - offset)^power. + + Attributes: + name: The name of the learning rate schedule. Defaults to + PowerDecayWithOffset. + initial_learning_rate: A float. The initial learning rate. Defaults to None. + power: A float. Defaults to -0.5, for sqrt decay. + offset: An integer. Power decay happens after `offset` steps. + pre_offset_learning_rate: A float. The constant learning rate before + `offset` steps. + """ + name: str = 'PowerDecayWithOffset' + initial_learning_rate: Optional[float] = None + power: float = -0.5 + offset: int = 0 + pre_offset_learning_rate: float = 1.0e6 + + +@dataclasses.dataclass +class StepCosineLrConfig(base_config.Config): + """Configuration for stepwise learning rate decay. + + This class is a container for the piecewise cosine learning rate scheduling + configs. It will configure an instance of StepConsineDecayWithOffset keras + learning rate schedule. + + ```python + boundaries: [100000, 110000] + values: [1.0, 0.5] + lr_decayed_fn = ( + lr_schedule.StepConsineDecayWithOffset( + boundaries, + values)) + ``` + from 0 to 100000 step, it will cosine decay from 1.0 to 0.5 + from 100000 to 110000 step, it cosine decay from 0.5 to 0.0 + + Attributes: + name: The name of the learning rate schedule. Defaults to PiecewiseConstant. + boundaries: A list of ints of strictly increasing entries. Defaults to None. + values: A list of floats that specifies the values for the intervals defined + by `boundaries`. It should have one more element than `boundaries`. + The learning rate is computed as follows: + [0, boundaries[0]] -> cosine from values[0] to values[1] + [boundaries[0], boundaries[1]] -> values[1] to values[2] + ... + [boundaries[n-1], boundaries[n]] -> values[n] to values[n+1] + [boundaries[n], end] -> values[n+1] to 0. + offset: An int. The offset applied to steps. Defaults to 0. + """ + name: str = 'StepConsineDecayWithOffset' + boundaries: Optional[List[int]] = None + values: Optional[List[float]] = None + offset: int = 0 + + +@dataclasses.dataclass +class LinearWarmupConfig(base_config.Config): + """Configuration for linear warmup schedule config. + + This class is a container for the linear warmup schedule configs. + Warmup_learning_rate is the initial learning rate, the final learning rate of + the warmup period is the learning_rate of the optimizer in use. The learning + rate at each step linearly increased according to the following formula: + warmup_learning_rate = warmup_learning_rate + + step / warmup_steps * (final_learning_rate - warmup_learning_rate). + Using warmup overrides the learning rate schedule by the number of warmup + steps. + + Attributes: + name: The name of warmup schedule. Defaults to linear. + warmup_learning_rate: Initial learning rate for the warmup. Defaults to 0. + warmup_steps: Warmup steps. Defaults to None. + """ + name: str = 'linear' + warmup_learning_rate: float = 0 + warmup_steps: Optional[int] = None + + +@dataclasses.dataclass +class PolynomialWarmupConfig(base_config.Config): + """Configuration for linear warmup schedule config. + + This class is a container for the polynomial warmup schedule configs. + + Attributes: + name: The name of warmup schedule. Defaults to Polynomial. + power: Polynomial power. Defaults to 1. + warmup_steps: Warmup steps. Defaults to None. + """ + name: str = 'polynomial' + power: float = 1 + warmup_steps: Optional[int] = None diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/configs/optimization_config.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/configs/optimization_config.py new file mode 100644 index 0000000000000000000000000000000000000000..1bf87e420fb7e36a45f233520baec398d04a8057 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/configs/optimization_config.py @@ -0,0 +1,118 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dataclasses for optimization configs. + +This file define the dataclass for optimization configs (OptimizationConfig). +It also has two helper functions get_optimizer_config, and get_lr_config from +an OptimizationConfig class. +""" +from typing import Optional + +import dataclasses + +from official.modeling.hyperparams import base_config +from official.modeling.hyperparams import oneof +from official.modeling.optimization.configs import learning_rate_config as lr_cfg +from official.modeling.optimization.configs import optimizer_config as opt_cfg + + +@dataclasses.dataclass +class OptimizerConfig(oneof.OneOfConfig): + """Configuration for optimizer. + + Attributes: + type: 'str', type of optimizer to be used, on the of fields below. + sgd: sgd optimizer config. + adam: adam optimizer config. + adamw: adam with weight decay. + lamb: lamb optimizer. + rmsprop: rmsprop optimizer. + lars: lars optimizer. + adagrad: adagrad optimizer. + slide: slide optimizer. + """ + type: Optional[str] = None + sgd: opt_cfg.SGDConfig = opt_cfg.SGDConfig() + adam: opt_cfg.AdamConfig = opt_cfg.AdamConfig() + adamw: opt_cfg.AdamWeightDecayConfig = opt_cfg.AdamWeightDecayConfig() + lamb: opt_cfg.LAMBConfig = opt_cfg.LAMBConfig() + rmsprop: opt_cfg.RMSPropConfig = opt_cfg.RMSPropConfig() + lars: opt_cfg.LARSConfig = opt_cfg.LARSConfig() + adagrad: opt_cfg.AdagradConfig = opt_cfg.AdagradConfig() + slide: opt_cfg.SLIDEConfig = opt_cfg.SLIDEConfig() + adafactor: opt_cfg.AdafactorConfig = opt_cfg.AdafactorConfig() + + +@dataclasses.dataclass +class LrConfig(oneof.OneOfConfig): + """Configuration for lr schedule. + + Attributes: + type: 'str', type of lr schedule to be used, one of the fields below. + constant: constant learning rate config. + stepwise: stepwise learning rate config. + exponential: exponential learning rate config. + polynomial: polynomial learning rate config. + cosine: cosine learning rate config. + power: step^power learning rate config. + power_linear: learning rate config of step^power followed by + step^power*linear. + power_with_offset: power decay with a step offset. + step_cosine_with_offset: Step cosine with a step offset. + """ + type: Optional[str] = None + constant: lr_cfg.ConstantLrConfig = lr_cfg.ConstantLrConfig() + stepwise: lr_cfg.StepwiseLrConfig = lr_cfg.StepwiseLrConfig() + exponential: lr_cfg.ExponentialLrConfig = lr_cfg.ExponentialLrConfig() + polynomial: lr_cfg.PolynomialLrConfig = lr_cfg.PolynomialLrConfig() + cosine: lr_cfg.CosineLrConfig = lr_cfg.CosineLrConfig() + power: lr_cfg.DirectPowerLrConfig = lr_cfg.DirectPowerLrConfig() + power_linear: lr_cfg.PowerAndLinearDecayLrConfig = ( + lr_cfg.PowerAndLinearDecayLrConfig()) + power_with_offset: lr_cfg.PowerDecayWithOffsetLrConfig = ( + lr_cfg.PowerDecayWithOffsetLrConfig()) + step_cosine_with_offset: lr_cfg.StepCosineLrConfig = ( + lr_cfg.StepCosineLrConfig()) + + +@dataclasses.dataclass +class WarmupConfig(oneof.OneOfConfig): + """Configuration for lr schedule. + + Attributes: + type: 'str', type of warmup schedule to be used, one of the fields below. + linear: linear warmup config. + polynomial: polynomial warmup config. + """ + type: Optional[str] = None + linear: lr_cfg.LinearWarmupConfig = lr_cfg.LinearWarmupConfig() + polynomial: lr_cfg.PolynomialWarmupConfig = lr_cfg.PolynomialWarmupConfig() + + +@dataclasses.dataclass +class OptimizationConfig(base_config.Config): + """Configuration for optimizer and learning rate schedule. + + Attributes: + optimizer: optimizer oneof config. + ema: optional exponential moving average optimizer config, if specified, ema + optimizer will be used. + learning_rate: learning rate oneof config. + warmup: warmup oneof config. + """ + optimizer: OptimizerConfig = OptimizerConfig() + ema: Optional[opt_cfg.EMAConfig] = None + learning_rate: LrConfig = LrConfig() + warmup: WarmupConfig = WarmupConfig() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/configs/optimization_config_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/configs/optimization_config_test.py new file mode 100644 index 0000000000000000000000000000000000000000..02b99f592e9ba4f66ccd9e906eee5158b2b1b13e --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/configs/optimization_config_test.py @@ -0,0 +1,59 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for optimization_config.py.""" + +import tensorflow as tf + +from official.modeling.optimization.configs import learning_rate_config as lr_cfg +from official.modeling.optimization.configs import optimization_config +from official.modeling.optimization.configs import optimizer_config as opt_cfg + + +class OptimizerConfigTest(tf.test.TestCase): + + def test_no_optimizer(self): + optimizer = optimization_config.OptimizationConfig({}).optimizer.get() + self.assertIsNone(optimizer) + + def test_no_lr_schedule(self): + lr = optimization_config.OptimizationConfig({}).learning_rate.get() + self.assertIsNone(lr) + + def test_no_warmup_schedule(self): + warmup = optimization_config.OptimizationConfig({}).warmup.get() + self.assertIsNone(warmup) + + def test_config(self): + opt_config = optimization_config.OptimizationConfig({ + 'optimizer': { + 'type': 'sgd', + 'sgd': {} # default config + }, + 'learning_rate': { + 'type': 'polynomial', + 'polynomial': {} + }, + 'warmup': { + 'type': 'linear' + } + }) + self.assertEqual(opt_config.optimizer.get(), opt_cfg.SGDConfig()) + self.assertEqual(opt_config.learning_rate.get(), + lr_cfg.PolynomialLrConfig()) + self.assertEqual(opt_config.warmup.get(), lr_cfg.LinearWarmupConfig()) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/configs/optimizer_config.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/configs/optimizer_config.py new file mode 100644 index 0000000000000000000000000000000000000000..a4696d26548b491d1131211e418a5f0468411291 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/configs/optimizer_config.py @@ -0,0 +1,268 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dataclasses for optimizer configs.""" +from typing import List, Optional + +import dataclasses +from official.modeling.hyperparams import base_config + + +@dataclasses.dataclass +class BaseOptimizerConfig(base_config.Config): + """Base optimizer config. + + Attributes: + clipnorm: float >= 0 or None. If not None, Gradients will be clipped when + their L2 norm exceeds this value. + clipvalue: float >= 0 or None. If not None, Gradients will be clipped when + their absolute value exceeds this value. + global_clipnorm: float >= 0 or None. If not None, gradient of all weights is + clipped so that their global norm is no higher than this value + """ + clipnorm: Optional[float] = None + clipvalue: Optional[float] = None + global_clipnorm: Optional[float] = None + + +@dataclasses.dataclass +class SGDConfig(BaseOptimizerConfig): + """Configuration for SGD optimizer. + + The attributes for this class matches the arguments of tf.keras.optimizer.SGD. + + Attributes: + name: name of the optimizer. + decay: decay rate for SGD optimizer. + nesterov: nesterov for SGD optimizer. + momentum: momentum for SGD optimizer. + """ + name: str = "SGD" + decay: float = 0.0 + nesterov: bool = False + momentum: float = 0.0 + + +@dataclasses.dataclass +class RMSPropConfig(BaseOptimizerConfig): + """Configuration for RMSProp optimizer. + + The attributes for this class matches the arguments of + tf.keras.optimizers.RMSprop. + + Attributes: + name: name of the optimizer. + rho: discounting factor for RMSprop optimizer. + momentum: momentum for RMSprop optimizer. + epsilon: epsilon value for RMSprop optimizer, help with numerical stability. + centered: Whether to normalize gradients or not. + """ + name: str = "RMSprop" + rho: float = 0.9 + momentum: float = 0.0 + epsilon: float = 1e-7 + centered: bool = False + + +@dataclasses.dataclass +class AdagradConfig(BaseOptimizerConfig): + """Configuration for Adagrad optimizer. + + The attributes of this class match the arguments of + tf.keras.optimizer.Adagrad. + + Attributes: + name: name of the optimizer. + initial_accumulator_value: A floating point value. Starting value for the + accumulators, must be non-negative. + epsilon: A small floating point value to avoid zero denominator. + """ + name: str = "Adagrad" + initial_accumulator_value: float = 0.1 + epsilon: float = 1e-07 + + +@dataclasses.dataclass +class AdamConfig(BaseOptimizerConfig): + """Configuration for Adam optimizer. + + The attributes for this class matches the arguments of + tf.keras.optimizer.Adam. + + Attributes: + name: name of the optimizer. + beta_1: decay rate for 1st order moments. + beta_2: decay rate for 2st order moments. + epsilon: epsilon value used for numerical stability in Adam optimizer. + amsgrad: boolean. Whether to apply AMSGrad variant of this algorithm from + the paper "On the Convergence of Adam and beyond". + """ + name: str = "Adam" + beta_1: float = 0.9 + beta_2: float = 0.999 + epsilon: float = 1e-07 + amsgrad: bool = False + + +@dataclasses.dataclass +class AdamWeightDecayConfig(BaseOptimizerConfig): + """Configuration for Adam optimizer with weight decay. + + Attributes: + name: name of the optimizer. + beta_1: decay rate for 1st order moments. + beta_2: decay rate for 2st order moments. + epsilon: epsilon value used for numerical stability in the optimizer. + amsgrad: boolean. Whether to apply AMSGrad variant of this algorithm from + the paper "On the Convergence of Adam and beyond". + weight_decay_rate: float. Weight decay rate. Default to 0. + include_in_weight_decay: list[str], or None. List of weight names to include + in weight decay. + exclude_from_weight_decay: list[str], or None. List of weight names to not + include in weight decay. + gradient_clip_norm: A positive float. Clips the gradients to this maximum + L2-norm. Default to 1.0. + """ + name: str = "AdamWeightDecay" + beta_1: float = 0.9 + beta_2: float = 0.999 + epsilon: float = 1e-07 + amsgrad: bool = False + weight_decay_rate: float = 0.0 + include_in_weight_decay: Optional[List[str]] = None + exclude_from_weight_decay: Optional[List[str]] = None + gradient_clip_norm: float = 1.0 + + +@dataclasses.dataclass +class LAMBConfig(BaseOptimizerConfig): + """Configuration for LAMB optimizer. + + The attributes for this class matches the arguments of + tensorflow_addons.optimizers.LAMB. + + Attributes: + name: name of the optimizer. + beta_1: decay rate for 1st order moments. + beta_2: decay rate for 2st order moments. + epsilon: epsilon value used for numerical stability in LAMB optimizer. + weight_decay_rate: float. Weight decay rate. Default to 0. + exclude_from_weight_decay: List of regex patterns of variables excluded from + weight decay. Variables whose name contain a substring matching the + pattern will be excluded. + exclude_from_layer_adaptation: List of regex patterns of variables excluded + from layer adaptation. Variables whose name contain a substring matching + the pattern will be excluded. + """ + name: str = "LAMB" + beta_1: float = 0.9 + beta_2: float = 0.999 + epsilon: float = 1e-6 + weight_decay_rate: float = 0.0 + exclude_from_weight_decay: Optional[List[str]] = None + exclude_from_layer_adaptation: Optional[List[str]] = None + + +@dataclasses.dataclass +class EMAConfig(BaseOptimizerConfig): + """Exponential moving average optimizer config. + + Attributes: + name: 'str', name of the optimizer. + trainable_weights_only: 'bool', if True, only model trainable weights will + be updated. Otherwise, all model weights will be updated. This mainly + affects batch normalization parameters. + average_decay: 'float', average decay value. + start_step: 'int', start step to apply moving average. + dynamic_decay: 'bool', whether to apply dynamic decay or not. + """ + name: str = "ExponentialMovingAverage" + trainable_weights_only: bool = True + average_decay: float = 0.99 + start_step: int = 0 + dynamic_decay: bool = True + + +@dataclasses.dataclass +class LARSConfig(BaseOptimizerConfig): + """Layer-wise adaptive rate scaling config. + + Attributes: + name: 'str', name of the optimizer. + momentum: `float` hyperparameter >= 0 that accelerates gradient descent in + the relevant direction and dampens oscillations. Defaults to 0.9. + eeta: `float` LARS coefficient as used in the paper. Default set to LARS + coefficient from the paper. (eeta / weight_decay) determines the highest + scaling factor in LARS.. + weight_decay_rate: `float` for weight decay. + nesterov: 'boolean' for whether to use nesterov momentum. + classic_momentum: `boolean` for whether to use classic (or popular) + momentum. The learning rate is applied during momentum update in classic + momentum, but after momentum for popular momentum. + exclude_from_weight_decay: A list of `string` for variable screening, if any + of the string appears in a variable's name, the variable will be excluded + for computing weight decay. For example, one could specify the list like + ['batch_normalization', 'bias'] to exclude BN and bias from weight decay. + exclude_from_layer_adaptation: Similar to exclude_from_weight_decay, but for + layer adaptation. If it is None, it will be defaulted the same as + exclude_from_weight_decay. + """ + name: str = "LARS" + momentum: float = 0.9 + eeta: float = 0.001 + weight_decay_rate: float = 0.0 + nesterov: bool = False + classic_momentum: bool = True + exclude_from_weight_decay: Optional[List[str]] = None + exclude_from_layer_adaptation: Optional[List[str]] = None + + +@dataclasses.dataclass +class SLIDEConfig(BaseOptimizerConfig): + """Configuration for SLIDE optimizer. + + Details coming soon. + """ + name: str = "SLIDE" + beta_1: float = 0.9 + beta_2: float = 0.999 + epsilon: float = 1e-6 + weight_decay_rate: float = 0.0 + weight_decay_type: str = "inner" + exclude_from_weight_decay: Optional[List[str]] = None + exclude_from_layer_adaptation: Optional[List[str]] = None + include_in_sparse_layer_adaptation: Optional[List[str]] = None + sparse_layer_learning_rate: float = 0.1 + do_gradient_rescaling: bool = True + norm_type: str = "layer" + ratio_clip_norm: float = 1e5 + + +@dataclasses.dataclass +class AdafactorConfig(BaseOptimizerConfig): + """Configuration for Adafactor optimizer. + + The attributes for this class matches the arguments of the Adafactor + implementation. + """ + name: str = "Adafactor" + factored: bool = True + multiply_by_parameter_scale: bool = True + beta1: Optional[float] = None + decay_rate: float = 0.8 + step_offset: int = 0 + clipping_threshold: float = 1.0 + min_dim_size_to_factor: int = 128 + epsilon1: float = 1e-30 + epsilon2: float = 1e-3 diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/ema_optimizer.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/ema_optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..c4f44d7124d888a7b0403442b1f57385d820e789 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/ema_optimizer.py @@ -0,0 +1,255 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exponential moving average optimizer.""" + +from typing import List, Optional, Text + +import tensorflow as tf + +# pylint: disable=protected-access + + +class ExponentialMovingAverage(tf.keras.optimizers.Optimizer): + """Optimizer that computes an exponential moving average of the variables. + + Empirically it has been found that using the moving average of the trained + parameters of a deep network is better than using its trained parameters + directly. This optimizer allows you to compute this moving average and swap + the variables at save time so that any code outside of the training loop + will use by default the average values instead of the original ones. + + Example of usage for training: + ```python + opt = tf.keras.optimizers.SGD(learning_rate) + opt = ExponentialMovingAverage(opt) + + opt.shadow_copy(model) + ``` + + At test time, swap the shadow variables to evaluate on the averaged weights: + ```python + opt.swap_weights() + # Test eval the model here + opt.swap_weights() + ``` + """ + + def __init__(self, + optimizer: tf.keras.optimizers.Optimizer, + trainable_weights_only: bool = True, + average_decay: float = 0.99, + start_step: int = 0, + dynamic_decay: bool = True, + name: Text = 'ExponentialMovingAverage', + **kwargs): + """Construct a new ExponentialMovingAverage optimizer. + + Args: + optimizer: `tf.keras.optimizers.Optimizer` that will be + used to compute and apply gradients. + trainable_weights_only: 'bool', if True, only model trainable weights will + be updated. Otherwise, all model weights will be updated. This mainly + affects batch normalization parameters. + average_decay: float. Decay to use to maintain the moving averages + of trained variables. + start_step: int. What step to start the moving average. + dynamic_decay: bool. Whether to change the decay based on the number + of optimizer updates. Decay will start at 0.1 and gradually increase + up to `average_decay` after each optimizer update. This behavior is + similar to `tf.train.ExponentialMovingAverage` in TF 1.x. + name: Optional name for the operations created when applying + gradients. Defaults to "moving_average". + **kwargs: keyword arguments. Allowed to be {`clipnorm`, + `clipvalue`, `lr`, `decay`}. + """ + super().__init__(name, **kwargs) + self._average_decay = average_decay + self._trainable_weights_only = trainable_weights_only + self._start_step = tf.constant(start_step, tf.float32) + self._dynamic_decay = dynamic_decay + self._optimizer = optimizer + self._track_trackable(self._optimizer, 'base_optimizer') + self._average_weights = None + self._model_weights = None + + def shadow_copy(self, model: tf.keras.Model): + """Creates shadow variables for the given model weights.""" + + if self._trainable_weights_only: + self._model_weights = model.trainable_variables + else: + self._model_weights = model.variables + for var in self._model_weights: + self.add_slot(var, 'average', initializer='zeros') + + self._average_weights = [ + self.get_slot(var, 'average') for var in self._model_weights + ] + + @property + def has_shadow_copy(self): + """Whether this optimizer has created shadow variables.""" + return self._model_weights is not None and self._average_weights is not None + + def _create_slots(self, var_list): + self._optimizer._create_slots(var_list=var_list) # pylint: disable=protected-access + + def apply_gradients(self, grads_and_vars, name: Optional[Text] = None): + result = self._optimizer.apply_gradients(grads_and_vars, name) + self.update_average(self.iterations) + return result + + @tf.function + def update_average(self, step: tf.Tensor): + step = tf.cast(step, tf.float32) + if step < self._start_step: + decay = tf.constant(0., tf.float32) + elif self._dynamic_decay: + decay = step - self._start_step + decay = tf.minimum(self._average_decay, (1. + decay) / (10. + decay)) + else: + decay = self._average_decay + + def _apply_moving(v_moving, v_normal): + diff = v_moving - v_normal + v_moving.assign_sub(tf.cast(1. - decay, v_moving.dtype) * diff) + return v_moving + + def _update(strategy, v_moving_and_v_normal): + for v_moving, v_normal in v_moving_and_v_normal: + strategy.extended.update(v_moving, _apply_moving, args=(v_normal,)) + + ctx = tf.distribute.get_replica_context() + return ctx.merge_call(_update, args=(zip(self._average_weights, + self._model_weights),)) + + def swap_weights(self): + """Swap the average and moving weights. + + This is a convenience method to allow one to evaluate the averaged weights + at test time. Loads the weights stored in `self._average` into the model, + keeping a copy of the original model weights. Swapping twice will return + the original weights. + """ + if tf.distribute.in_cross_replica_context(): + strategy = tf.distribute.get_strategy() + strategy.run(self._swap_weights, args=()) + else: + raise ValueError('Swapping weights must occur under a ' + 'tf.distribute.Strategy') + + @tf.function + def _swap_weights(self): + def fn_0(a, b): + a.assign_add(b) + return a + def fn_1(b, a): + b.assign(a - b) + return b + def fn_2(a, b): + a.assign_sub(b) + return a + + def swap(strategy, a_and_b): + """Swap `a` and `b` and mirror to all devices.""" + for a, b in a_and_b: + strategy.extended.update(a, fn_0, args=(b,)) # a = a + b + strategy.extended.update(b, fn_1, args=(a,)) # b = a - b + strategy.extended.update(a, fn_2, args=(b,)) # a = a - b + + ctx = tf.distribute.get_replica_context() + return ctx.merge_call( + swap, args=(zip(self._average_weights, self._model_weights),)) + + def assign_average_vars(self, var_list: List[tf.Variable]): + """Assign variables in var_list with their respective averages. + + Args: + var_list: List of model variables to be assigned to their average. + Returns: + assign_op: The op corresponding to the assignment operation of + variables to their average. + """ + assign_op = tf.group([ + var.assign(self.get_slot(var, 'average')) for var in var_list + if var.trainable + ]) + return assign_op + + def _create_hypers(self): + self._optimizer._create_hypers() # pylint: disable=protected-access + + def _prepare(self, var_list): + return self._optimizer._prepare(var_list=var_list) # pylint: disable=protected-access + + @property + def iterations(self): + return self._optimizer.iterations + + @iterations.setter + def iterations(self, variable): + self._optimizer.iterations = variable + + @property + def weights(self): + # return self._weights + self._optimizer.weights + return self._optimizer.weights + + def variables(self): + return self._weights + [self.iterations] + + @property + def lr(self): + return self._optimizer._get_hyper('learning_rate') + + @lr.setter + def lr(self, lr): + self._optimizer._set_hyper('learning_rate', lr) + + @property + def learning_rate(self): + return self._optimizer._get_hyper('learning_rate') + + @learning_rate.setter + def learning_rate(self, learning_rate): # pylint: disable=redefined-outer-name + self._optimizer._set_hyper('learning_rate', learning_rate) + + def _resource_apply_dense(self, grad, var): + return self._optimizer._resource_apply_dense(grad, var) + + def _resource_apply_sparse(self, grad, var, indices): + return self._optimizer._resource_apply_sparse(grad, var, indices) + + def _resource_apply_sparse_duplicate_indices(self, grad, var, indices): + return self._optimizer._resource_apply_sparse_duplicate_indices( + grad, var, indices) + + def get_config(self): + config = { + 'optimizer': tf.keras.optimizers.serialize(self._optimizer), + 'average_decay': self._average_decay, + 'start_step': self._start_step, + 'dynamic_decay': self._dynamic_decay, + } + base_config = super(ExponentialMovingAverage, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + @classmethod + def from_config(cls, config, custom_objects=None): + optimizer = tf.keras.optimizers.deserialize( + config.pop('optimizer'), + custom_objects=custom_objects, + ) + return cls(optimizer, **config) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/lars_optimizer.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/lars_optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..ac15042756c02c3d3e2da22419cac2e04522b57e --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/lars_optimizer.py @@ -0,0 +1,186 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Layer-wise adaptive rate scaling optimizer.""" +import re +from typing import Text, List, Optional + +import tensorflow as tf + + +# pylint: disable=protected-access + + +class LARS(tf.keras.optimizers.Optimizer): + """Layer-wise Adaptive Rate Scaling for large batch training. + + Introduced by "Large Batch Training of Convolutional Networks" by Y. You, + I. Gitman, and B. Ginsburg. (https://arxiv.org/abs/1708.03888) + """ + + def __init__(self, + learning_rate: float = 0.01, + momentum: float = 0.9, + weight_decay_rate: float = 0.0, + eeta: float = 0.001, + nesterov: bool = False, + classic_momentum: bool = True, + exclude_from_weight_decay: Optional[List[Text]] = None, + exclude_from_layer_adaptation: Optional[List[Text]] = None, + name: Text = "LARS", + **kwargs): + """Constructs a LARSOptimizer. + + Args: + learning_rate: `float` for learning rate. Defaults to 0.01. + momentum: `float` hyperparameter >= 0 that accelerates gradient descent + in the relevant direction and dampens oscillations. Defaults to 0.9. + weight_decay_rate: `float` for weight decay. + eeta: `float` LARS coefficient as used in the paper. Default set to LARS + coefficient from the paper. (eeta / weight_decay) determines the + highest scaling factor in LARS.. + nesterov: 'boolean' for whether to use nesterov momentum. + classic_momentum: `boolean` for whether to use classic (or popular) + momentum. The learning rate is applied during momentum update in + classic momentum, but after momentum for popular momentum. + exclude_from_weight_decay: A list of `string` for variable screening, if + any of the string appears in a variable's name, the variable will be + excluded for computing weight decay. For example, one could specify + the list like ['batch_normalization', 'bias'] to exclude BN and bias + from weight decay. + exclude_from_layer_adaptation: Similar to exclude_from_weight_decay, but + for layer adaptation. If it is None, it will be defaulted the same as + exclude_from_weight_decay. + name: `Text` as optional name for the operations created when applying + gradients. Defaults to "LARS". + **kwargs: keyword arguments. Allowed to be {`clipnorm`, `clipvalue`, `lr`, + `decay`}. `clipnorm` is clip gradients by norm; `clipvalue` is clip + gradients by value, `decay` is included for backward compatibility to + allow time inverse decay of learning rate. `lr` is included for + backward compatibility, recommended to use `learning_rate` instead. + """ + super(LARS, self).__init__(name, **kwargs) + + self._set_hyper("learning_rate", learning_rate) + self._set_hyper("decay", self._initial_decay) + self.momentum = momentum + self.weight_decay_rate = weight_decay_rate + self.eeta = eeta + self.nesterov = nesterov + self.classic_momentum = classic_momentum + self.exclude_from_weight_decay = exclude_from_weight_decay + # exclude_from_layer_adaptation is set to exclude_from_weight_decay if the + # arg is None. + if exclude_from_layer_adaptation: + self.exclude_from_layer_adaptation = exclude_from_layer_adaptation + else: + self.exclude_from_layer_adaptation = exclude_from_weight_decay + + def _create_slots(self, var_list): + for v in var_list: + self.add_slot(v, "momentum") + + def _resource_apply_dense(self, grad, param, apply_state=None): + if grad is None or param is None: + return tf.no_op() + + var_device, var_dtype = param.device, param.dtype.base_dtype + coefficients = ((apply_state or {}).get((var_device, var_dtype)) or + self._fallback_apply_state(var_device, var_dtype)) + learning_rate = coefficients["lr_t"] + + param_name = param.name + + v = self.get_slot(param, "momentum") + + if self._use_weight_decay(param_name): + grad += self.weight_decay_rate * param + + if self.classic_momentum: + trust_ratio = 1.0 + if self._do_layer_adaptation(param_name): + w_norm = tf.norm(param, ord=2) + g_norm = tf.norm(grad, ord=2) + trust_ratio = tf.where( + tf.greater(w_norm, 0), + tf.where(tf.greater(g_norm, 0), (self.eeta * w_norm / g_norm), 1.0), + 1.0) + scaled_lr = learning_rate * trust_ratio + + next_v = tf.multiply(self.momentum, v) + scaled_lr * grad + if self.nesterov: + update = tf.multiply(self.momentum, next_v) + scaled_lr * grad + else: + update = next_v + next_param = param - update + else: + next_v = tf.multiply(self.momentum, v) + grad + if self.nesterov: + update = tf.multiply(self.momentum, next_v) + grad + else: + update = next_v + + trust_ratio = 1.0 + if self._do_layer_adaptation(param_name): + w_norm = tf.norm(param, ord=2) + v_norm = tf.norm(update, ord=2) + trust_ratio = tf.where( + tf.greater(w_norm, 0), + tf.where(tf.greater(v_norm, 0), (self.eeta * w_norm / v_norm), 1.0), + 1.0) + scaled_lr = trust_ratio * learning_rate + next_param = param - scaled_lr * update + + return tf.group(*[ + param.assign(next_param, use_locking=False), + v.assign(next_v, use_locking=False) + ]) + + def _resource_apply_sparse(self, grad, handle, indices, apply_state): + raise NotImplementedError("Applying sparse gradients is not implemented.") + + def _use_weight_decay(self, param_name): + """Whether to use L2 weight decay for `param_name`.""" + if not self.weight_decay_rate: + return False + if self.exclude_from_weight_decay: + for r in self.exclude_from_weight_decay: + if re.search(r, param_name) is not None: + return False + return True + + def _do_layer_adaptation(self, param_name): + """Whether to do layer-wise learning rate adaptation for `param_name`.""" + if self.exclude_from_layer_adaptation: + for r in self.exclude_from_layer_adaptation: + if re.search(r, param_name) is not None: + return False + return True + + def get_config(self): + config = super(LARS, self).get_config() + config.update({ + "learning_rate": self._serialize_hyperparameter("learning_rate"), + "decay": self._serialize_hyperparameter("decay"), + "momentum": self.momentum, + "classic_momentum": self.classic_momentum, + "weight_decay_rate": self.weight_decay_rate, + "eeta": self.eeta, + "nesterov": self.nesterov, + }) + return config + + @classmethod + def from_config(cls, config, custom_objects=None): + return cls(**config) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/lr_schedule.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/lr_schedule.py new file mode 100644 index 0000000000000000000000000000000000000000..5f62f10b11501003c3f066c748811fc19f5882ec --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/lr_schedule.py @@ -0,0 +1,496 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Learning rate schedule classes.""" + +import math +from typing import Mapping, Any, Union, Optional + +import tensorflow as tf + + +def _make_offset_wrapper(new_class_name: str, base_lr_class): + """Generates a offset wrapper of learning rate schedule. + + It will returns a subclass of the the `base_lr_class`, the subclass takes an + `offset` argument in the constructor. When the new class instance is called, + the behavior is: + new_class_object(step) = base_lr_class_object(step - offset) + + Example: + CosineDecayWithOffset = _make_offset_wrapper( + 'CosineDecayWithOffset', tf.keras.experimental.CosineDecay) + # Use the lr: + lr = CosineDecayWithOffset(offset=100, initial_learning_rate=0.1, + decay_steps=1000) + lr(101) # equals to tf.keras.experimental.CosineDecay(...)(101-100) + + Args: + new_class_name: the name of the new class. + base_lr_class: the base learning rate schedule class. Should be subclass of + tf.keras.optimizers.schedules.LearningRateSchedule + + Returns: + A new class (subclass of the base_lr_class) that can take an offset. + """ + assert issubclass(base_lr_class, + tf.keras.optimizers.schedules.LearningRateSchedule), ( + "base_lr_class should be subclass of keras " + f"LearningRateSchedule, got {base_lr_class}") + + # pylint: disable=protected-access,pointless-statement + def offset_learning_rate_init(self, offset=0, **kwargs): + """Construct learning rate schedule object. + + When this object is called, its behavior is + self.__call__(step) == base_lr_class.__call__(step - offset) + Args: + self: this object. + offset: The offset when computing the learning rate schedule. + **kwargs: Pass through to base learning rate class constructor. + """ + base_lr_class.__init__(self, **kwargs) + self._offset = offset + + def offset_learning_rate_call(self, step): + step = tf.cast(step - self._offset, tf.float32) + return base_lr_class.__call__(self, step) + + # pylint: enable=protected-access,pointless-statement + + return type( + new_class_name, (base_lr_class,), { + "base_lr_class": base_lr_class, + "__init__": offset_learning_rate_init, + "__call__": offset_learning_rate_call + }) + + +PiecewiseConstantDecayWithOffset = _make_offset_wrapper( + "PiecewiseConstantDecayWithOffset", + tf.keras.optimizers.schedules.PiecewiseConstantDecay) +PolynomialDecayWithOffset = _make_offset_wrapper( + "PolynomialDecayWithOffset", tf.keras.optimizers.schedules.PolynomialDecay) +ExponentialDecayWithOffset = _make_offset_wrapper( + "ExponentialDecayWithOffset", + tf.keras.optimizers.schedules.ExponentialDecay) +CosineDecayWithOffset = _make_offset_wrapper("CosineDecayWithOffset", + tf.keras.experimental.CosineDecay) + + +class LinearWarmup(tf.keras.optimizers.schedules.LearningRateSchedule): + """Linear warmup schedule.""" + + def __init__(self, + after_warmup_lr_sched: Union[ + tf.keras.optimizers.schedules.LearningRateSchedule, float], + warmup_steps: int, + warmup_learning_rate: float, + name: Optional[str] = None): + """Add linear warmup schedule to a learning rate schedule. + + warmup_lr is the initial learning rate, the final learning rate of the + init_warmup period is the initial learning rate of lr_schedule in use. + The learning rate at each step linearly increased according to the following + formula: + learning_rate = warmup_lr + step / warmup_steps + * (final_warmup_lr - warmup_lr). + Using warmup overrides the learning rate schedule by the number of warmup + steps. + + Args: + after_warmup_lr_sched: tf.keras.optimizers.schedules .LearningRateSchedule + or a constant. + warmup_steps: Number of the warmup steps. + warmup_learning_rate: Initial learning rate for the warmup. + name: Optional, name of warmup schedule. + """ + super().__init__() + self._name = name + self._after_warmup_lr_sched = after_warmup_lr_sched + self._warmup_steps = warmup_steps + self._init_warmup_lr = warmup_learning_rate + if isinstance(after_warmup_lr_sched, + tf.keras.optimizers.schedules.LearningRateSchedule): + self._final_warmup_lr = after_warmup_lr_sched(warmup_steps) + else: + self._final_warmup_lr = tf.cast(after_warmup_lr_sched, dtype=tf.float32) + + def __call__(self, step: int): + + global_step = tf.cast(step, dtype=tf.float32) + + linear_warmup_lr = ( + self._init_warmup_lr + global_step / self._warmup_steps * + (self._final_warmup_lr - self._init_warmup_lr)) + + if isinstance(self._after_warmup_lr_sched, + tf.keras.optimizers.schedules.LearningRateSchedule): + after_warmup_lr = self._after_warmup_lr_sched(step) + else: + after_warmup_lr = tf.cast(self._after_warmup_lr_sched, dtype=tf.float32) + + lr = tf.cond(global_step < self._warmup_steps, + lambda: linear_warmup_lr, + lambda: after_warmup_lr) + return lr + + def get_config(self) -> Mapping[str, Any]: + if isinstance(self._after_warmup_lr_sched, + tf.keras.optimizers.schedules.LearningRateSchedule): + config = { + "after_warmup_lr_sched": self._after_warmup_lr_sched.get_config()} # pytype: disable=attribute-error + else: + config = {"after_warmup_lr_sched": self._after_warmup_lr_sched} # pytype: disable=attribute-error + + config.update({ + "warmup_steps": self._warmup_steps, + "warmup_learning_rate": self._init_warmup_lr, + "name": self._name + }) + return config + + +class PolynomialWarmUp(tf.keras.optimizers.schedules.LearningRateSchedule): + """Applies polynomial warmup schedule on a given learning rate decay schedule.""" + + def __init__(self, + after_warmup_lr_sched: Union[ + tf.keras.optimizers.schedules.LearningRateSchedule, float], + warmup_steps: int, + power: float = 1.0, + name: str = "PolynomialWarmup"): + super().__init__() + if isinstance(after_warmup_lr_sched, + tf.keras.optimizers.schedules.LearningRateSchedule): + self._initial_learning_rate = after_warmup_lr_sched(warmup_steps) + else: + self._initial_learning_rate = tf.cast( + after_warmup_lr_sched, dtype=tf.float32) + + self._warmup_steps = warmup_steps + self._power = power + self._after_warmup_lr_sched = after_warmup_lr_sched + self._name = name + + def __call__(self, step): + with tf.name_scope(self._name or "PolynomialWarmUp") as name: + # Implements polynomial warmup. i.e., if global_step < warmup_steps, the + # learning rate will be `global_step/num_warmup_steps * init_lr`. + global_step_float = tf.cast(step, tf.float32) + warmup_steps_float = tf.cast(self._warmup_steps, tf.float32) + + if self._warmup_steps <= 0: + warmup_percent_done = 1.0 + else: + # A zero `step` may cause Inf. So make `step` positive. + step_non_zero = tf.math.maximum(global_step_float, 1.0) + warmup_percent_done = step_non_zero / warmup_steps_float + + warmup_learning_rate = ( + self._initial_learning_rate * + tf.math.pow(warmup_percent_done, self._power)) + + if isinstance(self._after_warmup_lr_sched, + tf.keras.optimizers.schedules.LearningRateSchedule): + after_warmup_lr = self._after_warmup_lr_sched(step) + else: + after_warmup_lr = tf.cast(self._after_warmup_lr_sched, dtype=tf.float32) + + return tf.cond( + global_step_float < warmup_steps_float, + lambda: warmup_learning_rate, + lambda: after_warmup_lr, + name=name) + + def get_config(self) -> Mapping[str, Any]: + if isinstance(self._after_warmup_lr_sched, + tf.keras.optimizers.schedules.LearningRateSchedule): + config = { + "after_warmup_lr_sched": self._after_warmup_lr_sched.get_config()} # pytype: disable=attribute-error + else: + config = {"after_warmup_lr_sched": self._after_warmup_lr_sched} # pytype: disable=attribute-error + + config.update({ + "warmup_steps": self._warmup_steps, + "power": self._power, + "name": self._name + }) + return config + + +class DirectPowerDecay(tf.keras.optimizers.schedules.LearningRateSchedule): + """Learning rate schedule follows lr * (step)^power.""" + + def __init__(self, + initial_learning_rate: float, + power: float = 1.0, + name: str = "DirectPowerDecay"): + """Initialize configuration of the learning rate schedule. + + Args: + initial_learning_rate: The initial learning rate. + power: The order of the polynomial. + name: Optional, name of learning rate schedule. + """ + super().__init__() + self._initial_learning_rate = initial_learning_rate + self._power = power + self._name = name + + def __call__(self, step): + with tf.name_scope(self._name or "DirectPowerDecay"): + step = tf.cast(step, tf.float32) + learning_rate = self._initial_learning_rate + # A zero `step` may cause Inf. So make `step` positive. + step_non_zero = tf.math.maximum(step, 1.0) + learning_rate *= tf.math.pow(step_non_zero, self._power) + return learning_rate + + def get_config(self): + """Get the configuration of the learning rate schedule.""" + return { + "initial_learning_rate": self._initial_learning_rate, + "power": self._power, + "name": self._name, + } + + +class PowerAndLinearDecay(tf.keras.optimizers.schedules.LearningRateSchedule): + """Learning rate schedule with multiplied by linear decay at the end. + + The schedule has the following behavoir. + Let offset_step = step - offset. + 1) offset_step < 0, the actual learning rate equals initial_learning_rate. + 2) offset_step <= total_decay_steps * (1 - linear_decay_fraction), the + actual learning rate equals lr * offset_step^power. + 3) total_decay_steps * (1 - linear_decay_fraction) <= offset_step < + total_decay_steps, the actual learning rate equals lr * offset_step^power * + (total_decay_steps - offset_step) / (total_decay_steps * + linear_decay_fraction). + 4) offset_step >= total_decay_steps, the actual learning rate equals zero. + """ + + def __init__(self, + initial_learning_rate: float, + total_decay_steps: int, + power: float = 1.0, + linear_decay_fraction: float = 0.1, + offset: int = 0, + name: str = "PowerAndLinearDecay"): + """Initialize configuration of the learning rate schedule. + + Args: + initial_learning_rate: The initial learning rate. + total_decay_steps: The total number of steps for power + linear decay. + power: The order of the polynomial. + linear_decay_fraction: In the last `linear_decay_fraction` steps, the + learning rate will be multiplied by a linear decay. + offset: The offset applied to steps. + name: Optional, name of learning rate schedule. + """ + super().__init__() + self._initial_learning_rate = initial_learning_rate + self._total_decay_steps = total_decay_steps + self._power = power + self._linear_decay_fraction = linear_decay_fraction + self._offset = offset + self._name = name + + def __call__(self, step): + with tf.name_scope(self._name or "PowerAndLinearDecay"): + step = tf.cast(step - self._offset, tf.float32) + learning_rate = self._initial_learning_rate + # A zero `step` may cause Inf. So make `step` positive. + step_non_zero = tf.math.maximum(step, 1.0) + learning_rate *= tf.math.pow(step_non_zero, self._power) + if self._total_decay_steps * self._linear_decay_fraction > 0: + learning_rate *= tf.minimum( + 1.0, (self._total_decay_steps - step) / + (self._total_decay_steps * self._linear_decay_fraction)) + learning_rate = tf.maximum(0.0, learning_rate) + return learning_rate + + def get_config(self): + """Get the configuration of the learning rate schedule.""" + return { + "initial_learning_rate": self._initial_learning_rate, + "total_decay_steps": self._total_decay_steps, + "power": self._power, + "linear_decay_fraction": self._linear_decay_fraction, + "offset": self._offset, + "name": self._name, + } + + +class PowerDecayWithOffset(tf.keras.optimizers.schedules.LearningRateSchedule): + """Power learning rate decay with offset. + + Learning rate equals to `pre_offset_learning_rate` if `step` < `offset`. + Otherwise, learning rate equals to lr * (step - offset)^power. + """ + + def __init__(self, + initial_learning_rate: float, + power: float = 1.0, + offset: int = 0, + pre_offset_learning_rate: float = 1.0e6, + name: str = "PowerDecayWithOffset"): + """Initialize configuration of the learning rate schedule. + + Args: + initial_learning_rate: The initial learning rate. + power: The order of the polynomial. + offset: The offset when computing the power decay. + pre_offset_learning_rate: The maximum learning rate we'll use. + name: Optional, name of learning rate schedule. + """ + super().__init__() + self._initial_learning_rate = initial_learning_rate + self._power = power + self._offset = offset + self._pre_offset_lr = pre_offset_learning_rate + self._name = name + + def __call__(self, step): + with tf.name_scope(self._name or "PowerDecayWithOffset"): + step = tf.cast(step, tf.float32) + lr_after_offset = tf.math.pow( + tf.math.maximum(step - self._offset, 1.0), self._power) * ( + self._initial_learning_rate) + + sign = tf.cast(step > self._offset, tf.float32) + lr_combined = (1.0 - sign) * self._pre_offset_lr + sign * lr_after_offset + # Power may give infinitely large LR. So cap it with pre_offset_lr. + return tf.math.minimum(lr_combined, self._pre_offset_lr) + + def get_config(self): + """Get the configuration of the learning rate schedule.""" + return { + "initial_learning_rate": self._initial_learning_rate, + "power": self._power, + "offset": self._offset, + "pre_offset_learning_rate": self._pre_offset_lr, + "name": self._name, + } + + +class StepConsineDecayWithOffset( + tf.keras.optimizers.schedules.LearningRateSchedule): + """Stepwise cosine learning rate decay with offset. + + Learning rate is equivalent to one or more consine decay(s) starting and + ending at each interval. + + ExampleL + + ```python + boundaries: [100000, 110000] + values: [1.0, 0.5] + lr_decayed_fn = ( + lr_schedule.StepConsineDecayWithOffset( + boundaries, + values)) + ``` + + from 0 to 100000 step, it will cosine decay from 1.0 to 0.5 + from 100000 to 110000 step, it cosine decay from 0.5 to 0.0 + """ + + def __init__(self, + boundaries, + values, + offset: int = 0, + name: str = "StepConsineDecayWithOffset"): + """Initialize configuration of the learning rate schedule. + + Args: + boundaries: A list of `Tensor`s or `int`s with strictly + increasing entries, and with all elements having the same type as the + optimizer step. + values: A list of `Tensor`s or `float`s that specifies the + values for the intervals defined by `boundaries`. It should have one + more element than `boundaries`, and all elements should have the same + type. + offset: The offset when computing the power decay. + name: Optional, name of learning rate schedule. + """ + super().__init__() + self.values = values + self.boundaries = boundaries + self.offset = offset + self.name = name + + if len(self.values) < 1: + raise ValueError(f"Expect non empty {self.values}") + if len(self.boundaries) != len(self.values): + raise ValueError( + "Boundaries length is equal to learning rate levels length" + f"{len(self.boundaries)} != {len(self.values)}") + + self.total_steps = ( + [boundaries[i + 1] - boundaries[i] for i in range(len(boundaries) - 1) + ] + [0]) + + def __call__(self, global_step): + with tf.name_scope(self.name or "StepConsineDecayWithOffset"): + global_step = tf.cast(global_step - self.offset, tf.float32) + lr_levels = self.values + lr_steps = self.boundaries + level_total_steps = self.total_steps + num_levels = len(lr_levels) + + init_lr = lr_levels[0] + next_init_lr = lr_levels[1] if num_levels > 1 else 0. + + init_total_steps = level_total_steps[0] + + cosine_learning_rate = ((init_lr - next_init_lr) * (tf.cos( + tf.constant(math.pi) * (global_step) / + (init_total_steps)) + 1.0) / 2.0 + next_init_lr) + learning_rate = cosine_learning_rate + tf.compat.v1.logging.info("DEBUG lr %r next lr %r", learning_rate, + cosine_learning_rate) + tf.compat.v1.logging.info("DEBUG lr %r next lr %r inittotalstep %r", + init_lr, next_init_lr, init_total_steps) + + for i in range(1, num_levels): + next_init_lr = lr_levels[i] + next_start_step = lr_steps[i] + next_total_steps = level_total_steps[i] + next_next_init_lr = lr_levels[i + 1] if num_levels > i + 1 else 0. + + tf.compat.v1.logging.info( + "DEBUG step %r nilr %r nss %r nts %r nnilr %r", global_step, + next_init_lr, next_start_step, next_total_steps, next_next_init_lr) + next_cosine_learning_rate = ((next_init_lr - next_next_init_lr) * + (tf.cos( + tf.constant(math.pi) * + (global_step - next_start_step) / + (next_total_steps)) + 1.0) / 2.0 + + next_next_init_lr) + learning_rate = tf.where(global_step >= next_start_step, + next_cosine_learning_rate, learning_rate) + tf.compat.v1.logging.info("DEBUG lr %r next lr %r", learning_rate, + next_cosine_learning_rate) + + return learning_rate + + def get_config(self): + return { + "boundaries": self.boundaries, + "values": self.values, + "offset": self.offset, + "name": self.name + } diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/lr_schedule_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/lr_schedule_test.py new file mode 100644 index 0000000000000000000000000000000000000000..bafd8be1fad277cfd66579e6336e23493337730a --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/lr_schedule_test.py @@ -0,0 +1,109 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for lr_schedule.""" +from absl.testing import parameterized +import tensorflow as tf + +from official.modeling.optimization import lr_schedule + + +class PowerAndLinearDecayTest(tf.test.TestCase, parameterized.TestCase): + + @parameterized.named_parameters( + dict( + testcase_name='power_only', + init_lr=1.0, + power=-1.0, + linear_decay_fraction=0.0, + total_decay_steps=100, + offset=0, + expected=[[0, 1.0], [1, 1.0], [40, 1. / 40.], [60, 1. / 60], + [100, 1. / 100]]), + dict( + testcase_name='linear_only', + init_lr=1.0, + power=0.0, + linear_decay_fraction=1.0, + total_decay_steps=100, + offset=0, + expected=[[0, 1.0], [1, 0.99], [40, 0.6], [60, 0.4], [100, 0.0]]), + dict( + testcase_name='general', + init_lr=1.0, + power=-1.0, + linear_decay_fraction=0.5, + total_decay_steps=100, + offset=0, + expected=[[0, 1.0], [1, 1.0], [40, 1. / 40.], + [60, 1. / 60. * 0.8], [100, 0.0]]), + dict( + testcase_name='offset', + init_lr=1.0, + power=-1.0, + linear_decay_fraction=0.5, + total_decay_steps=100, + offset=90, + expected=[[0, 1.0], [90, 1.0], [91, 1.0], [130, 1. / 40.], + [150, 1. / 60. * 0.8], [190, 0.0], [200, 0.0]]), + ) + def test_power_linear_lr_schedule(self, init_lr, power, linear_decay_fraction, + total_decay_steps, offset, expected): + lr = lr_schedule.PowerAndLinearDecay( + initial_learning_rate=init_lr, + power=power, + linear_decay_fraction=linear_decay_fraction, + total_decay_steps=total_decay_steps, + offset=offset) + for step, value in expected: + self.assertAlmostEqual(lr(step).numpy(), value) + + +class OffsetLearningRateTest(tf.test.TestCase, parameterized.TestCase): + + @parameterized.parameters( + dict(class_name=lr_schedule.PiecewiseConstantDecayWithOffset), + dict(class_name=lr_schedule.PolynomialDecayWithOffset), + dict(class_name=lr_schedule.ExponentialDecayWithOffset), + dict(class_name=lr_schedule.CosineDecayWithOffset), + ) + def test_generated_docstring(self, class_name): + self.assertNotEmpty(class_name.__init__.__doc__) + + @parameterized.parameters( + dict( + class_name=lr_schedule.PiecewiseConstantDecayWithOffset, + kwarg=dict(boundaries=[50, 80], values=[1.0, 0.5, 0.1])), + dict( + class_name=lr_schedule.PolynomialDecayWithOffset, + kwarg=dict(initial_learning_rate=1.0, decay_steps=100)), + dict( + class_name=lr_schedule.ExponentialDecayWithOffset, + kwarg=dict( + initial_learning_rate=1.0, decay_steps=100, decay_rate=0.5)), + dict( + class_name=lr_schedule.CosineDecayWithOffset, + kwarg=dict(initial_learning_rate=1.0, decay_steps=100)), + ) + def test_offset(self, class_name, kwarg): + offset = 10 + offset_lr = class_name(offset=offset, **kwarg) + base_lr = class_name.base_lr_class(**kwarg) + self.assertIsInstance(offset_lr, class_name) + for step in range(10, 101, 10): + self.assertEqual(offset_lr(step), base_lr(step - offset)) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/optimizer_factory.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/optimizer_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..4f5b8929b0d87bcb6a8023e85e49df238cd3228e --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/optimizer_factory.py @@ -0,0 +1,208 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Optimizer factory class.""" +from typing import Callable, Optional, Union, List, Tuple + +import gin +import tensorflow as tf +import tensorflow_addons.optimizers as tfa_optimizers + +from official.modeling.optimization import slide_optimizer +from official.modeling.optimization import adafactor_optimizer +from official.modeling.optimization import ema_optimizer +from official.modeling.optimization import lars_optimizer +from official.modeling.optimization import lr_schedule +from official.modeling.optimization.configs import optimization_config as opt_cfg +from official.nlp import optimization as nlp_optimization + +OPTIMIZERS_CLS = { + 'sgd': tf.keras.optimizers.SGD, + 'adam': tf.keras.optimizers.Adam, + 'adamw': nlp_optimization.AdamWeightDecay, + 'lamb': tfa_optimizers.LAMB, + 'rmsprop': tf.keras.optimizers.RMSprop, + 'lars': lars_optimizer.LARS, + 'adagrad': tf.keras.optimizers.Adagrad, + 'slide': slide_optimizer.SLIDE, + 'adafactor': adafactor_optimizer.Adafactor, +} + +LR_CLS = { + 'stepwise': lr_schedule.PiecewiseConstantDecayWithOffset, + 'polynomial': lr_schedule.PolynomialDecayWithOffset, + 'exponential': lr_schedule.ExponentialDecayWithOffset, + 'cosine': lr_schedule.CosineDecayWithOffset, + 'power': lr_schedule.DirectPowerDecay, + 'power_linear': lr_schedule.PowerAndLinearDecay, + 'power_with_offset': lr_schedule.PowerDecayWithOffset, + 'step_cosine_with_offset': lr_schedule.StepConsineDecayWithOffset, +} + +WARMUP_CLS = { + 'linear': lr_schedule.LinearWarmup, + 'polynomial': lr_schedule.PolynomialWarmUp +} + + +def register_optimizer_cls( + key: str, optimizer_config_cls: tf.keras.optimizers.Optimizer): + """Register customize optimizer cls. + + The user will still need to subclass data classes in + configs.optimization_config to be used with OptimizerFactory. + + Args: + key: A string to that the optimizer_config_cls is registered with. + optimizer_config_cls: A class which inherits tf.keras.optimizers.Optimizer. + """ + if key in OPTIMIZERS_CLS: + raise ValueError('%s already registered in OPTIMIZER_CLS.' % key) + OPTIMIZERS_CLS[key] = optimizer_config_cls + + +class OptimizerFactory: + """Optimizer factory class. + + This class builds learning rate and optimizer based on an optimization config. + To use this class, you need to do the following: + (1) Define optimization config, this includes optimizer, and learning rate + schedule. + (2) Initialize the class using the optimization config. + (3) Build learning rate. + (4) Build optimizer. + + This is a typical example for using this class: + params = { + 'optimizer': { + 'type': 'sgd', + 'sgd': {'momentum': 0.9} + }, + 'learning_rate': { + 'type': 'stepwise', + 'stepwise': {'boundaries': [10000, 20000], + 'values': [0.1, 0.01, 0.001]} + }, + 'warmup': { + 'type': 'linear', + 'linear': {'warmup_steps': 500, 'warmup_learning_rate': 0.01} + } + } + opt_config = OptimizationConfig(params) + opt_factory = OptimizerFactory(opt_config) + lr = opt_factory.build_learning_rate() + optimizer = opt_factory.build_optimizer(lr) + """ + + def __init__(self, config: opt_cfg.OptimizationConfig): + """Initializing OptimizerFactory. + + Args: + config: OptimizationConfig instance contain optimization config. + """ + self._config = config + self._optimizer_config = config.optimizer.get() + self._optimizer_type = config.optimizer.type + + self._use_ema = config.ema is not None + self._ema_config = config.ema + + if self._optimizer_config is None: + raise ValueError('Optimizer type must be specified') + + self._lr_config = config.learning_rate.get() + self._lr_type = config.learning_rate.type + + if self._lr_type is None: + raise ValueError('Learning rate type must be specified') + + self._warmup_config = config.warmup.get() + self._warmup_type = config.warmup.type + + def build_learning_rate(self): + """Build learning rate. + + Builds learning rate from config. Learning rate schedule is built according + to the learning rate config. If learning rate type is consant, + lr_config.learning_rate is returned. + + Returns: + tf.keras.optimizers.schedules.LearningRateSchedule instance. If + learning rate type is consant, lr_config.learning_rate is returned. + """ + if self._lr_type == 'constant': + lr = self._lr_config.learning_rate + else: + lr = LR_CLS[self._lr_type](**self._lr_config.as_dict()) + + if self._warmup_config: + lr = WARMUP_CLS[self._warmup_type](lr, **self._warmup_config.as_dict()) + + return lr + + @gin.configurable + def build_optimizer( + self, + lr: Union[tf.keras.optimizers.schedules.LearningRateSchedule, float], + gradient_transformers: Optional[List[Callable[ + [List[Tuple[tf.Tensor, tf.Tensor]]], List[Tuple[tf.Tensor, tf.Tensor]] + ]]] = None, + postprocessor: Optional[Callable[[tf.keras.optimizers.Optimizer], + tf.keras.optimizers.Optimizer]] = None): + """Build optimizer. + + Builds optimizer from config. It takes learning rate as input, and builds + the optimizer according to the optimizer config. Typically, the learning + rate built using self.build_lr() is passed as an argument to this method. + + Args: + lr: A floating point value, or a + tf.keras.optimizers.schedules.LearningRateSchedule instance. + gradient_transformers: Optional list of functions to use to transform + gradients before applying updates to Variables. The functions are + applied after gradient_aggregator. The functions should accept and + return a list of (gradient, variable) tuples. clipvalue, clipnorm, + global_clipnorm should not be set when gradient_transformers is passed. + postprocessor: An optional function for postprocessing the optimizer. It + takes an optimizer and returns an optimizer. + + Returns: + tf.keras.optimizers.Optimizer instance. + """ + + optimizer_dict = self._optimizer_config.as_dict() + ## Delete clipnorm, clipvalue, global_clipnorm if None + if optimizer_dict['clipnorm'] is None: + del optimizer_dict['clipnorm'] + if optimizer_dict['clipvalue'] is None: + del optimizer_dict['clipvalue'] + if optimizer_dict['global_clipnorm'] is None: + del optimizer_dict['global_clipnorm'] + + optimizer_dict['learning_rate'] = lr + if gradient_transformers is not None: + optimizer_dict['gradient_transformers'] = gradient_transformers + + optimizer = OPTIMIZERS_CLS[self._optimizer_type](**optimizer_dict) + + if self._use_ema: + optimizer = ema_optimizer.ExponentialMovingAverage( + optimizer, **self._ema_config.as_dict()) + if postprocessor: + optimizer = postprocessor(optimizer) + assert isinstance(optimizer, tf.keras.optimizers.Optimizer), ( + 'OptimizerFactory.build_optimizer returning a non-optimizer object: ' + '{}'.format(optimizer)) + + return optimizer diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/optimizer_factory_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/optimizer_factory_test.py new file mode 100644 index 0000000000000000000000000000000000000000..e0cc714483b5a06464436b94b3e2ffe1cf65364a --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/optimizer_factory_test.py @@ -0,0 +1,443 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for optimizer_factory.py.""" +from absl.testing import parameterized +import numpy as np +import tensorflow as tf + +from official.modeling.optimization import optimizer_factory +from official.modeling.optimization.configs import optimization_config + + +class OptimizerFactoryTest(tf.test.TestCase, parameterized.TestCase): + + @parameterized.parameters(('sgd'), ('rmsprop'), ('adam'), ('adamw'), ('lamb'), + ('lars'), ('adagrad')) + def test_optimizers(self, optimizer_type): + params = { + 'optimizer': { + 'type': optimizer_type + }, + 'learning_rate': { + 'type': 'constant', + 'constant': { + 'learning_rate': 0.1 + } + } + } + optimizer_cls = optimizer_factory.OPTIMIZERS_CLS[optimizer_type] + expected_optimizer_config = optimizer_cls().get_config() + expected_optimizer_config['learning_rate'] = 0.1 + + opt_config = optimization_config.OptimizationConfig(params) + opt_factory = optimizer_factory.OptimizerFactory(opt_config) + lr = opt_factory.build_learning_rate() + optimizer = opt_factory.build_optimizer(lr, postprocessor=lambda x: x) + + self.assertIsInstance(optimizer, optimizer_cls) + self.assertEqual(expected_optimizer_config, optimizer.get_config()) + + @parameterized.parameters((None, None), (1.0, None), (None, 1.0)) + def test_gradient_clipping(self, clipnorm, clipvalue): + params = { + 'optimizer': { + 'type': 'sgd', + 'sgd': { + 'clipnorm': clipnorm, + 'clipvalue': clipvalue + } + }, + 'learning_rate': { + 'type': 'constant', + 'constant': { + 'learning_rate': 1.0 + } + } + } + + opt_config = optimization_config.OptimizationConfig(params) + opt_factory = optimizer_factory.OptimizerFactory(opt_config) + lr = opt_factory.build_learning_rate() + optimizer = opt_factory.build_optimizer(lr) + + var0 = tf.Variable([1.0, 2.0]) + var1 = tf.Variable([3.0, 4.0]) + + grads0 = tf.constant([0.1, 0.1]) + grads1 = tf.constant([2.0, 3.0]) + + grads_and_vars = list(zip([grads0, grads1], [var0, var1])) + optimizer.apply_gradients(grads_and_vars) + + self.assertAllClose(np.array([0.9, 1.9]), var0.numpy()) + if clipvalue is not None: + self.assertAllClose(np.array([2.0, 3.0]), var1.numpy()) + elif clipnorm is not None: + self.assertAllClose(np.array([2.4452999, 3.1679497]), var1.numpy()) + else: + self.assertAllClose(np.array([1.0, 1.0]), var1.numpy()) + + def test_missing_types(self): + params = {'optimizer': {'type': 'sgd', 'sgd': {'momentum': 0.9}}} + with self.assertRaises(ValueError): + optimizer_factory.OptimizerFactory( + optimization_config.OptimizationConfig(params)) + params = { + 'learning_rate': { + 'type': 'stepwise', + 'stepwise': { + 'boundaries': [10000, 20000], + 'values': [0.1, 0.01, 0.001] + } + } + } + with self.assertRaises(ValueError): + optimizer_factory.OptimizerFactory( + optimization_config.OptimizationConfig(params)) + + +# TODO(b/187559334) refactor lr_schedule tests into `lr_schedule_test.py`. + + def test_stepwise_lr_schedule(self): + params = { + 'optimizer': { + 'type': 'sgd', + 'sgd': { + 'momentum': 0.9 + } + }, + 'learning_rate': { + 'type': 'stepwise', + 'stepwise': { + 'boundaries': [10000, 20000], + 'values': [0.1, 0.01, 0.001] + } + } + } + expected_lr_step_values = [[0, 0.1], [5000, 0.1], [10000, 0.1], + [10001, 0.01], [20000, 0.01], [20001, 0.001]] + opt_config = optimization_config.OptimizationConfig(params) + opt_factory = optimizer_factory.OptimizerFactory(opt_config) + lr = opt_factory.build_learning_rate() + + for step, value in expected_lr_step_values: + self.assertAlmostEqual(lr(step).numpy(), value) + + def test_stepwise_lr_with_warmup_schedule(self): + params = { + 'optimizer': { + 'type': 'sgd', + 'sgd': { + 'momentum': 0.9 + } + }, + 'learning_rate': { + 'type': 'stepwise', + 'stepwise': { + 'boundaries': [10000, 20000], + 'values': [0.1, 0.01, 0.001] + } + }, + 'warmup': { + 'type': 'linear', + 'linear': { + 'warmup_steps': 500, + 'warmup_learning_rate': 0.01 + } + } + } + expected_lr_step_values = [[0, 0.01], [250, 0.055], [500, 0.1], [5500, 0.1], + [10000, 0.1], [10001, 0.01], [20000, 0.01], + [20001, 0.001]] + opt_config = optimization_config.OptimizationConfig(params) + opt_factory = optimizer_factory.OptimizerFactory(opt_config) + lr = opt_factory.build_learning_rate() + + for step, value in expected_lr_step_values: + self.assertAlmostEqual(lr(step).numpy(), value) + + def test_exponential_lr_schedule(self): + params = { + 'optimizer': { + 'type': 'sgd', + 'sgd': { + 'momentum': 0.9 + } + }, + 'learning_rate': { + 'type': 'exponential', + 'exponential': { + 'initial_learning_rate': 0.1, + 'decay_steps': 1000, + 'decay_rate': 0.96, + 'staircase': True + } + } + } + expected_lr_step_values = [ + [0, 0.1], + [999, 0.1], + [1000, 0.096], + [1999, 0.096], + [2000, 0.09216], + ] + opt_config = optimization_config.OptimizationConfig(params) + opt_factory = optimizer_factory.OptimizerFactory(opt_config) + lr = opt_factory.build_learning_rate() + + for step, value in expected_lr_step_values: + self.assertAlmostEqual(lr(step).numpy(), value) + + def test_polynomial_lr_schedule(self): + params = { + 'optimizer': { + 'type': 'sgd', + 'sgd': { + 'momentum': 0.9 + } + }, + 'learning_rate': { + 'type': 'polynomial', + 'polynomial': { + 'initial_learning_rate': 0.1, + 'decay_steps': 1000, + 'end_learning_rate': 0.001 + } + } + } + + expected_lr_step_values = [[0, 0.1], [500, 0.0505], [1000, 0.001]] + opt_config = optimization_config.OptimizationConfig(params) + opt_factory = optimizer_factory.OptimizerFactory(opt_config) + lr = opt_factory.build_learning_rate() + + for step, value in expected_lr_step_values: + self.assertAlmostEqual(lr(step).numpy(), value) + + def test_cosine_lr_schedule(self): + params = { + 'optimizer': { + 'type': 'sgd', + 'sgd': { + 'momentum': 0.9 + } + }, + 'learning_rate': { + 'type': 'cosine', + 'cosine': { + 'initial_learning_rate': 0.1, + 'decay_steps': 1000 + } + } + } + expected_lr_step_values = [[0, 0.1], [250, 0.08535534], [500, 0.04999999], + [750, 0.01464466], [1000, 0]] + opt_config = optimization_config.OptimizationConfig(params) + opt_factory = optimizer_factory.OptimizerFactory(opt_config) + lr = opt_factory.build_learning_rate() + + for step, value in expected_lr_step_values: + self.assertAlmostEqual(lr(step).numpy(), value) + + def test_constant_lr_with_warmup_schedule(self): + params = { + 'optimizer': { + 'type': 'sgd', + 'sgd': { + 'momentum': 0.9 + } + }, + 'learning_rate': { + 'type': 'constant', + 'constant': { + 'learning_rate': 0.1 + } + }, + 'warmup': { + 'type': 'linear', + 'linear': { + 'warmup_steps': 500, + 'warmup_learning_rate': 0.01 + } + } + } + + expected_lr_step_values = [[0, 0.01], [250, 0.055], [500, 0.1], [5000, 0.1], + [10000, 0.1], [20000, 0.1]] + opt_config = optimization_config.OptimizationConfig(params) + opt_factory = optimizer_factory.OptimizerFactory(opt_config) + lr = opt_factory.build_learning_rate() + + for step, value in expected_lr_step_values: + self.assertAlmostEqual(lr(step).numpy(), value) + + def test_stepwise_lr_with_polynomial_warmup_schedule(self): + params = { + 'optimizer': { + 'type': 'sgd', + 'sgd': { + 'momentum': 0.9 + } + }, + 'learning_rate': { + 'type': 'stepwise', + 'stepwise': { + 'boundaries': [10000, 20000], + 'values': [0.1, 0.01, 0.001] + } + }, + 'warmup': { + 'type': 'polynomial', + 'polynomial': { + 'warmup_steps': 500, + 'power': 2. + } + } + } + expected_lr_step_values = [[0, 0.0], [250, 0.025], [500, 0.1], [5500, 0.1], + [10000, 0.1], [10001, 0.01], [20000, 0.01], + [20001, 0.001]] + opt_config = optimization_config.OptimizationConfig(params) + opt_factory = optimizer_factory.OptimizerFactory(opt_config) + lr = opt_factory.build_learning_rate() + + for step, value in expected_lr_step_values: + self.assertAlmostEqual(lr(step).numpy(), value, places=6) + + def test_power_lr_schedule(self): + params = { + 'optimizer': { + 'type': 'sgd', + 'sgd': { + 'momentum': 0.9 + } + }, + 'learning_rate': { + 'type': 'power', + 'power': { + 'initial_learning_rate': 1.0, + 'power': -1.0 + } + } + } + expected_lr_step_values = [[0, 1.0], [1, 1.0], [250, 1. / 250.]] + opt_config = optimization_config.OptimizationConfig(params) + opt_factory = optimizer_factory.OptimizerFactory(opt_config) + lr = opt_factory.build_learning_rate() + + for step, value in expected_lr_step_values: + self.assertAlmostEqual(lr(step).numpy(), value) + + def test_power_linear_lr_schedule(self): + params = { + 'optimizer': { + 'type': 'sgd', + 'sgd': { + 'momentum': 0.9 + } + }, + 'learning_rate': { + 'type': 'power_linear', + 'power_linear': { + 'initial_learning_rate': 1.0, + 'power': -1.0, + 'linear_decay_fraction': 0.5, + 'total_decay_steps': 100, + 'offset': 0, + } + } + } + expected_lr_step_values = [[0, 1.0], [1, 1.0], [40, 1. / 40.], + [60, 1. / 60. * 0.8]] + opt_config = optimization_config.OptimizationConfig(params) + opt_factory = optimizer_factory.OptimizerFactory(opt_config) + lr = opt_factory.build_learning_rate() + + for step, value in expected_lr_step_values: + self.assertAlmostEqual(lr(step).numpy(), value) + + def test_power_with_offset_lr_schedule(self): + params = { + 'optimizer': { + 'type': 'sgd', + 'sgd': { + 'momentum': 0.9 + } + }, + 'learning_rate': { + 'type': 'power_with_offset', + 'power_with_offset': { + 'initial_learning_rate': 1.0, + 'power': -1.0, + 'offset': 10, + 'pre_offset_learning_rate': 3.0, + } + } + } + expected_lr_step_values = [[1, 3.0], [10, 3.0], [20, 1. / 10.]] + opt_config = optimization_config.OptimizationConfig(params) + opt_factory = optimizer_factory.OptimizerFactory(opt_config) + lr = opt_factory.build_learning_rate() + + for step, value in expected_lr_step_values: + self.assertAlmostEqual(lr(step).numpy(), value) + + def test_step_cosine_lr_schedule_with_warmup(self): + params = { + 'optimizer': { + 'type': 'sgd', + 'sgd': { + 'momentum': 0.9 + } + }, + 'learning_rate': { + 'type': 'step_cosine_with_offset', + 'step_cosine_with_offset': { + 'values': (0.0001, 0.00005), + 'boundaries': (0, 500000), + 'offset': 10000, + } + }, + 'warmup': { + 'type': 'linear', + 'linear': { + 'warmup_steps': 10000, + 'warmup_learning_rate': 0.0 + } + } + } + expected_lr_step_values = [[0, 0.0], [5000, 1e-4/2.0], [10000, 1e-4], + [20000, 9.994863e-05], [499999, 5e-05]] + opt_config = optimization_config.OptimizationConfig(params) + opt_factory = optimizer_factory.OptimizerFactory(opt_config) + lr = opt_factory.build_learning_rate() + + for step, value in expected_lr_step_values: + self.assertAlmostEqual(lr(step).numpy(), value) + + +class OptimizerFactoryRegistryTest(tf.test.TestCase): + + def test_registry(self): + + class MyClass(): + pass + optimizer_factory.register_optimizer_cls('test', MyClass) + self.assertIn('test', optimizer_factory.OPTIMIZERS_CLS) + with self.assertRaisesRegex(ValueError, 'test already registered.*'): + optimizer_factory.register_optimizer_cls('test', MyClass) + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/slide_optimizer.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/slide_optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..c1975a3111e109bbb0e40dfb45cb04bc98246ad2 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/optimization/slide_optimizer.py @@ -0,0 +1,20 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""SLIDE optimizer. + +A new optimizer that will be open sourced soon. +""" + +SLIDE = "Unimplemented" diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/performance.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/performance.py new file mode 100644 index 0000000000000000000000000000000000000000..4539e59b5e7d6e0603ebd2de9be92103d9573069 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/performance.py @@ -0,0 +1,55 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Functions and classes related to training performance.""" + +import tensorflow as tf + + +def configure_optimizer(optimizer, + use_float16=False, + use_graph_rewrite=False, + loss_scale=None): + """Configures optimizer object with performance options.""" + if use_float16: + if loss_scale in (None, 'dynamic'): + optimizer = tf.keras.mixed_precision.LossScaleOptimizer(optimizer) + else: + # loss_scale is a number. We interpret that as a fixed loss scale. + optimizer = tf.keras.mixed_precision.LossScaleOptimizer( + optimizer, dynamic=False, initial_scale=loss_scale) + if use_graph_rewrite: + # Note: the model dtype must be 'float32', which will ensure + # tf.keras.mixed_precision and enable_mixed_precision_graph_rewrite do not + # double up. + optimizer = ( + tf.compat.v1.mixed_precision.enable_mixed_precision_graph_rewrite( + optimizer)) + return optimizer + + +def set_mixed_precision_policy(dtype, loss_scale=None): + """Sets the global `tf.keras.mixed_precision.Policy`.""" + # TODO(b/191894773): Remove loss_scale argument + assert loss_scale is None, ( + 'The loss_scale argument must be None. The argument exists for ' + 'historical reasons and will be removed soon.') + if dtype == tf.float16: + tf.keras.mixed_precision.set_global_policy('mixed_float16') + elif dtype == tf.bfloat16: + tf.keras.mixed_precision.set_global_policy('mixed_bfloat16') + elif dtype == tf.float32: + tf.keras.mixed_precision.set_global_policy('float32') + else: + raise ValueError('Unexpected dtype: %s' % dtype) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/tf_utils.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/tf_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ba7cdd2490e7ef31f089758f7126d6711897608b --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/modeling/tf_utils.py @@ -0,0 +1,201 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common TF utilities.""" + +import six +import tensorflow as tf + +from tensorflow.python.util import deprecation +from official.modeling import activations + + +@deprecation.deprecated( + None, + "tf.keras.layers.Layer supports multiple positional args and kwargs as " + "input tensors. pack/unpack inputs to override __call__ is no longer " + "needed.") +def pack_inputs(inputs): + """Pack a list of `inputs` tensors to a tuple. + + Args: + inputs: a list of tensors. + + Returns: + a tuple of tensors. if any input is None, replace it with a special constant + tensor. + """ + inputs = tf.nest.flatten(inputs) + outputs = [] + for x in inputs: + if x is None: + outputs.append(tf.constant(0, shape=[], dtype=tf.int32)) + else: + outputs.append(x) + return tuple(outputs) + + +@deprecation.deprecated( + None, + "tf.keras.layers.Layer supports multiple positional args and kwargs as " + "input tensors. pack/unpack inputs to override __call__ is no longer " + "needed.") +def unpack_inputs(inputs): + """unpack a tuple of `inputs` tensors to a tuple. + + Args: + inputs: a list of tensors. + + Returns: + a tuple of tensors. if any input is a special constant tensor, replace it + with None. + """ + inputs = tf.nest.flatten(inputs) + outputs = [] + for x in inputs: + if is_special_none_tensor(x): + outputs.append(None) + else: + outputs.append(x) + x = tuple(outputs) + + # To trick the very pointless 'unbalanced-tuple-unpacking' pylint check + # from triggering. + if len(x) == 1: + return x[0] + return tuple(outputs) + + +def is_special_none_tensor(tensor): + """Checks if a tensor is a special None Tensor.""" + return tensor.shape.ndims == 0 and tensor.dtype == tf.int32 + + +def get_activation(identifier, use_keras_layer=False): + """Maps a identifier to a Python function, e.g., "relu" => `tf.nn.relu`. + + It checks string first and if it is one of customized activation not in TF, + the corresponding activation will be returned. For non-customized activation + names and callable identifiers, always fallback to tf.keras.activations.get. + + Prefers using keras layers when use_keras_layer=True. Now it only supports + 'relu', 'linear', 'identity', 'swish'. + + Args: + identifier: String name of the activation function or callable. + use_keras_layer: If True, use keras layer if identifier is allow-listed. + + Returns: + A Python function corresponding to the activation function or a keras + activation layer when use_keras_layer=True. + """ + if isinstance(identifier, six.string_types): + identifier = str(identifier).lower() + if use_keras_layer: + keras_layer_allowlist = { + "relu": "relu", + "linear": "linear", + "identity": "linear", + "swish": "swish", + "sigmoid": "sigmoid", + "relu6": tf.nn.relu6, + } + if identifier in keras_layer_allowlist: + return tf.keras.layers.Activation(keras_layer_allowlist[identifier]) + name_to_fn = { + "gelu": activations.gelu, + "simple_swish": activations.simple_swish, + "hard_swish": activations.hard_swish, + "relu6": activations.relu6, + "hard_sigmoid": activations.hard_sigmoid, + "identity": activations.identity, + } + if identifier in name_to_fn: + return tf.keras.activations.get(name_to_fn[identifier]) + return tf.keras.activations.get(identifier) + + +def get_shape_list(tensor, expected_rank=None, name=None): + """Returns a list of the shape of tensor, preferring static dimensions. + + Args: + tensor: A tf.Tensor object to find the shape of. + expected_rank: (optional) int. The expected rank of `tensor`. If this is + specified and the `tensor` has a different rank, and exception will be + thrown. + name: Optional name of the tensor for the error message. + + Returns: + A list of dimensions of the shape of tensor. All static dimensions will + be returned as python integers, and dynamic dimensions will be returned + as tf.Tensor scalars. + """ + if expected_rank is not None: + assert_rank(tensor, expected_rank, name) + + shape = tensor.shape.as_list() + + non_static_indexes = [] + for (index, dim) in enumerate(shape): + if dim is None: + non_static_indexes.append(index) + + if not non_static_indexes: + return shape + + dyn_shape = tf.shape(tensor) + for index in non_static_indexes: + shape[index] = dyn_shape[index] + return shape + + +def assert_rank(tensor, expected_rank, name=None): + """Raises an exception if the tensor rank is not of the expected rank. + + Args: + tensor: A tf.Tensor to check the rank of. + expected_rank: Python integer or list of integers, expected rank. + name: Optional name of the tensor for the error message. + + Raises: + ValueError: If the expected shape doesn't match the actual shape. + """ + expected_rank_dict = {} + if isinstance(expected_rank, six.integer_types): + expected_rank_dict[expected_rank] = True + else: + for x in expected_rank: + expected_rank_dict[x] = True + + actual_rank = tensor.shape.ndims + if actual_rank not in expected_rank_dict: + raise ValueError( + "For the tensor `%s`, the actual tensor rank `%d` (shape = %s) is not " + "equal to the expected tensor rank `%s`" % + (name, actual_rank, str(tensor.shape), str(expected_rank))) + + +def safe_mean(losses): + """Computes a safe mean of the losses. + + Args: + losses: `Tensor` whose elements contain individual loss measurements. + + Returns: + A scalar representing the mean of `losses`. If `num_present` is zero, + then zero is returned. + """ + total = tf.reduce_sum(losses) + num_elements = tf.cast(tf.size(losses), dtype=losses.dtype) + return tf.math.divide_no_nan(total, num_elements) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/README.md b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f3fbc633cd8af26ef55a46ac799fd30282b4ec55 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/README.md @@ -0,0 +1,59 @@ +# TensorFlow NLP Modelling Toolkit + +This codebase provides a Natrual Language Processing modeling toolkit written in +[TF2](https://www.tensorflow.org/guide/effective_tf2). It allows researchers and +developers to reproduce state-of-the-art model results and train custom models +to experiment new research ideas. + +## Features + +* Reusable and modularized modeling building blocks +* State-of-the-art reproducible +* Easy to customize and extend +* End-to-end training +* Distributed trainable on both GPUs and TPUs + +## Major components + +### Libraries + +We provide modeling library to allow users to train custom models for new +research ideas. Detailed intructions can be found in READMEs in each folder. + +* [modeling/](modeling): modeling library that provides building blocks + (e.g.,Layers, Networks, and Models) that can be assembled into + transformer-based achitectures . +* [data/](data): binaries and utils for input preprocessing, tokenization, + etc. + +### State-of-the-Art models and examples + +We provide SoTA model implementations, pre-trained models, training and +evaluation examples, and command lines. Detail instructions can be found in the +READMEs for specific papers. + +1. [BERT](bert): [BERT: Pre-training of Deep Bidirectional Transformers for + Language Understanding](https://arxiv.org/abs/1810.04805) by Devlin et al., + 2018 +2. [ALBERT](albert): + [A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942) + by Lan et al., 2019 +3. [XLNet](xlnet): + [XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) + by Yang et al., 2019 +4. [Transformer for translation](transformer): + [Attention Is All You Need](https://arxiv.org/abs/1706.03762) by Vaswani et + al., 2017 + +### Common Training Driver + +We provide a single common driver [train.py](train.py) to train above SoTA +models on popluar tasks. Please see [docs/train.md](docs/train.md) for +more details. + + +### Pre-trained models with checkpoints and TF-Hub + +We provide a large collection of baselines and checkpoints for NLP pre-trained +models. Please see [docs/pretrained_models.md](docs/pretrained_models.md) for +more details. diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/__init__.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e419af524b5f349fe04abfa820c3cb51b777d422 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/albert/README.md b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/albert/README.md new file mode 100644 index 0000000000000000000000000000000000000000..559fd5400489bd9b1170acdb7cf96f9a7a3d529e --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/albert/README.md @@ -0,0 +1,335 @@ +# ALBERT (ALBERT: A Lite BERT for Self-supervised Learning of Language Representations) + +**WARNING**: We are on the way to deprecate this directory. +We will add documentation in `nlp/docs` to use the new code in `nlp/modeling`. + +The academic paper which describes ALBERT in detail and provides full results on +a number of tasks can be found here: https://arxiv.org/abs/1909.11942. + +This repository contains TensorFlow 2.x implementation for ALBERT. + +## Contents + * [Contents](#contents) + * [Pre-trained Models](#pre-trained-models) + * [Restoring from Checkpoints](#restoring-from-checkpoints) + * [Set Up](#set-up) + * [Process Datasets](#process-datasets) + * [Fine-tuning with BERT](#fine-tuning-with-bert) + * [Cloud GPUs and TPUs](#cloud-gpus-and-tpus) + * [Sentence and Sentence-pair Classification Tasks](#sentence-and-sentence-pair-classification-tasks) + * [SQuAD 1.1](#squad-1.1) + + +## Pre-trained Models + +We released both checkpoints and tf.hub modules as the pretrained models for +fine-tuning. They are TF 2.x compatible and are converted from the ALBERT v2 +checkpoints released in TF 1.x official ALBERT repository +[google-research/albert](https://github.com/google-research/albert) +in order to keep consistent with ALBERT paper. + +Our current released checkpoints are exactly the same as TF 1.x official ALBERT +repository. + +### Access to Pretrained Checkpoints + +Pretrained checkpoints can be found in the following links: + +**Note: We implemented ALBERT using Keras functional-style networks in [nlp/modeling](../modeling). +ALBERT V2 models compatible with TF 2.x checkpoints are:** + +* **[`ALBERT V2 Base`](https://storage.googleapis.com/cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base.tar.gz)**: + 12-layer, 768-hidden, 12-heads, 12M parameters +* **[`ALBERT V2 Large`](https://storage.googleapis.com/cloud-tpu-checkpoints/albert/checkpoints/albert_v2_large.tar.gz)**: + 24-layer, 1024-hidden, 16-heads, 18M parameters +* **[`ALBERT V2 XLarge`](https://storage.googleapis.com/cloud-tpu-checkpoints/albert/checkpoints/albert_v2_xlarge.tar.gz)**: + 24-layer, 2048-hidden, 32-heads, 60M parameters +* **[`ALBERT V2 XXLarge`](https://storage.googleapis.com/cloud-tpu-checkpoints/albert/checkpoints/albert_v2_xxlarge.tar.gz)**: + 12-layer, 4096-hidden, 64-heads, 235M parameters + +We recommend to host checkpoints on Google Cloud storage buckets when you use +Cloud GPU/TPU. + +### Restoring from Checkpoints + +`tf.train.Checkpoint` is used to manage model checkpoints in TF 2. To restore +weights from provided pre-trained checkpoints, you can use the following code: + +```python +init_checkpoint='the pretrained model checkpoint path.' +model=tf.keras.Model() # Bert pre-trained model as feature extractor. +checkpoint = tf.train.Checkpoint(model=model) +checkpoint.restore(init_checkpoint) +``` + +Checkpoints featuring native serialized Keras models +(i.e. model.load()/load_weights()) will be available soon. + +### Access to Pretrained hub modules. + +Pretrained tf.hub modules in TF 2.x SavedModel format can be found in the +following links: + +* **[`ALBERT V2 Base`](https://tfhub.dev/tensorflow/albert_en_base/1)**: + 12-layer, 768-hidden, 12-heads, 12M parameters +* **[`ALBERT V2 Large`](https://tfhub.dev/tensorflow/albert_en_large/1)**: + 24-layer, 1024-hidden, 16-heads, 18M parameters +* **[`ALBERT V2 XLarge`](https://tfhub.dev/tensorflow/albert_en_xlarge/1)**: + 24-layer, 2048-hidden, 32-heads, 60M parameters +* **[`ALBERT V2 XXLarge`](https://tfhub.dev/tensorflow/albert_en_xxlarge/1)**: + 12-layer, 4096-hidden, 64-heads, 235M parameters + +## Set Up + +```shell +export PYTHONPATH="$PYTHONPATH:/path/to/models" +``` + +Install `tf-nightly` to get latest updates: + +```shell +pip install tf-nightly-gpu +``` + +With TPU, GPU support is not necessary. First, you need to create a `tf-nightly` +TPU with [ctpu tool](https://github.com/tensorflow/tpu/tree/master/tools/ctpu): + +```shell +ctpu up -name --tf-version=”nightly” +``` + +Second, you need to install TF 2 `tf-nightly` on your VM: + +```shell +pip install tf-nightly +``` + +Warning: More details TPU-specific set-up instructions and tutorial should come +along with official TF 2.x release for TPU. Note that this repo is not +officially supported by Google Cloud TPU team yet until TF 2.1 released. + +## Process Datasets + +### Pre-training + +Pre-train ALBERT using TF2.x will come soon. +For now, please use [ALBERT research repo](https://github.com/google-research/ALBERT) +to pretrain the model and convert the checkpoint to TF2.x compatible ones using +[tf2_albert_encoder_checkpoint_converter.py](tf2_albert_encoder_checkpoint_converter.py). + + + +### Fine-tuning + +To prepare the fine-tuning data for final model training, use the +[`../data/create_finetuning_data.py`](../data/create_finetuning_data.py) script. +Note that different from BERT models that use word piece tokenzer, +ALBERT models employ sentence piece tokenizer. So the FLAG tokenizer_impl has +to be set to 'sentence_piece'. +Resulting datasets in `tf_record` format and training meta data should be later +passed to training or evaluation scripts. The task-specific arguments are +described in following sections: + +* GLUE + +Users can download the +[GLUE data](https://gluebenchmark.com/tasks) by running +[this script](https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e) +and unpack it to some directory `$GLUE_DIR`. + +```shell +export GLUE_DIR=~/glue +export ALBERT_DIR=gs://cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base + +export TASK_NAME=MNLI +export OUTPUT_DIR=gs://some_bucket/datasets +python ../data/create_finetuning_data.py \ + --input_data_dir=${GLUE_DIR}/${TASK_NAME}/ \ + --sp_model_file=${ALBERT_DIR}/30k-clean.model \ + --train_data_output_path=${OUTPUT_DIR}/${TASK_NAME}_train.tf_record \ + --eval_data_output_path=${OUTPUT_DIR}/${TASK_NAME}_eval.tf_record \ + --meta_data_file_path=${OUTPUT_DIR}/${TASK_NAME}_meta_data \ + --fine_tuning_task_type=classification --max_seq_length=128 \ + --classification_task_name=${TASK_NAME} \ + --tokenization=SentencePiece +``` + +* SQUAD + +The [SQuAD website](https://rajpurkar.github.io/SQuAD-explorer/) contains +detailed information about the SQuAD datasets and evaluation. + +The necessary files can be found here: + +* [train-v1.1.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json) +* [dev-v1.1.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json) +* [evaluate-v1.1.py](https://github.com/allenai/bi-att-flow/blob/master/squad/evaluate-v1.1.py) +* [train-v2.0.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json) +* [dev-v2.0.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json) +* [evaluate-v2.0.py](https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/) + +```shell +export SQUAD_DIR=~/squad +export SQUAD_VERSION=v1.1 +export ALBERT_DIR=gs://cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base +export OUTPUT_DIR=gs://some_bucket/datasets + +python ../data/create_finetuning_data.py \ + --squad_data_file=${SQUAD_DIR}/train-${SQUAD_VERSION}.json \ + --sp_model_file=${ALBERT_DIR}/30k-clean.model \ + --train_data_output_path=${OUTPUT_DIR}/squad_${SQUAD_VERSION}_train.tf_record \ + --meta_data_file_path=${OUTPUT_DIR}/squad_${SQUAD_VERSION}_meta_data \ + --fine_tuning_task_type=squad --max_seq_length=384 \ + --tokenization=SentencePiece +``` + +## Fine-tuning with ALBERT + +### Cloud GPUs and TPUs + +* Cloud Storage + +The unzipped pre-trained model files can also be found in the Google Cloud +Storage folder `gs://cloud-tpu-checkpoints/albert/checkpoints`. For example: + +```shell +export ALBERT_DIR=gs://cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base +export MODEL_DIR=gs://some_bucket/my_output_dir +``` + +Currently, users are able to access to `tf-nightly` TPUs and the following TPU +script should run with `tf-nightly`. + +* GPU -> TPU + +Just add the following flags to `run_classifier.py` or `run_squad.py`: + +```shell + --distribution_strategy=tpu + --tpu=grpc://${TPU_IP_ADDRESS}:8470 +``` + +### Sentence and Sentence-pair Classification Tasks + +This example code fine-tunes `albert_v2_base` on the Microsoft Research +Paraphrase Corpus (MRPC) corpus, which only contains 3,600 examples and can +fine-tune in a few minutes on most GPUs. + +We use the `albert_v2_base` as an example throughout the +workflow. + + +```shell +export ALBERT_DIR=gs://cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base +export MODEL_DIR=gs://some_bucket/my_output_dir +export GLUE_DIR=gs://some_bucket/datasets +export TASK=MRPC + +python run_classifier.py \ + --mode='train_and_eval' \ + --input_meta_data_path=${GLUE_DIR}/${TASK}_meta_data \ + --train_data_path=${GLUE_DIR}/${TASK}_train.tf_record \ + --eval_data_path=${GLUE_DIR}/${TASK}_eval.tf_record \ + --bert_config_file=${ALBERT_DIR}/albert_config.json \ + --init_checkpoint=${ALBERT_DIR}/bert_model.ckpt \ + --train_batch_size=4 \ + --eval_batch_size=4 \ + --steps_per_loop=1 \ + --learning_rate=2e-5 \ + --num_train_epochs=3 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=mirrored +``` + +Alternatively, instead of specifying `init_checkpoint`, you can specify +`hub_module_url` to employ a pretraind BERT hub module, e.g., +` --hub_module_url=https://tfhub.dev/tensorflow/albert_en_base/1`. + +To use TPU, you only need to switch distribution strategy type to `tpu` with TPU +information and use remote storage for model checkpoints. + +```shell +export ALBERT_DIR=gs://cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base +export TPU_IP_ADDRESS='???' +export MODEL_DIR=gs://some_bucket/my_output_dir +export GLUE_DIR=gs://some_bucket/datasets + +python run_classifier.py \ + --mode='train_and_eval' \ + --input_meta_data_path=${GLUE_DIR}/${TASK}_meta_data \ + --train_data_path=${GLUE_DIR}/${TASK}_train.tf_record \ + --eval_data_path=${GLUE_DIR}/${TASK}_eval.tf_record \ + --bert_config_file=$ALBERT_DIR/albert_config.json \ + --init_checkpoint=$ALBERT_DIR/bert_model.ckpt \ + --train_batch_size=32 \ + --eval_batch_size=32 \ + --learning_rate=2e-5 \ + --num_train_epochs=3 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=tpu \ + --tpu=grpc://${TPU_IP_ADDRESS}:8470 +``` + +### SQuAD 1.1 + +The Stanford Question Answering Dataset (SQuAD) is a popular question answering +benchmark dataset. See more in [SQuAD website](https://rajpurkar.github.io/SQuAD-explorer/). + +We use the `albert_v2_base` as an example throughout the +workflow. + +```shell +export ALBERT_DIR=gs://cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base +export SQUAD_DIR=gs://some_bucket/datasets +export MODEL_DIR=gs://some_bucket/my_output_dir +export SQUAD_VERSION=v1.1 + +python run_squad.py \ + --input_meta_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_meta_data \ + --train_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_train.tf_record \ + --predict_file=${SQUAD_DIR}/dev-v1.1.json \ + --sp_model_file=${ALBERT_DIR}/30k-clean.model \ + --bert_config_file=$ALBERT_DIR/albert_config.json \ + --init_checkpoint=$ALBERT_DIR/bert_model.ckpt \ + --train_batch_size=4 \ + --predict_batch_size=4 \ + --learning_rate=8e-5 \ + --num_train_epochs=2 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=mirrored +``` + +Similarily, you can replace `init_checkpoint` FLAGS with `hub_module_url` to +specify a hub module path. + +To use TPU, you need switch distribution strategy type to `tpu` with TPU +information. + +```shell +export ALBERT_DIR=gs://cloud-tpu-checkpoints/albert/checkpoints/albert_v2_base +export TPU_IP_ADDRESS='???' +export MODEL_DIR=gs://some_bucket/my_output_dir +export SQUAD_DIR=gs://some_bucket/datasets +export SQUAD_VERSION=v1.1 + +python run_squad.py \ + --input_meta_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_meta_data \ + --train_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_train.tf_record \ + --predict_file=${SQUAD_DIR}/dev-v1.1.json \ + --sp_model_file=${ALBERT_DIR}/30k-clean.model \ + --bert_config_file=$ALBERT_DIR/albert_config.json \ + --init_checkpoint=$ALBERT_DIR/bert_model.ckpt \ + --train_batch_size=32 \ + --learning_rate=8e-5 \ + --num_train_epochs=2 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=tpu \ + --tpu=grpc://${TPU_IP_ADDRESS}:8470 +``` + +The dev set predictions will be saved into a file called predictions.json in the +model_dir: + +```shell +python $SQUAD_DIR/evaluate-v1.1.py $SQUAD_DIR/dev-v1.1.json ./squad/predictions.json +``` diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/albert/__init__.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/albert/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e419af524b5f349fe04abfa820c3cb51b777d422 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/albert/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/albert/configs.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/albert/configs.py new file mode 100644 index 0000000000000000000000000000000000000000..6fd6fdff7b97e7a0dce385eb4edd22de6d23b6d0 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/albert/configs.py @@ -0,0 +1,50 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The ALBERT configurations.""" + +import six + +from official.nlp.bert import configs + + +class AlbertConfig(configs.BertConfig): + """Configuration for `ALBERT`.""" + + def __init__(self, num_hidden_groups=1, inner_group_num=1, **kwargs): + """Constructs AlbertConfig. + + Args: + num_hidden_groups: Number of group for the hidden layers, parameters in + the same group are shared. Note that this value and also the following + 'inner_group_num' has to be 1 for now, because all released ALBERT + models set them to 1. We may support arbitary valid values in future. + inner_group_num: Number of inner repetition of attention and ffn. + **kwargs: The remaining arguments are the same as above 'BertConfig'. + """ + super(AlbertConfig, self).__init__(**kwargs) + + # TODO(chendouble): 'inner_group_num' and 'num_hidden_groups' are always 1 + # in the released ALBERT. Support other values in AlbertEncoder if needed. + if inner_group_num != 1 or num_hidden_groups != 1: + raise ValueError("We only support 'inner_group_num' and " + "'num_hidden_groups' as 1.") + + @classmethod + def from_dict(cls, json_object): + """Constructs a `AlbertConfig` from a Python dictionary of parameters.""" + config = AlbertConfig(vocab_size=None) + for (key, value) in six.iteritems(json_object): + config.__dict__[key] = value + return config diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/albert/run_classifier.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/albert/run_classifier.py new file mode 100644 index 0000000000000000000000000000000000000000..70e26267f32d98b68a7758615961302333e75c50 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/albert/run_classifier.py @@ -0,0 +1,105 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ALBERT classification finetuning runner in tf2.x.""" + +import json +import os +# Import libraries +from absl import app +from absl import flags +from absl import logging +import tensorflow as tf +from official.common import distribute_utils +from official.nlp.albert import configs as albert_configs +from official.nlp.bert import bert_models +from official.nlp.bert import run_classifier as run_classifier_bert + + +FLAGS = flags.FLAGS + + +def predict(strategy, albert_config, input_meta_data, predict_input_fn): + """Function outputs both the ground truth predictions as .tsv files.""" + with strategy.scope(): + classifier_model = bert_models.classifier_model( + albert_config, input_meta_data['num_labels'])[0] + checkpoint = tf.train.Checkpoint(model=classifier_model) + latest_checkpoint_file = ( + FLAGS.predict_checkpoint_path or + tf.train.latest_checkpoint(FLAGS.model_dir)) + assert latest_checkpoint_file + logging.info('Checkpoint file %s found and restoring from ' + 'checkpoint', latest_checkpoint_file) + checkpoint.restore( + latest_checkpoint_file).assert_existing_objects_matched() + preds, ground_truth = run_classifier_bert.get_predictions_and_labels( + strategy, classifier_model, predict_input_fn, return_probs=True) + output_predict_file = os.path.join(FLAGS.model_dir, 'test_results.tsv') + with tf.io.gfile.GFile(output_predict_file, 'w') as writer: + logging.info('***** Predict results *****') + for probabilities in preds: + output_line = '\t'.join( + str(class_probability) + for class_probability in probabilities) + '\n' + writer.write(output_line) + ground_truth_labels_file = os.path.join(FLAGS.model_dir, + 'output_labels.tsv') + with tf.io.gfile.GFile(ground_truth_labels_file, 'w') as writer: + logging.info('***** Ground truth results *****') + for label in ground_truth: + output_line = '\t'.join(str(label)) + '\n' + writer.write(output_line) + return + + +def main(_): + with tf.io.gfile.GFile(FLAGS.input_meta_data_path, 'rb') as reader: + input_meta_data = json.loads(reader.read().decode('utf-8')) + + if not FLAGS.model_dir: + FLAGS.model_dir = '/tmp/bert20/' + + strategy = distribute_utils.get_distribution_strategy( + distribution_strategy=FLAGS.distribution_strategy, + num_gpus=FLAGS.num_gpus, + tpu_address=FLAGS.tpu) + max_seq_length = input_meta_data['max_seq_length'] + train_input_fn = run_classifier_bert.get_dataset_fn( + FLAGS.train_data_path, + max_seq_length, + FLAGS.train_batch_size, + is_training=True) + eval_input_fn = run_classifier_bert.get_dataset_fn( + FLAGS.eval_data_path, + max_seq_length, + FLAGS.eval_batch_size, + is_training=False) + + albert_config = albert_configs.AlbertConfig.from_json_file( + FLAGS.bert_config_file) + if FLAGS.mode == 'train_and_eval': + run_classifier_bert.run_bert(strategy, input_meta_data, albert_config, + train_input_fn, eval_input_fn) + elif FLAGS.mode == 'predict': + predict(strategy, albert_config, input_meta_data, eval_input_fn) + else: + raise ValueError('Unsupported mode is specified: %s' % FLAGS.mode) + return + +if __name__ == '__main__': + flags.mark_flag_as_required('bert_config_file') + flags.mark_flag_as_required('input_meta_data_path') + flags.mark_flag_as_required('model_dir') + app.run(main) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/albert/run_squad.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/albert/run_squad.py new file mode 100644 index 0000000000000000000000000000000000000000..cd635443ffd039c248cdc77bf38bb52148c05685 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/albert/run_squad.py @@ -0,0 +1,132 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run ALBERT on SQuAD 1.1 and SQuAD 2.0 in TF 2.x.""" + +import json +import os +import time + +# Import libraries +from absl import app +from absl import flags +from absl import logging +import tensorflow as tf +from official.common import distribute_utils +from official.nlp.albert import configs as albert_configs +from official.nlp.bert import run_squad_helper +from official.nlp.bert import tokenization +from official.nlp.data import squad_lib_sp + +flags.DEFINE_string( + 'sp_model_file', None, + 'The path to the sentence piece model. Used by sentence piece tokenizer ' + 'employed by ALBERT.') + +# More flags can be found in run_squad_helper. +run_squad_helper.define_common_squad_flags() + +FLAGS = flags.FLAGS + + +def train_squad(strategy, + input_meta_data, + custom_callbacks=None, + run_eagerly=False): + """Runs bert squad training.""" + bert_config = albert_configs.AlbertConfig.from_json_file( + FLAGS.bert_config_file) + run_squad_helper.train_squad(strategy, input_meta_data, bert_config, + custom_callbacks, run_eagerly) + + +def predict_squad(strategy, input_meta_data): + """Makes predictions for the squad dataset.""" + bert_config = albert_configs.AlbertConfig.from_json_file( + FLAGS.bert_config_file) + tokenizer = tokenization.FullSentencePieceTokenizer( + sp_model_file=FLAGS.sp_model_file) + + run_squad_helper.predict_squad(strategy, input_meta_data, tokenizer, + bert_config, squad_lib_sp) + + +def eval_squad(strategy, input_meta_data): + """Evaluate on the squad dataset.""" + bert_config = albert_configs.AlbertConfig.from_json_file( + FLAGS.bert_config_file) + tokenizer = tokenization.FullSentencePieceTokenizer( + sp_model_file=FLAGS.sp_model_file) + + eval_metrics = run_squad_helper.eval_squad( + strategy, input_meta_data, tokenizer, bert_config, squad_lib_sp) + return eval_metrics + + +def export_squad(model_export_path, input_meta_data): + """Exports a trained model as a `SavedModel` for inference. + + Args: + model_export_path: a string specifying the path to the SavedModel directory. + input_meta_data: dictionary containing meta data about input and model. + + Raises: + Export path is not specified, got an empty string or None. + """ + bert_config = albert_configs.AlbertConfig.from_json_file( + FLAGS.bert_config_file) + run_squad_helper.export_squad(model_export_path, input_meta_data, bert_config) + + +def main(_): + with tf.io.gfile.GFile(FLAGS.input_meta_data_path, 'rb') as reader: + input_meta_data = json.loads(reader.read().decode('utf-8')) + + if FLAGS.mode == 'export_only': + export_squad(FLAGS.model_export_path, input_meta_data) + return + + # Configures cluster spec for multi-worker distribution strategy. + if FLAGS.num_gpus > 0: + _ = distribute_utils.configure_cluster(FLAGS.worker_hosts, FLAGS.task_index) + strategy = distribute_utils.get_distribution_strategy( + distribution_strategy=FLAGS.distribution_strategy, + num_gpus=FLAGS.num_gpus, + all_reduce_alg=FLAGS.all_reduce_alg, + tpu_address=FLAGS.tpu) + + if 'train' in FLAGS.mode: + train_squad(strategy, input_meta_data, run_eagerly=FLAGS.run_eagerly) + if 'predict' in FLAGS.mode: + predict_squad(strategy, input_meta_data) + if 'eval' in FLAGS.mode: + eval_metrics = eval_squad(strategy, input_meta_data) + f1_score = eval_metrics['final_f1'] + logging.info('SQuAD eval F1-score: %f', f1_score) + summary_dir = os.path.join(FLAGS.model_dir, 'summaries', 'eval') + summary_writer = tf.summary.create_file_writer(summary_dir) + with summary_writer.as_default(): + # TODO(lehou): write to the correct step number. + tf.summary.scalar('F1-score', f1_score, step=0) + summary_writer.flush() + # Also write eval_metrics to json file. + squad_lib_sp.write_to_json_files( + eval_metrics, os.path.join(summary_dir, 'eval_metrics.json')) + time.sleep(60) + + +if __name__ == '__main__': + flags.mark_flag_as_required('bert_config_file') + flags.mark_flag_as_required('model_dir') + app.run(main) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/README.md b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/README.md new file mode 100644 index 0000000000000000000000000000000000000000..037ff0b1ff8c6ea22bcf692bb8f786320b7d2d48 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/README.md @@ -0,0 +1,395 @@ +# BERT (Bidirectional Encoder Representations from Transformers) + +**WARNING**: We are on the way to deprecate most of the code in this directory. +Please see +[this link](https://github.com/tensorflow/models/blob/master/official/nlp/docs/train.md) +for the new tutorial and use the new code in `nlp/modeling`. This README is +still correct for this legacy implementation. + +The academic paper which describes BERT in detail and provides full results on a +number of tasks can be found here: https://arxiv.org/abs/1810.04805. + +This repository contains TensorFlow 2.x implementation for BERT. + +## Contents + * [Contents](#contents) + * [Pre-trained Models](#pre-trained-models) + * [Restoring from Checkpoints](#restoring-from-checkpoints) + * [Set Up](#set-up) + * [Process Datasets](#process-datasets) + * [Fine-tuning with BERT](#fine-tuning-with-bert) + * [Cloud GPUs and TPUs](#cloud-gpus-and-tpus) + * [Sentence and Sentence-pair Classification Tasks](#sentence-and-sentence-pair-classification-tasks) + * [SQuAD 1.1](#squad-1.1) + + +## Pre-trained Models + +We released both checkpoints and tf.hub modules as the pretrained models for +fine-tuning. They are TF 2.x compatible and are converted from the checkpoints +released in TF 1.x official BERT repository +[google-research/bert](https://github.com/google-research/bert) +in order to keep consistent with BERT paper. + + +### Access to Pretrained Checkpoints + +Pretrained checkpoints can be found in the following links: + +**Note: We have switched BERT implementation +to use Keras functional-style networks in [nlp/modeling](../modeling). +The new checkpoints are:** + +* **[`BERT-Large, Uncased (Whole Word Masking)`](https://storage.googleapis.com/cloud-tpu-checkpoints/bert/keras_bert/wwm_uncased_L-24_H-1024_A-16.tar.gz)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters +* **[`BERT-Large, Cased (Whole Word Masking)`](https://storage.googleapis.com/cloud-tpu-checkpoints/bert/keras_bert/wwm_cased_L-24_H-1024_A-16.tar.gz)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters +* **[`BERT-Base, Uncased`](https://storage.googleapis.com/cloud-tpu-checkpoints/bert/keras_bert/uncased_L-12_H-768_A-12.tar.gz)**: + 12-layer, 768-hidden, 12-heads, 110M parameters +* **[`BERT-Large, Uncased`](https://storage.googleapis.com/cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16.tar.gz)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters +* **[`BERT-Base, Cased`](https://storage.googleapis.com/cloud-tpu-checkpoints/bert/keras_bert/cased_L-12_H-768_A-12.tar.gz)**: + 12-layer, 768-hidden, 12-heads , 110M parameters +* **[`BERT-Large, Cased`](https://storage.googleapis.com/cloud-tpu-checkpoints/bert/keras_bert/cased_L-24_H-1024_A-16.tar.gz)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters +* **[`BERT-Base, Multilingual Cased`](https://storage.googleapis.com/cloud-tpu-checkpoints/bert/keras_bert/multi_cased_L-12_H-768_A-12.tar.gz)**: + 104 languages, 12-layer, 768-hidden, 12-heads, 110M parameters + +We recommend to host checkpoints on Google Cloud storage buckets when you use +Cloud GPU/TPU. + +### Restoring from Checkpoints + +`tf.train.Checkpoint` is used to manage model checkpoints in TF 2. To restore +weights from provided pre-trained checkpoints, you can use the following code: + +```python +init_checkpoint='the pretrained model checkpoint path.' +model=tf.keras.Model() # Bert pre-trained model as feature extractor. +checkpoint = tf.train.Checkpoint(model=model) +checkpoint.restore(init_checkpoint) +``` + +Checkpoints featuring native serialized Keras models +(i.e. model.load()/load_weights()) will be available soon. + +### Access to Pretrained hub modules. + +Pretrained tf.hub modules in TF 2.x SavedModel format can be found in the +following links: + +* **[`BERT-Large, Uncased (Whole Word Masking)`](https://tfhub.dev/tensorflow/bert_en_wwm_uncased_L-24_H-1024_A-16/)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters +* **[`BERT-Large, Cased (Whole Word Masking)`](https://tfhub.dev/tensorflow/bert_en_wwm_cased_L-24_H-1024_A-16/)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters +* **[`BERT-Base, Uncased`](https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/)**: + 12-layer, 768-hidden, 12-heads, 110M parameters +* **[`BERT-Large, Uncased`](https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters +* **[`BERT-Base, Cased`](https://tfhub.dev/tensorflow/bert_en_cased_L-12_H-768_A-12/)**: + 12-layer, 768-hidden, 12-heads , 110M parameters +* **[`BERT-Large, Cased`](https://tfhub.dev/tensorflow/bert_en_cased_L-24_H-1024_A-16/)**: + 24-layer, 1024-hidden, 16-heads, 340M parameters +* **[`BERT-Base, Multilingual Cased`](https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/)**: + 104 languages, 12-layer, 768-hidden, 12-heads, 110M parameters +* **[`BERT-Base, Chinese`](https://tfhub.dev/tensorflow/bert_zh_L-12_H-768_A-12/)**: + Chinese Simplified and Traditional, 12-layer, 768-hidden, 12-heads, + 110M parameters + +## Set Up + +```shell +export PYTHONPATH="$PYTHONPATH:/path/to/models" +``` + +Install `tf-nightly` to get latest updates: + +```shell +pip install tf-nightly-gpu +``` + +With TPU, GPU support is not necessary. First, you need to create a `tf-nightly` +TPU with [ctpu tool](https://github.com/tensorflow/tpu/tree/master/tools/ctpu): + +```shell +ctpu up -name --tf-version=”nightly” +``` + +Second, you need to install TF 2 `tf-nightly` on your VM: + +```shell +pip install tf-nightly +``` + +## Process Datasets + +### Pre-training + +There is no change to generate pre-training data. Please use the script +[`../data/create_pretraining_data.py`](../data/create_pretraining_data.py) +which is essentially branched from [BERT research repo](https://github.com/google-research/bert) +to get processed pre-training data and it adapts to TF2 symbols and python3 +compatibility. + +Running the pre-training script requires an input and output directory, as well as a vocab file. Note that max_seq_length will need to match the sequence length parameter you specify when you run pre-training. + +Example shell script to call create_pretraining_data.py +``` +export WORKING_DIR='local disk or cloud location' +export BERT_DIR='local disk or cloud location' +python models/official/nlp/data/create_pretraining_data.py \ + --input_file=$WORKING_DIR/input/input.txt \ + --output_file=$WORKING_DIR/output/tf_examples.tfrecord \ + --vocab_file=$BERT_DIR/wwm_uncased_L-24_H-1024_A-16/vocab.txt \ + --do_lower_case=True \ + --max_seq_length=512 \ + --max_predictions_per_seq=76 \ + --masked_lm_prob=0.15 \ + --random_seed=12345 \ + --dupe_factor=5 +``` + +### Fine-tuning + +To prepare the fine-tuning data for final model training, use the +[`../data/create_finetuning_data.py`](../data/create_finetuning_data.py) script. +Resulting datasets in `tf_record` format and training meta data should be later +passed to training or evaluation scripts. The task-specific arguments are +described in following sections: + +* GLUE + +Users can download the +[GLUE data](https://gluebenchmark.com/tasks) by running +[this script](https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e) +and unpack it to some directory `$GLUE_DIR`. +Also, users can download [Pretrained Checkpoint](#access-to-pretrained-checkpoints) and locate on some directory `$BERT_DIR` instead of using checkpoints on Google Cloud Storage. + +```shell +export GLUE_DIR=~/glue +export BERT_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 + +export TASK_NAME=MNLI +export OUTPUT_DIR=gs://some_bucket/datasets +python ../data/create_finetuning_data.py \ + --input_data_dir=${GLUE_DIR}/${TASK_NAME}/ \ + --vocab_file=${BERT_DIR}/vocab.txt \ + --train_data_output_path=${OUTPUT_DIR}/${TASK_NAME}_train.tf_record \ + --eval_data_output_path=${OUTPUT_DIR}/${TASK_NAME}_eval.tf_record \ + --meta_data_file_path=${OUTPUT_DIR}/${TASK_NAME}_meta_data \ + --fine_tuning_task_type=classification --max_seq_length=128 \ + --classification_task_name=${TASK_NAME} +``` + +* SQUAD + +The [SQuAD website](https://rajpurkar.github.io/SQuAD-explorer/) contains +detailed information about the SQuAD datasets and evaluation. + +The necessary files can be found here: + +* [train-v1.1.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json) +* [dev-v1.1.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json) +* [evaluate-v1.1.py](https://github.com/allenai/bi-att-flow/blob/master/squad/evaluate-v1.1.py) +* [train-v2.0.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json) +* [dev-v2.0.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v2.0.json) +* [evaluate-v2.0.py](https://worksheets.codalab.org/rest/bundles/0x6b567e1cf2e041ec80d7098f031c5c9e/contents/blob/) + +```shell +export SQUAD_DIR=~/squad +export SQUAD_VERSION=v1.1 +export BERT_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 +export OUTPUT_DIR=gs://some_bucket/datasets + +python ../data/create_finetuning_data.py \ + --squad_data_file=${SQUAD_DIR}/train-${SQUAD_VERSION}.json \ + --vocab_file=${BERT_DIR}/vocab.txt \ + --train_data_output_path=${OUTPUT_DIR}/squad_${SQUAD_VERSION}_train.tf_record \ + --meta_data_file_path=${OUTPUT_DIR}/squad_${SQUAD_VERSION}_meta_data \ + --fine_tuning_task_type=squad --max_seq_length=384 +``` + +Note: To create fine-tuning data with SQUAD 2.0, you need to add flag `--version_2_with_negative=True`. + +## Fine-tuning with BERT + +### Cloud GPUs and TPUs + +* Cloud Storage + +The unzipped pre-trained model files can also be found in the Google Cloud +Storage folder `gs://cloud-tpu-checkpoints/bert/keras_bert`. For example: + +```shell +export BERT_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 +export MODEL_DIR=gs://some_bucket/my_output_dir +``` + +Currently, users are able to access to `tf-nightly` TPUs and the following TPU +script should run with `tf-nightly`. + +* GPU -> TPU + +Just add the following flags to `run_classifier.py` or `run_squad.py`: + +```shell + --distribution_strategy=tpu + --tpu=grpc://${TPU_IP_ADDRESS}:8470 +``` + +### Sentence and Sentence-pair Classification Tasks + +This example code fine-tunes `BERT-Large` on the Microsoft Research Paraphrase +Corpus (MRPC) corpus, which only contains 3,600 examples and can fine-tune in a +few minutes on most GPUs. + +We use the `BERT-Large` (uncased_L-24_H-1024_A-16) as an example throughout the +workflow. +For GPU memory of 16GB or smaller, you may try to use `BERT-Base` +(uncased_L-12_H-768_A-12). + +```shell +export BERT_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 +export MODEL_DIR=gs://some_bucket/my_output_dir +export GLUE_DIR=gs://some_bucket/datasets +export TASK=MRPC + +python run_classifier.py \ + --mode='train_and_eval' \ + --input_meta_data_path=${GLUE_DIR}/${TASK}_meta_data \ + --train_data_path=${GLUE_DIR}/${TASK}_train.tf_record \ + --eval_data_path=${GLUE_DIR}/${TASK}_eval.tf_record \ + --bert_config_file=${BERT_DIR}/bert_config.json \ + --init_checkpoint=${BERT_DIR}/bert_model.ckpt \ + --train_batch_size=4 \ + --eval_batch_size=4 \ + --steps_per_loop=1 \ + --learning_rate=2e-5 \ + --num_train_epochs=3 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=mirrored +``` + +Alternatively, instead of specifying `init_checkpoint`, you can specify +`hub_module_url` to employ a pretraind BERT hub module, e.g., +` --hub_module_url=https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1`. + +After training a model, to get predictions from the classifier, you can set the +`--mode=predict` and offer the test set tfrecords to `--eval_data_path`. +Output will be created in file called test_results.tsv in the output folder. +Each line will contain output for each sample, columns are the class +probabilities. + +```shell +python run_classifier.py \ + --mode='predict' \ + --input_meta_data_path=${GLUE_DIR}/${TASK}_meta_data \ + --eval_data_path=${GLUE_DIR}/${TASK}_eval.tf_record \ + --bert_config_file=${BERT_DIR}/bert_config.json \ + --eval_batch_size=4 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=mirrored +``` + +To use TPU, you only need to switch distribution strategy type to `tpu` with TPU +information and use remote storage for model checkpoints. + +```shell +export BERT_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 +export TPU_IP_ADDRESS='???' +export MODEL_DIR=gs://some_bucket/my_output_dir +export GLUE_DIR=gs://some_bucket/datasets +export TASK=MRPC + +python run_classifier.py \ + --mode='train_and_eval' \ + --input_meta_data_path=${GLUE_DIR}/${TASK}_meta_data \ + --train_data_path=${GLUE_DIR}/${TASK}_train.tf_record \ + --eval_data_path=${GLUE_DIR}/${TASK}_eval.tf_record \ + --bert_config_file=${BERT_DIR}/bert_config.json \ + --init_checkpoint=${BERT_DIR}/bert_model.ckpt \ + --train_batch_size=32 \ + --eval_batch_size=32 \ + --steps_per_loop=1000 \ + --learning_rate=2e-5 \ + --num_train_epochs=3 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=tpu \ + --tpu=grpc://${TPU_IP_ADDRESS}:8470 +``` + +Note that, we specify `steps_per_loop=1000` for TPU, because running a loop of +training steps inside a `tf.function` can significantly increase TPU utilization +and callbacks will not be called inside the loop. + +### SQuAD 1.1 + +The Stanford Question Answering Dataset (SQuAD) is a popular question answering +benchmark dataset. See more in [SQuAD website](https://rajpurkar.github.io/SQuAD-explorer/). + +We use the `BERT-Large` (uncased_L-24_H-1024_A-16) as an example throughout the +workflow. +For GPU memory of 16GB or smaller, you may try to use `BERT-Base` +(uncased_L-12_H-768_A-12). + +```shell +export BERT_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 +export SQUAD_DIR=gs://some_bucket/datasets +export MODEL_DIR=gs://some_bucket/my_output_dir +export SQUAD_VERSION=v1.1 + +python run_squad.py \ + --input_meta_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_meta_data \ + --train_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_train.tf_record \ + --predict_file=${SQUAD_DIR}/dev-v1.1.json \ + --vocab_file=${BERT_DIR}/vocab.txt \ + --bert_config_file=${BERT_DIR}/bert_config.json \ + --init_checkpoint=${BERT_DIR}/bert_model.ckpt \ + --train_batch_size=4 \ + --predict_batch_size=4 \ + --learning_rate=8e-5 \ + --num_train_epochs=2 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=mirrored +``` + +Similarily, you can replace `init_checkpoint` FLAG with `hub_module_url` to +specify a hub module path. + +`run_squad.py` writes the prediction for `--predict_file` by default. If you set +the `--model=predict` and offer the SQuAD test data, the scripts will generate +the prediction json file. + +To use TPU, you need switch distribution strategy type to `tpu` with TPU +information. + +```shell +export BERT_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 +export TPU_IP_ADDRESS='???' +export MODEL_DIR=gs://some_bucket/my_output_dir +export SQUAD_DIR=gs://some_bucket/datasets +export SQUAD_VERSION=v1.1 + +python run_squad.py \ + --input_meta_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_meta_data \ + --train_data_path=${SQUAD_DIR}/squad_${SQUAD_VERSION}_train.tf_record \ + --predict_file=${SQUAD_DIR}/dev-v1.1.json \ + --vocab_file=${BERT_DIR}/vocab.txt \ + --bert_config_file=${BERT_DIR}/bert_config.json \ + --init_checkpoint=${BERT_DIR}/bert_model.ckpt \ + --train_batch_size=32 \ + --learning_rate=8e-5 \ + --num_train_epochs=2 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=tpu \ + --tpu=grpc://${TPU_IP_ADDRESS}:8470 +``` + +The dev set predictions will be saved into a file called predictions.json in the +model_dir: + +```shell +python $SQUAD_DIR/evaluate-v1.1.py $SQUAD_DIR/dev-v1.1.json ./squad/predictions.json +``` + + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/__init__.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a25710c222e3327cb20e000db5df5c5651c4a2cc --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/bert_cloud_tpu.md b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/bert_cloud_tpu.md new file mode 100644 index 0000000000000000000000000000000000000000..baf6f9bdc0c155cb53b30cea5f404aa166c3a2c6 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/bert_cloud_tpu.md @@ -0,0 +1,110 @@ +# BERT FineTuning with Cloud TPU: Sentence and Sentence-Pair Classification Tasks (TF 2.1) +This tutorial shows you how to train the Bidirectional Encoder Representations from Transformers (BERT) model on Cloud TPU. + + +## Set up Cloud Storage and Compute Engine VM +1. [Open a cloud shell window](https://console.cloud.google.com/?cloudshell=true&_ga=2.11844148.-1612541229.1552429951) +2. Create a variable for the project's id: +``` +export PROJECT_ID=your-project_id +``` +3. Configure `gcloud` command-line tool to use the project where you want to create Cloud TPU. +``` +gcloud config set project ${PROJECT_ID} +``` +4. Create a Cloud Storage bucket using the following command: +``` +gsutil mb -p ${PROJECT_ID} -c standard -l europe-west4 -b on gs://your-bucket-name +``` +This Cloud Storage bucket stores the data you use to train your model and the training results. +5. Launch a Compute Engine VM and Cloud TPU using the ctpu up command. +``` +ctpu up --tpu-size=v3-8 \ + --machine-type=n1-standard-8 \ + --zone=europe-west4-a \ + --tf-version=2.1 [optional flags: --project, --name] +``` +6. The configuration you specified appears. Enter y to approve or n to cancel. +7. When the ctpu up command has finished executing, verify that your shell prompt has changed from username@project to username@tpuname. This change shows that you are now logged into your Compute Engine VM. +``` +gcloud compute ssh vm-name --zone=europe-west4-a +(vm)$ export TPU_NAME=vm-name +``` +As you continue these instructions, run each command that begins with `(vm)$` in your VM session window. + +## Prepare the Dataset +1. From your Compute Engine virtual machine (VM), install requirements.txt. +``` +(vm)$ cd /usr/share/models +(vm)$ sudo pip3 install -r official/requirements.txt +``` +2. Optional: download download_glue_data.py + +This tutorial uses the General Language Understanding Evaluation (GLUE) benchmark to evaluate and analyze the performance of the model. The GLUE data is provided for this tutorial at gs://cloud-tpu-checkpoints/bert/classification. + +## Define parameter values +Next, define several parameter values that are required when you train and evaluate your model: + +``` +(vm)$ export PYTHONPATH="$PYTHONPATH:/usr/share/tpu/models" +(vm)$ export STORAGE_BUCKET=gs://your-bucket-name +(vm)$ export BERT_BASE_DIR=gs://cloud-tpu-checkpoints/bert/keras_bert/uncased_L-24_H-1024_A-16 +(vm)$ export MODEL_DIR=${STORAGE_BUCKET}/bert-output +(vm)$ export GLUE_DIR=gs://cloud-tpu-checkpoints/bert/classification +(vm)$ export TASK=mnli +``` + +## Train the model +From your Compute Engine VM, run the following command. + +``` +(vm)$ python3 official/nlp/bert/run_classifier.py \ + --mode='train_and_eval' \ + --input_meta_data_path=${GLUE_DIR}/${TASK}_meta_data \ + --train_data_path=${GLUE_DIR}/${TASK}_train.tf_record \ + --eval_data_path=${GLUE_DIR}/${TASK}_eval.tf_record \ + --bert_config_file=$BERT_BASE_DIR/bert_config.json \ + --init_checkpoint=$BERT_BASE_DIR/bert_model.ckpt \ + --train_batch_size=32 \ + --eval_batch_size=32 \ + --learning_rate=2e-5 \ + --num_train_epochs=3 \ + --model_dir=${MODEL_DIR} \ + --distribution_strategy=tpu \ + --tpu=${TPU_NAME} +``` + +## Verify your results +The training takes approximately 1 hour on a v3-8 TPU. When script completes, you should see results similar to the following: +``` +Training Summary: +{'train_loss': 0.28142181038856506, +'last_train_metrics': 0.9467429518699646, +'eval_metrics': 0.8599063158035278, +'total_training_steps': 36813} +``` + +## Clean up +To avoid incurring charges to your GCP account for the resources used in this topic: +1. Disconnect from the Compute Engine VM: +``` +(vm)$ exit +``` +2. In your Cloud Shell, run ctpu delete with the --zone flag you used when you set up the Cloud TPU to delete your Compute Engine VM and your Cloud TPU: +``` +$ ctpu delete --zone=your-zone +``` +3. Run ctpu status specifying your zone to make sure you have no instances allocated to avoid unnecessary charges for TPU usage. The deletion might take several minutes. A response like the one below indicates there are no more allocated instances: +``` +$ ctpu status --zone=your-zone +``` +4. Run gsutil as shown, replacing your-bucket with the name of the Cloud Storage bucket you created for this tutorial: +``` +$ gsutil rm -r gs://your-bucket +``` + + + + + + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/bert_models.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/bert_models.py new file mode 100644 index 0000000000000000000000000000000000000000..8a0fceb725e9eddc8e8ea25924d4013fe88b2048 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/bert_models.py @@ -0,0 +1,366 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""BERT models that are compatible with TF 2.0.""" + +import gin +import tensorflow as tf +import tensorflow_hub as hub + +from official.modeling import tf_utils +from official.nlp.albert import configs as albert_configs +from official.nlp.bert import configs +from official.nlp.modeling import models +from official.nlp.modeling import networks + + +class BertPretrainLossAndMetricLayer(tf.keras.layers.Layer): + """Returns layer that computes custom loss and metrics for pretraining.""" + + def __init__(self, vocab_size, **kwargs): + super(BertPretrainLossAndMetricLayer, self).__init__(**kwargs) + self._vocab_size = vocab_size + self.config = { + 'vocab_size': vocab_size, + } + + def _add_metrics(self, lm_output, lm_labels, lm_label_weights, + lm_example_loss, sentence_output, sentence_labels, + next_sentence_loss): + """Adds metrics.""" + masked_lm_accuracy = tf.keras.metrics.sparse_categorical_accuracy( + lm_labels, lm_output) + numerator = tf.reduce_sum(masked_lm_accuracy * lm_label_weights) + denominator = tf.reduce_sum(lm_label_weights) + 1e-5 + masked_lm_accuracy = numerator / denominator + self.add_metric( + masked_lm_accuracy, name='masked_lm_accuracy', aggregation='mean') + + self.add_metric(lm_example_loss, name='lm_example_loss', aggregation='mean') + + if sentence_labels is not None: + next_sentence_accuracy = tf.keras.metrics.sparse_categorical_accuracy( + sentence_labels, sentence_output) + self.add_metric( + next_sentence_accuracy, + name='next_sentence_accuracy', + aggregation='mean') + + if next_sentence_loss is not None: + self.add_metric( + next_sentence_loss, name='next_sentence_loss', aggregation='mean') + + def call(self, + lm_output_logits, + sentence_output_logits, + lm_label_ids, + lm_label_weights, + sentence_labels=None): + """Implements call() for the layer.""" + lm_label_weights = tf.cast(lm_label_weights, tf.float32) + lm_output_logits = tf.cast(lm_output_logits, tf.float32) + + lm_prediction_losses = tf.keras.losses.sparse_categorical_crossentropy( + lm_label_ids, lm_output_logits, from_logits=True) + lm_numerator_loss = tf.reduce_sum(lm_prediction_losses * lm_label_weights) + lm_denominator_loss = tf.reduce_sum(lm_label_weights) + mask_label_loss = tf.math.divide_no_nan(lm_numerator_loss, + lm_denominator_loss) + + if sentence_labels is not None: + sentence_output_logits = tf.cast(sentence_output_logits, tf.float32) + sentence_loss = tf.keras.losses.sparse_categorical_crossentropy( + sentence_labels, sentence_output_logits, from_logits=True) + sentence_loss = tf.reduce_mean(sentence_loss) + loss = mask_label_loss + sentence_loss + else: + sentence_loss = None + loss = mask_label_loss + + batch_shape = tf.slice(tf.shape(lm_label_ids), [0], [1]) + # TODO(hongkuny): Avoids the hack and switches add_loss. + final_loss = tf.fill(batch_shape, loss) + + self._add_metrics(lm_output_logits, lm_label_ids, lm_label_weights, + mask_label_loss, sentence_output_logits, sentence_labels, + sentence_loss) + return final_loss + + +@gin.configurable +def get_transformer_encoder(bert_config, + sequence_length=None, + transformer_encoder_cls=None, + output_range=None): + """Gets a 'TransformerEncoder' object. + + Args: + bert_config: A 'modeling.BertConfig' or 'modeling.AlbertConfig' object. + sequence_length: [Deprecated]. + transformer_encoder_cls: A EncoderScaffold class. If it is None, uses the + default BERT encoder implementation. + output_range: the sequence output range, [0, output_range). Default setting + is to return the entire sequence output. + + Returns: + A encoder object. + """ + del sequence_length + if transformer_encoder_cls is not None: + # TODO(hongkuny): evaluate if it is better to put cfg definition in gin. + embedding_cfg = dict( + vocab_size=bert_config.vocab_size, + type_vocab_size=bert_config.type_vocab_size, + hidden_size=bert_config.hidden_size, + max_seq_length=bert_config.max_position_embeddings, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=bert_config.initializer_range), + dropout_rate=bert_config.hidden_dropout_prob, + ) + hidden_cfg = dict( + num_attention_heads=bert_config.num_attention_heads, + intermediate_size=bert_config.intermediate_size, + intermediate_activation=tf_utils.get_activation(bert_config.hidden_act), + dropout_rate=bert_config.hidden_dropout_prob, + attention_dropout_rate=bert_config.attention_probs_dropout_prob, + kernel_initializer=tf.keras.initializers.TruncatedNormal( + stddev=bert_config.initializer_range), + ) + kwargs = dict( + embedding_cfg=embedding_cfg, + hidden_cfg=hidden_cfg, + num_hidden_instances=bert_config.num_hidden_layers, + pooled_output_dim=bert_config.hidden_size, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=bert_config.initializer_range)) + + # Relies on gin configuration to define the Transformer encoder arguments. + return transformer_encoder_cls(**kwargs) + + kwargs = dict( + vocab_size=bert_config.vocab_size, + hidden_size=bert_config.hidden_size, + num_layers=bert_config.num_hidden_layers, + num_attention_heads=bert_config.num_attention_heads, + intermediate_size=bert_config.intermediate_size, + activation=tf_utils.get_activation(bert_config.hidden_act), + dropout_rate=bert_config.hidden_dropout_prob, + attention_dropout_rate=bert_config.attention_probs_dropout_prob, + max_sequence_length=bert_config.max_position_embeddings, + type_vocab_size=bert_config.type_vocab_size, + embedding_width=bert_config.embedding_size, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=bert_config.initializer_range)) + if isinstance(bert_config, albert_configs.AlbertConfig): + return networks.AlbertEncoder(**kwargs) + else: + assert isinstance(bert_config, configs.BertConfig) + kwargs['output_range'] = output_range + return networks.BertEncoder(**kwargs) + + +def pretrain_model(bert_config, + seq_length, + max_predictions_per_seq, + initializer=None, + use_next_sentence_label=True, + return_core_pretrainer_model=False): + """Returns model to be used for pre-training. + + Args: + bert_config: Configuration that defines the core BERT model. + seq_length: Maximum sequence length of the training data. + max_predictions_per_seq: Maximum number of tokens in sequence to mask out + and use for pretraining. + initializer: Initializer for weights in BertPretrainer. + use_next_sentence_label: Whether to use the next sentence label. + return_core_pretrainer_model: Whether to also return the `BertPretrainer` + object. + + Returns: + A Tuple of (1) Pretraining model, (2) core BERT submodel from which to + save weights after pretraining, and (3) optional core `BertPretrainer` + object if argument `return_core_pretrainer_model` is True. + """ + input_word_ids = tf.keras.layers.Input( + shape=(seq_length,), name='input_word_ids', dtype=tf.int32) + input_mask = tf.keras.layers.Input( + shape=(seq_length,), name='input_mask', dtype=tf.int32) + input_type_ids = tf.keras.layers.Input( + shape=(seq_length,), name='input_type_ids', dtype=tf.int32) + masked_lm_positions = tf.keras.layers.Input( + shape=(max_predictions_per_seq,), + name='masked_lm_positions', + dtype=tf.int32) + masked_lm_ids = tf.keras.layers.Input( + shape=(max_predictions_per_seq,), name='masked_lm_ids', dtype=tf.int32) + masked_lm_weights = tf.keras.layers.Input( + shape=(max_predictions_per_seq,), + name='masked_lm_weights', + dtype=tf.int32) + + if use_next_sentence_label: + next_sentence_labels = tf.keras.layers.Input( + shape=(1,), name='next_sentence_labels', dtype=tf.int32) + else: + next_sentence_labels = None + + transformer_encoder = get_transformer_encoder(bert_config, seq_length) + if initializer is None: + initializer = tf.keras.initializers.TruncatedNormal( + stddev=bert_config.initializer_range) + pretrainer_model = models.BertPretrainer( + network=transformer_encoder, + embedding_table=transformer_encoder.get_embedding_table(), + num_classes=2, # The next sentence prediction label has two classes. + activation=tf_utils.get_activation(bert_config.hidden_act), + num_token_predictions=max_predictions_per_seq, + initializer=initializer, + output='logits') + + outputs = pretrainer_model( + [input_word_ids, input_mask, input_type_ids, masked_lm_positions]) + lm_output = outputs['masked_lm'] + sentence_output = outputs['classification'] + pretrain_loss_layer = BertPretrainLossAndMetricLayer( + vocab_size=bert_config.vocab_size) + output_loss = pretrain_loss_layer(lm_output, sentence_output, masked_lm_ids, + masked_lm_weights, next_sentence_labels) + inputs = { + 'input_word_ids': input_word_ids, + 'input_mask': input_mask, + 'input_type_ids': input_type_ids, + 'masked_lm_positions': masked_lm_positions, + 'masked_lm_ids': masked_lm_ids, + 'masked_lm_weights': masked_lm_weights, + } + if use_next_sentence_label: + inputs['next_sentence_labels'] = next_sentence_labels + + keras_model = tf.keras.Model(inputs=inputs, outputs=output_loss) + if return_core_pretrainer_model: + return keras_model, transformer_encoder, pretrainer_model + else: + return keras_model, transformer_encoder + + +def squad_model(bert_config, + max_seq_length, + initializer=None, + hub_module_url=None, + hub_module_trainable=True): + """Returns BERT Squad model along with core BERT model to import weights. + + Args: + bert_config: BertConfig, the config defines the core Bert model. + max_seq_length: integer, the maximum input sequence length. + initializer: Initializer for the final dense layer in the span labeler. + Defaulted to TruncatedNormal initializer. + hub_module_url: TF-Hub path/url to Bert module. + hub_module_trainable: True to finetune layers in the hub module. + + Returns: + A tuple of (1) keras model that outputs start logits and end logits and + (2) the core BERT transformer encoder. + """ + if initializer is None: + initializer = tf.keras.initializers.TruncatedNormal( + stddev=bert_config.initializer_range) + if not hub_module_url: + bert_encoder = get_transformer_encoder(bert_config, max_seq_length) + return models.BertSpanLabeler( + network=bert_encoder, initializer=initializer), bert_encoder + + input_word_ids = tf.keras.layers.Input( + shape=(max_seq_length,), dtype=tf.int32, name='input_word_ids') + input_mask = tf.keras.layers.Input( + shape=(max_seq_length,), dtype=tf.int32, name='input_mask') + input_type_ids = tf.keras.layers.Input( + shape=(max_seq_length,), dtype=tf.int32, name='input_type_ids') + core_model = hub.KerasLayer(hub_module_url, trainable=hub_module_trainable) + pooled_output, sequence_output = core_model( + [input_word_ids, input_mask, input_type_ids]) + bert_encoder = tf.keras.Model( + inputs={ + 'input_word_ids': input_word_ids, + 'input_mask': input_mask, + 'input_type_ids': input_type_ids, + }, + outputs=[sequence_output, pooled_output], + name='core_model') + return models.BertSpanLabeler( + network=bert_encoder, initializer=initializer), bert_encoder + + +def classifier_model(bert_config, + num_labels, + max_seq_length=None, + final_layer_initializer=None, + hub_module_url=None, + hub_module_trainable=True): + """BERT classifier model in functional API style. + + Construct a Keras model for predicting `num_labels` outputs from an input with + maximum sequence length `max_seq_length`. + + Args: + bert_config: BertConfig or AlbertConfig, the config defines the core BERT or + ALBERT model. + num_labels: integer, the number of classes. + max_seq_length: integer, the maximum input sequence length. + final_layer_initializer: Initializer for final dense layer. Defaulted + TruncatedNormal initializer. + hub_module_url: TF-Hub path/url to Bert module. + hub_module_trainable: True to finetune layers in the hub module. + + Returns: + Combined prediction model (words, mask, type) -> (one-hot labels) + BERT sub-model (words, mask, type) -> (bert_outputs) + """ + if final_layer_initializer is not None: + initializer = final_layer_initializer + else: + initializer = tf.keras.initializers.TruncatedNormal( + stddev=bert_config.initializer_range) + + if not hub_module_url: + bert_encoder = get_transformer_encoder( + bert_config, max_seq_length, output_range=1) + return models.BertClassifier( + bert_encoder, + num_classes=num_labels, + dropout_rate=bert_config.hidden_dropout_prob, + initializer=initializer), bert_encoder + + input_word_ids = tf.keras.layers.Input( + shape=(max_seq_length,), dtype=tf.int32, name='input_word_ids') + input_mask = tf.keras.layers.Input( + shape=(max_seq_length,), dtype=tf.int32, name='input_mask') + input_type_ids = tf.keras.layers.Input( + shape=(max_seq_length,), dtype=tf.int32, name='input_type_ids') + bert_model = hub.KerasLayer(hub_module_url, trainable=hub_module_trainable) + pooled_output, _ = bert_model([input_word_ids, input_mask, input_type_ids]) + output = tf.keras.layers.Dropout(rate=bert_config.hidden_dropout_prob)( + pooled_output) + + output = tf.keras.layers.Dense( + num_labels, kernel_initializer=initializer, name='output')( + output) + return tf.keras.Model( + inputs={ + 'input_word_ids': input_word_ids, + 'input_mask': input_mask, + 'input_type_ids': input_type_ids + }, + outputs=output), bert_model diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/bert_models_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/bert_models_test.py new file mode 100644 index 0000000000000000000000000000000000000000..8c4a52a20d343e3d7cc5f0ccac250d5f4f036667 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/bert_models_test.py @@ -0,0 +1,106 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tensorflow as tf + +from official.nlp.bert import bert_models +from official.nlp.bert import configs as bert_configs +from official.nlp.modeling import networks + + +class BertModelsTest(tf.test.TestCase): + + def setUp(self): + super(BertModelsTest, self).setUp() + self._bert_test_config = bert_configs.BertConfig( + attention_probs_dropout_prob=0.0, + hidden_act='gelu', + hidden_dropout_prob=0.0, + hidden_size=16, + initializer_range=0.02, + intermediate_size=32, + max_position_embeddings=128, + num_attention_heads=2, + num_hidden_layers=2, + type_vocab_size=2, + vocab_size=30522) + + def test_pretrain_model(self): + model, encoder = bert_models.pretrain_model( + self._bert_test_config, + seq_length=5, + max_predictions_per_seq=2, + initializer=None, + use_next_sentence_label=True) + self.assertIsInstance(model, tf.keras.Model) + self.assertIsInstance(encoder, networks.BertEncoder) + + # model has one scalar output: loss value. + self.assertEqual(model.output.shape.as_list(), [ + None, + ]) + + # Expect two output from encoder: sequence and classification output. + self.assertIsInstance(encoder.output, list) + self.assertLen(encoder.output, 2) + # shape should be [batch size, hidden_size] + self.assertEqual(encoder.output[1].shape.as_list(), [None, 16]) + + def test_squad_model(self): + model, core_model = bert_models.squad_model( + self._bert_test_config, + max_seq_length=5, + initializer=None, + hub_module_url=None, + hub_module_trainable=None) + self.assertIsInstance(model, tf.keras.Model) + self.assertIsInstance(core_model, tf.keras.Model) + + # Expect two output from model: start positions and end positions + self.assertIsInstance(model.output, list) + self.assertLen(model.output, 2) + + # Expect two output from core_model: sequence and classification output. + self.assertIsInstance(core_model.output, list) + self.assertLen(core_model.output, 2) + # shape should be [batch size, None, hidden_size] + self.assertEqual(core_model.output[0].shape.as_list(), [None, None, 16]) + # shape should be [batch size, hidden_size] + self.assertEqual(core_model.output[1].shape.as_list(), [None, 16]) + + def test_classifier_model(self): + model, core_model = bert_models.classifier_model( + self._bert_test_config, + num_labels=3, + max_seq_length=5, + final_layer_initializer=None, + hub_module_url=None, + hub_module_trainable=None) + self.assertIsInstance(model, tf.keras.Model) + self.assertIsInstance(core_model, tf.keras.Model) + + # model has one classification output with num_labels=3. + self.assertEqual(model.output.shape.as_list(), [None, 3]) + + # Expect two output from core_model: sequence and classification output. + self.assertIsInstance(core_model.output, list) + self.assertLen(core_model.output, 2) + # shape should be [batch size, None, hidden_size] + self.assertEqual(core_model.output[0].shape.as_list(), [None, None, 16]) + # shape should be [batch size, hidden_size] + self.assertEqual(core_model.output[1].shape.as_list(), [None, 16]) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/common_flags.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/common_flags.py new file mode 100644 index 0000000000000000000000000000000000000000..55570052b99e850c6e6861f00c39a7a3fabf3729 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/common_flags.py @@ -0,0 +1,129 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Defining common flags used across all BERT models/applications.""" + +from absl import flags +import tensorflow as tf + +from official.utils import hyperparams_flags +from official.utils.flags import core as flags_core + + +def define_common_bert_flags(): + """Define common flags for BERT tasks.""" + flags_core.define_base( + data_dir=False, + model_dir=True, + clean=False, + train_epochs=False, + epochs_between_evals=False, + stop_threshold=False, + batch_size=False, + num_gpu=True, + export_dir=False, + distribution_strategy=True, + run_eagerly=True) + flags_core.define_distribution() + flags.DEFINE_string('bert_config_file', None, + 'Bert configuration file to define core bert layers.') + flags.DEFINE_string( + 'model_export_path', None, + 'Path to the directory, where trainined model will be ' + 'exported.') + flags.DEFINE_string('tpu', '', 'TPU address to connect to.') + flags.DEFINE_string( + 'init_checkpoint', None, + 'Initial checkpoint (usually from a pre-trained BERT model).') + flags.DEFINE_integer('num_train_epochs', 3, + 'Total number of training epochs to perform.') + flags.DEFINE_integer( + 'steps_per_loop', None, + 'Number of steps per graph-mode loop. Only training step ' + 'happens inside the loop. Callbacks will not be called ' + 'inside. If not set the value will be configured depending on the ' + 'devices available.') + flags.DEFINE_float('learning_rate', 5e-5, + 'The initial learning rate for Adam.') + flags.DEFINE_float('end_lr', 0.0, + 'The end learning rate for learning rate decay.') + flags.DEFINE_string('optimizer_type', 'adamw', + 'The type of optimizer to use for training (adamw|lamb)') + flags.DEFINE_boolean( + 'scale_loss', False, + 'Whether to divide the loss by number of replica inside the per-replica ' + 'loss function.') + flags.DEFINE_boolean( + 'use_keras_compile_fit', False, + 'If True, uses Keras compile/fit() API for training logic. Otherwise ' + 'use custom training loop.') + flags.DEFINE_string( + 'hub_module_url', None, 'TF-Hub path/url to Bert module. ' + 'If specified, init_checkpoint flag should not be used.') + flags.DEFINE_bool('hub_module_trainable', True, + 'True to make keras layers in the hub module trainable.') + flags.DEFINE_string( + 'sub_model_export_name', None, + 'If set, `sub_model` checkpoints are exported into ' + 'FLAGS.model_dir/FLAGS.sub_model_export_name.') + flags.DEFINE_bool('explicit_allreduce', False, + 'True to use explicit allreduce instead of the implicit ' + 'allreduce in optimizer.apply_gradients(). If fp16 mixed ' + 'precision training is used, this also enables allreduce ' + 'gradients in fp16.') + flags.DEFINE_integer('allreduce_bytes_per_pack', 0, + 'Number of bytes of a gradient pack for allreduce. ' + 'Should be positive integer, if set to 0, all ' + 'gradients are in one pack. Breaking gradient into ' + 'packs could enable overlap between allreduce and ' + 'backprop computation. This flag only takes effect ' + 'when explicit_allreduce is set to True.') + + flags_core.define_log_steps() + + # Adds flags for mixed precision and multi-worker training. + flags_core.define_performance( + num_parallel_calls=False, + inter_op=False, + intra_op=False, + synthetic_data=False, + max_train_steps=False, + dtype=True, + loss_scale=True, + all_reduce_alg=True, + num_packs=False, + tf_gpu_thread_mode=True, + datasets_num_private_threads=True, + enable_xla=True, + fp16_implementation=True, + ) + + # Adds gin configuration flags. + hyperparams_flags.define_gin_flags() + + +def dtype(): + return flags_core.get_tf_dtype(flags.FLAGS) + + +def use_float16(): + return flags_core.get_tf_dtype(flags.FLAGS) == tf.float16 + + +def use_graph_rewrite(): + return flags.FLAGS.fp16_implementation == 'graph_rewrite' + + +def get_loss_scale(): + return flags_core.get_loss_scale(flags.FLAGS, default_for_fp16='dynamic') diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/configs.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/configs.py new file mode 100644 index 0000000000000000000000000000000000000000..950c32d0bfad3e06f3d14baf042a916de2eb2828 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/configs.py @@ -0,0 +1,104 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The main BERT model and related functions.""" + +import copy +import json + +import six +import tensorflow as tf + + +class BertConfig(object): + """Configuration for `BertModel`.""" + + def __init__(self, + vocab_size, + hidden_size=768, + num_hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + initializer_range=0.02, + embedding_size=None, + backward_compatible=True): + """Constructs BertConfig. + + Args: + vocab_size: Vocabulary size of `inputs_ids` in `BertModel`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. + hidden_dropout_prob: The dropout probability for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `BertModel`. + initializer_range: The stdev of the truncated_normal_initializer for + initializing all weight matrices. + embedding_size: (Optional) width of the factorized word embeddings. + backward_compatible: Boolean, whether the variables shape are compatible + with checkpoints converted from TF 1.x BERT. + """ + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.embedding_size = embedding_size + self.backward_compatible = backward_compatible + + @classmethod + def from_dict(cls, json_object): + """Constructs a `BertConfig` from a Python dictionary of parameters.""" + config = BertConfig(vocab_size=None) + for (key, value) in six.iteritems(json_object): + config.__dict__[key] = value + return config + + @classmethod + def from_json_file(cls, json_file): + """Constructs a `BertConfig` from a json file of parameters.""" + with tf.io.gfile.GFile(json_file, "r") as reader: + text = reader.read() + return cls.from_dict(json.loads(text)) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/export_tfhub.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/export_tfhub.py new file mode 100644 index 0000000000000000000000000000000000000000..833e7c10582f9252f59b3b7584a5bcca0b6f4991 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/export_tfhub.py @@ -0,0 +1,139 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A script to export BERT as a TF-Hub SavedModel. + +This script is **DEPRECATED** for exporting BERT encoder models; +see the error message in by main() for details. +""" + +from typing import Text + +# Import libraries +from absl import app +from absl import flags +from absl import logging +import tensorflow as tf +from official.nlp.bert import bert_models +from official.nlp.bert import configs + +FLAGS = flags.FLAGS + +flags.DEFINE_string("bert_config_file", None, + "Bert configuration file to define core bert layers.") +flags.DEFINE_string("model_checkpoint_path", None, + "File path to TF model checkpoint.") +flags.DEFINE_string("export_path", None, "TF-Hub SavedModel destination path.") +flags.DEFINE_string("vocab_file", None, + "The vocabulary file that the BERT model was trained on.") +flags.DEFINE_bool( + "do_lower_case", None, "Whether to lowercase. If None, " + "do_lower_case will be enabled if 'uncased' appears in the " + "name of --vocab_file") +flags.DEFINE_enum("model_type", "encoder", ["encoder", "squad"], + "What kind of BERT model to export.") + + +def create_bert_model(bert_config: configs.BertConfig) -> tf.keras.Model: + """Creates a BERT keras core model from BERT configuration. + + Args: + bert_config: A `BertConfig` to create the core model. + + Returns: + A keras model. + """ + # Adds input layers just as placeholders. + input_word_ids = tf.keras.layers.Input( + shape=(None,), dtype=tf.int32, name="input_word_ids") + input_mask = tf.keras.layers.Input( + shape=(None,), dtype=tf.int32, name="input_mask") + input_type_ids = tf.keras.layers.Input( + shape=(None,), dtype=tf.int32, name="input_type_ids") + transformer_encoder = bert_models.get_transformer_encoder( + bert_config, sequence_length=None) + sequence_output, pooled_output = transformer_encoder( + [input_word_ids, input_mask, input_type_ids]) + # To keep consistent with legacy hub modules, the outputs are + # "pooled_output" and "sequence_output". + return tf.keras.Model( + inputs=[input_word_ids, input_mask, input_type_ids], + outputs=[pooled_output, sequence_output]), transformer_encoder + + +def export_bert_tfhub(bert_config: configs.BertConfig, + model_checkpoint_path: Text, + hub_destination: Text, + vocab_file: Text, + do_lower_case: bool = None): + """Restores a tf.keras.Model and saves for TF-Hub.""" + # If do_lower_case is not explicit, default to checking whether "uncased" is + # in the vocab file name + if do_lower_case is None: + do_lower_case = "uncased" in vocab_file + logging.info("Using do_lower_case=%s based on name of vocab_file=%s", + do_lower_case, vocab_file) + core_model, encoder = create_bert_model(bert_config) + checkpoint = tf.train.Checkpoint( + model=encoder, # Legacy checkpoints. + encoder=encoder) + checkpoint.restore(model_checkpoint_path).assert_existing_objects_matched() + core_model.vocab_file = tf.saved_model.Asset(vocab_file) + core_model.do_lower_case = tf.Variable(do_lower_case, trainable=False) + core_model.save(hub_destination, include_optimizer=False, save_format="tf") + + +def export_bert_squad_tfhub(bert_config: configs.BertConfig, + model_checkpoint_path: Text, + hub_destination: Text, + vocab_file: Text, + do_lower_case: bool = None): + """Restores a tf.keras.Model for BERT with SQuAD and saves for TF-Hub.""" + # If do_lower_case is not explicit, default to checking whether "uncased" is + # in the vocab file name + if do_lower_case is None: + do_lower_case = "uncased" in vocab_file + logging.info("Using do_lower_case=%s based on name of vocab_file=%s", + do_lower_case, vocab_file) + span_labeling, _ = bert_models.squad_model(bert_config, max_seq_length=None) + checkpoint = tf.train.Checkpoint(model=span_labeling) + checkpoint.restore(model_checkpoint_path).assert_existing_objects_matched() + span_labeling.vocab_file = tf.saved_model.Asset(vocab_file) + span_labeling.do_lower_case = tf.Variable(do_lower_case, trainable=False) + span_labeling.save(hub_destination, include_optimizer=False, save_format="tf") + + +def main(_): + bert_config = configs.BertConfig.from_json_file(FLAGS.bert_config_file) + if FLAGS.model_type == "encoder": + deprecation_note = ( + "nlp/bert/export_tfhub is **DEPRECATED** for exporting BERT encoder " + "models. Please switch to nlp/tools/export_tfhub for exporting BERT " + "(and other) encoders with dict inputs/outputs conforming to " + "https://www.tensorflow.org/hub/common_saved_model_apis/text#transformer-encoders" + ) + logging.error(deprecation_note) + print("\n\nNOTICE:", deprecation_note, "\n") + export_bert_tfhub(bert_config, FLAGS.model_checkpoint_path, + FLAGS.export_path, FLAGS.vocab_file, FLAGS.do_lower_case) + elif FLAGS.model_type == "squad": + export_bert_squad_tfhub(bert_config, FLAGS.model_checkpoint_path, + FLAGS.export_path, FLAGS.vocab_file, + FLAGS.do_lower_case) + else: + raise ValueError("Unsupported model_type %s." % FLAGS.model_type) + + +if __name__ == "__main__": + app.run(main) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/export_tfhub_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/export_tfhub_test.py new file mode 100644 index 0000000000000000000000000000000000000000..77030dd3fde7d4c4d73bea0fdea017848b1e253f --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/export_tfhub_test.py @@ -0,0 +1,108 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests official.nlp.bert.export_tfhub.""" + +import os + +from absl.testing import parameterized +import numpy as np +import tensorflow as tf +import tensorflow_hub as hub + +from official.nlp.bert import configs +from official.nlp.bert import export_tfhub + + +class ExportTfhubTest(tf.test.TestCase, parameterized.TestCase): + + @parameterized.parameters("model", "encoder") + def test_export_tfhub(self, ckpt_key_name): + # Exports a savedmodel for TF-Hub + hidden_size = 16 + bert_config = configs.BertConfig( + vocab_size=100, + hidden_size=hidden_size, + intermediate_size=32, + max_position_embeddings=128, + num_attention_heads=2, + num_hidden_layers=1) + bert_model, encoder = export_tfhub.create_bert_model(bert_config) + model_checkpoint_dir = os.path.join(self.get_temp_dir(), "checkpoint") + checkpoint = tf.train.Checkpoint(**{ckpt_key_name: encoder}) + checkpoint.save(os.path.join(model_checkpoint_dir, "test")) + model_checkpoint_path = tf.train.latest_checkpoint(model_checkpoint_dir) + + vocab_file = os.path.join(self.get_temp_dir(), "uncased_vocab.txt") + with tf.io.gfile.GFile(vocab_file, "w") as f: + f.write("dummy content") + + hub_destination = os.path.join(self.get_temp_dir(), "hub") + export_tfhub.export_bert_tfhub(bert_config, model_checkpoint_path, + hub_destination, vocab_file) + + # Restores a hub KerasLayer. + hub_layer = hub.KerasLayer(hub_destination, trainable=True) + + if hasattr(hub_layer, "resolved_object"): + # Checks meta attributes. + self.assertTrue(hub_layer.resolved_object.do_lower_case.numpy()) + with tf.io.gfile.GFile( + hub_layer.resolved_object.vocab_file.asset_path.numpy()) as f: + self.assertEqual("dummy content", f.read()) + # Checks the hub KerasLayer. + for source_weight, hub_weight in zip(bert_model.trainable_weights, + hub_layer.trainable_weights): + self.assertAllClose(source_weight.numpy(), hub_weight.numpy()) + + seq_length = 10 + dummy_ids = np.zeros((2, seq_length), dtype=np.int32) + hub_outputs = hub_layer([dummy_ids, dummy_ids, dummy_ids]) + source_outputs = bert_model([dummy_ids, dummy_ids, dummy_ids]) + + # The outputs of hub module are "pooled_output" and "sequence_output", + # while the outputs of encoder is in reversed order, i.e., + # "sequence_output" and "pooled_output". + encoder_outputs = reversed(encoder([dummy_ids, dummy_ids, dummy_ids])) + self.assertEqual(hub_outputs[0].shape, (2, hidden_size)) + self.assertEqual(hub_outputs[1].shape, (2, seq_length, hidden_size)) + for source_output, hub_output, encoder_output in zip( + source_outputs, hub_outputs, encoder_outputs): + self.assertAllClose(source_output.numpy(), hub_output.numpy()) + self.assertAllClose(source_output.numpy(), encoder_output.numpy()) + + # Test that training=True makes a difference (activates dropout). + def _dropout_mean_stddev(training, num_runs=20): + input_ids = np.array([[14, 12, 42, 95, 99]], np.int32) + inputs = [input_ids, np.ones_like(input_ids), np.zeros_like(input_ids)] + outputs = np.concatenate( + [hub_layer(inputs, training=training)[0] for _ in range(num_runs)]) + return np.mean(np.std(outputs, axis=0)) + + self.assertLess(_dropout_mean_stddev(training=False), 1e-6) + self.assertGreater(_dropout_mean_stddev(training=True), 1e-3) + + # Test propagation of seq_length in shape inference. + input_word_ids = tf.keras.layers.Input(shape=(seq_length,), dtype=tf.int32) + input_mask = tf.keras.layers.Input(shape=(seq_length,), dtype=tf.int32) + input_type_ids = tf.keras.layers.Input(shape=(seq_length,), dtype=tf.int32) + pooled_output, sequence_output = hub_layer( + [input_word_ids, input_mask, input_type_ids]) + self.assertEqual(pooled_output.shape.as_list(), [None, hidden_size]) + self.assertEqual(sequence_output.shape.as_list(), + [None, seq_length, hidden_size]) + + +if __name__ == "__main__": + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/input_pipeline.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/input_pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..0c0f7615c37142ca039ad9fc68d98776a6b6b7b8 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/input_pipeline.py @@ -0,0 +1,302 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""BERT model input pipelines.""" + +import tensorflow as tf + + +def decode_record(record, name_to_features): + """Decodes a record to a TensorFlow example.""" + example = tf.io.parse_single_example(record, name_to_features) + + # tf.Example only supports tf.int64, but the TPU only supports tf.int32. + # So cast all int64 to int32. + for name in list(example.keys()): + t = example[name] + if t.dtype == tf.int64: + t = tf.cast(t, tf.int32) + example[name] = t + + return example + + +def single_file_dataset(input_file, name_to_features, num_samples=None): + """Creates a single-file dataset to be passed for BERT custom training.""" + # For training, we want a lot of parallel reading and shuffling. + # For eval, we want no shuffling and parallel reading doesn't matter. + d = tf.data.TFRecordDataset(input_file) + if num_samples: + d = d.take(num_samples) + d = d.map( + lambda record: decode_record(record, name_to_features), + num_parallel_calls=tf.data.experimental.AUTOTUNE) + + # When `input_file` is a path to a single file or a list + # containing a single path, disable auto sharding so that + # same input file is sent to all workers. + if isinstance(input_file, str) or len(input_file) == 1: + options = tf.data.Options() + options.experimental_distribute.auto_shard_policy = ( + tf.data.experimental.AutoShardPolicy.OFF) + d = d.with_options(options) + return d + + +def create_pretrain_dataset(input_patterns, + seq_length, + max_predictions_per_seq, + batch_size, + is_training=True, + input_pipeline_context=None, + use_next_sentence_label=True, + use_position_id=False, + output_fake_labels=True): + """Creates input dataset from (tf)records files for pretraining.""" + name_to_features = { + 'input_ids': + tf.io.FixedLenFeature([seq_length], tf.int64), + 'input_mask': + tf.io.FixedLenFeature([seq_length], tf.int64), + 'segment_ids': + tf.io.FixedLenFeature([seq_length], tf.int64), + 'masked_lm_positions': + tf.io.FixedLenFeature([max_predictions_per_seq], tf.int64), + 'masked_lm_ids': + tf.io.FixedLenFeature([max_predictions_per_seq], tf.int64), + 'masked_lm_weights': + tf.io.FixedLenFeature([max_predictions_per_seq], tf.float32), + } + if use_next_sentence_label: + name_to_features['next_sentence_labels'] = tf.io.FixedLenFeature([1], + tf.int64) + if use_position_id: + name_to_features['position_ids'] = tf.io.FixedLenFeature([seq_length], + tf.int64) + for input_pattern in input_patterns: + if not tf.io.gfile.glob(input_pattern): + raise ValueError('%s does not match any files.' % input_pattern) + + dataset = tf.data.Dataset.list_files(input_patterns, shuffle=is_training) + + if input_pipeline_context and input_pipeline_context.num_input_pipelines > 1: + dataset = dataset.shard(input_pipeline_context.num_input_pipelines, + input_pipeline_context.input_pipeline_id) + if is_training: + dataset = dataset.repeat() + + # We set shuffle buffer to exactly match total number of + # training files to ensure that training data is well shuffled. + input_files = [] + for input_pattern in input_patterns: + input_files.extend(tf.io.gfile.glob(input_pattern)) + dataset = dataset.shuffle(len(input_files)) + + # In parallel, create tf record dataset for each train files. + # cycle_length = 8 means that up to 8 files will be read and deserialized in + # parallel. You may want to increase this number if you have a large number of + # CPU cores. + dataset = dataset.interleave( + tf.data.TFRecordDataset, + cycle_length=8, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + + if is_training: + dataset = dataset.shuffle(100) + + decode_fn = lambda record: decode_record(record, name_to_features) + dataset = dataset.map( + decode_fn, num_parallel_calls=tf.data.experimental.AUTOTUNE) + + def _select_data_from_record(record): + """Filter out features to use for pretraining.""" + x = { + 'input_word_ids': record['input_ids'], + 'input_mask': record['input_mask'], + 'input_type_ids': record['segment_ids'], + 'masked_lm_positions': record['masked_lm_positions'], + 'masked_lm_ids': record['masked_lm_ids'], + 'masked_lm_weights': record['masked_lm_weights'], + } + if use_next_sentence_label: + x['next_sentence_labels'] = record['next_sentence_labels'] + if use_position_id: + x['position_ids'] = record['position_ids'] + + # TODO(hongkuny): Remove the fake labels after migrating bert pretraining. + if output_fake_labels: + return (x, record['masked_lm_weights']) + else: + return x + + dataset = dataset.map( + _select_data_from_record, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + dataset = dataset.batch(batch_size, drop_remainder=is_training) + dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) + return dataset + + +def create_classifier_dataset(file_path, + seq_length, + batch_size, + is_training=True, + input_pipeline_context=None, + label_type=tf.int64, + include_sample_weights=False, + num_samples=None): + """Creates input dataset from (tf)records files for train/eval.""" + name_to_features = { + 'input_ids': tf.io.FixedLenFeature([seq_length], tf.int64), + 'input_mask': tf.io.FixedLenFeature([seq_length], tf.int64), + 'segment_ids': tf.io.FixedLenFeature([seq_length], tf.int64), + 'label_ids': tf.io.FixedLenFeature([], label_type), + } + if include_sample_weights: + name_to_features['weight'] = tf.io.FixedLenFeature([], tf.float32) + dataset = single_file_dataset(file_path, name_to_features, + num_samples=num_samples) + + # The dataset is always sharded by number of hosts. + # num_input_pipelines is the number of hosts rather than number of cores. + if input_pipeline_context and input_pipeline_context.num_input_pipelines > 1: + dataset = dataset.shard(input_pipeline_context.num_input_pipelines, + input_pipeline_context.input_pipeline_id) + + def _select_data_from_record(record): + x = { + 'input_word_ids': record['input_ids'], + 'input_mask': record['input_mask'], + 'input_type_ids': record['segment_ids'] + } + y = record['label_ids'] + if include_sample_weights: + w = record['weight'] + return (x, y, w) + return (x, y) + + if is_training: + dataset = dataset.shuffle(100) + dataset = dataset.repeat() + + dataset = dataset.map( + _select_data_from_record, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + dataset = dataset.batch(batch_size, drop_remainder=is_training) + dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) + return dataset + + +def create_squad_dataset(file_path, + seq_length, + batch_size, + is_training=True, + input_pipeline_context=None): + """Creates input dataset from (tf)records files for train/eval.""" + name_to_features = { + 'input_ids': tf.io.FixedLenFeature([seq_length], tf.int64), + 'input_mask': tf.io.FixedLenFeature([seq_length], tf.int64), + 'segment_ids': tf.io.FixedLenFeature([seq_length], tf.int64), + } + if is_training: + name_to_features['start_positions'] = tf.io.FixedLenFeature([], tf.int64) + name_to_features['end_positions'] = tf.io.FixedLenFeature([], tf.int64) + else: + name_to_features['unique_ids'] = tf.io.FixedLenFeature([], tf.int64) + + dataset = single_file_dataset(file_path, name_to_features) + + # The dataset is always sharded by number of hosts. + # num_input_pipelines is the number of hosts rather than number of cores. + if input_pipeline_context and input_pipeline_context.num_input_pipelines > 1: + dataset = dataset.shard(input_pipeline_context.num_input_pipelines, + input_pipeline_context.input_pipeline_id) + + def _select_data_from_record(record): + """Dispatches record to features and labels.""" + x, y = {}, {} + for name, tensor in record.items(): + if name in ('start_positions', 'end_positions'): + y[name] = tensor + elif name == 'input_ids': + x['input_word_ids'] = tensor + elif name == 'segment_ids': + x['input_type_ids'] = tensor + else: + x[name] = tensor + return (x, y) + + if is_training: + dataset = dataset.shuffle(100) + dataset = dataset.repeat() + + dataset = dataset.map( + _select_data_from_record, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + dataset = dataset.batch(batch_size, drop_remainder=True) + dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) + return dataset + + +def create_retrieval_dataset(file_path, + seq_length, + batch_size, + input_pipeline_context=None): + """Creates input dataset from (tf)records files for scoring.""" + name_to_features = { + 'input_ids': tf.io.FixedLenFeature([seq_length], tf.int64), + 'input_mask': tf.io.FixedLenFeature([seq_length], tf.int64), + 'segment_ids': tf.io.FixedLenFeature([seq_length], tf.int64), + 'example_id': tf.io.FixedLenFeature([1], tf.int64), + } + dataset = single_file_dataset(file_path, name_to_features) + + # The dataset is always sharded by number of hosts. + # num_input_pipelines is the number of hosts rather than number of cores. + if input_pipeline_context and input_pipeline_context.num_input_pipelines > 1: + dataset = dataset.shard(input_pipeline_context.num_input_pipelines, + input_pipeline_context.input_pipeline_id) + + def _select_data_from_record(record): + x = { + 'input_word_ids': record['input_ids'], + 'input_mask': record['input_mask'], + 'input_type_ids': record['segment_ids'] + } + y = record['example_id'] + return (x, y) + + dataset = dataset.map( + _select_data_from_record, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + dataset = dataset.batch(batch_size, drop_remainder=False) + + def _pad_to_batch(x, y): + cur_size = tf.shape(y)[0] + pad_size = batch_size - cur_size + + pad_ids = tf.zeros(shape=[pad_size, seq_length], dtype=tf.int32) + for key in ('input_word_ids', 'input_mask', 'input_type_ids'): + x[key] = tf.concat([x[key], pad_ids], axis=0) + + pad_labels = -tf.ones(shape=[pad_size, 1], dtype=tf.int32) + y = tf.concat([y, pad_labels], axis=0) + return x, y + + dataset = dataset.map( + _pad_to_batch, + num_parallel_calls=tf.data.experimental.AUTOTUNE) + + dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE) + return dataset diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/model_saving_utils.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/model_saving_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1d69750878bd8a89482958874b5f059193f6d7f5 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/model_saving_utils.py @@ -0,0 +1,68 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities to save models.""" + +import os + +from absl import logging +import tensorflow as tf +import typing + + +def export_bert_model(model_export_path: typing.Text, + model: tf.keras.Model, + checkpoint_dir: typing.Optional[typing.Text] = None, + restore_model_using_load_weights: bool = False) -> None: + """Export BERT model for serving which does not include the optimizer. + + Args: + model_export_path: Path to which exported model will be saved. + model: Keras model object to export. + checkpoint_dir: Path from which model weights will be loaded, if + specified. + restore_model_using_load_weights: Whether to use checkpoint.restore() API + for custom checkpoint or to use model.load_weights() API. There are 2 + different ways to save checkpoints. One is using tf.train.Checkpoint and + another is using Keras model.save_weights(). Custom training loop + implementation uses tf.train.Checkpoint API and Keras ModelCheckpoint + callback internally uses model.save_weights() API. Since these two API's + cannot be used toghether, model loading logic must be take into account + how model checkpoint was saved. + + Raises: + ValueError when either model_export_path or model is not specified. + """ + if not model_export_path: + raise ValueError('model_export_path must be specified.') + if not isinstance(model, tf.keras.Model): + raise ValueError('model must be a tf.keras.Model object.') + + if checkpoint_dir: + if restore_model_using_load_weights: + model_weight_path = os.path.join(checkpoint_dir, 'checkpoint') + assert tf.io.gfile.exists(model_weight_path) + model.load_weights(model_weight_path) + else: + checkpoint = tf.train.Checkpoint(model=model) + + # Restores the model from latest checkpoint. + latest_checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir) + assert latest_checkpoint_file + logging.info('Checkpoint file %s found and restoring from ' + 'checkpoint', latest_checkpoint_file) + checkpoint.restore( + latest_checkpoint_file).assert_existing_objects_matched() + + model.save(model_export_path, include_optimizer=False, save_format='tf') diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/model_training_utils.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/model_training_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8cc11993bab397442c927ddcd399c2c620093205 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/model_training_utils.py @@ -0,0 +1,590 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A light weight utilities to train NLP models.""" + +import json +import os +import tempfile + +from absl import logging +import tensorflow as tf +from tensorflow.python.util import deprecation +from official.common import distribute_utils +from official.modeling import grad_utils + +_SUMMARY_TXT = 'training_summary.txt' +_MIN_SUMMARY_STEPS = 10 + + +def _should_export_checkpoint(strategy): + return (not strategy) or strategy.extended.should_checkpoint + + +def _should_export_summary(strategy): + return (not strategy) or strategy.extended.should_save_summary + + +def _save_checkpoint(strategy, checkpoint, model_dir, checkpoint_prefix): + """Saves model to with provided checkpoint prefix.""" + + if _should_export_checkpoint(strategy): + checkpoint_path = os.path.join(model_dir, checkpoint_prefix) + saved_path = checkpoint.save(checkpoint_path) + logging.info('Saving model as TF checkpoint: %s', saved_path) + else: + # In multi worker training we need every worker to save checkpoint, because + # variables can trigger synchronization on read and synchronization needs + # all workers to participate. To avoid workers overriding each other we save + # to a temporary directory on non-chief workers. + tmp_dir = tempfile.mkdtemp() + checkpoint.save(os.path.join(tmp_dir, 'ckpt')) + tf.io.gfile.rmtree(tmp_dir) + return + + +def _get_input_iterator(input_fn, strategy): + """Returns distributed dataset iterator.""" + # When training with TPU pods, datasets needs to be cloned across + # workers. Since Dataset instance cannot be cloned in eager mode, we instead + # pass callable that returns a dataset. + if not callable(input_fn): + raise ValueError('`input_fn` should be a closure that returns a dataset.') + iterator = iter(strategy.distribute_datasets_from_function(input_fn)) + return iterator + + +def _float_metric_value(metric): + """Gets the value of a float-value keras metric.""" + return metric.result().numpy().astype(float) + + +def clip_by_global_norm_callback(grads_and_vars): + """Performs gradient clipping.""" + grads, variables = zip(*grads_and_vars) + (clipped_grads, _) = tf.clip_by_global_norm(grads, clip_norm=1.0) + return zip(clipped_grads, variables) + + +def steps_to_run(current_step, steps_per_epoch, steps_per_loop): + """Calculates steps to run on device.""" + if steps_per_loop <= 0: + raise ValueError('steps_per_loop should be positive integer.') + if steps_per_loop == 1: + return steps_per_loop + remainder_in_epoch = current_step % steps_per_epoch + if remainder_in_epoch != 0: + return min(steps_per_epoch - remainder_in_epoch, steps_per_loop) + else: + return steps_per_loop + + +def write_txt_summary(training_summary, summary_dir): + """Writes a summary text file to record stats.""" + if not tf.io.gfile.exists(summary_dir): + tf.io.gfile.mkdir(summary_dir) + summary_path = os.path.join(summary_dir, _SUMMARY_TXT) + with tf.io.gfile.GFile(summary_path, 'wb') as f: + logging.info('Training Summary: \n%s', str(training_summary)) + f.write(json.dumps(training_summary, indent=4)) + + +@deprecation.deprecated( + None, 'This function is deprecated and we do not expect adding new ' + 'functionalities. Please do not have your code depending ' + 'on this library.') +def run_customized_training_loop( + # pylint: disable=invalid-name + _sentinel=None, + # pylint: enable=invalid-name + strategy=None, + model_fn=None, + loss_fn=None, + scale_loss=True, + model_dir=None, + train_input_fn=None, + steps_per_epoch=None, + num_eval_per_epoch=1, + steps_per_loop=None, + epochs=1, + eval_input_fn=None, + eval_steps=None, + metric_fn=None, + init_checkpoint=None, + custom_callbacks=None, + run_eagerly=False, + sub_model_export_name=None, + explicit_allreduce=False, + pre_allreduce_callbacks=None, + post_allreduce_callbacks=None, + train_summary_interval=0, + allreduce_bytes_per_pack=0): + """Run BERT pretrain model training using low-level API. + + Args: + _sentinel: Used to prevent positional parameters. Internal, do not use. + strategy: Distribution strategy on which to run low level training loop. + model_fn: Function that returns a tuple (model, sub_model). Caller of this + function should add optimizer to the `model` via calling + `model.compile()` API or manually setting `model.optimizer` attribute. + Second element of the returned tuple(sub_model) is an optional sub model + to be used for initial checkpoint -- if provided. + loss_fn: Function with signature func(labels, logits) and returns a loss + tensor. + scale_loss: Whether to divide the raw loss by number of replicas before + gradients calculation. + model_dir: Model directory used during training for restoring/saving model + weights. + train_input_fn: Function that returns a tf.data.Dataset used for training. + steps_per_epoch: Number of steps to run per epoch. At the end of each + epoch, model checkpoint will be saved and evaluation will be conducted + if evaluation dataset is provided. + num_eval_per_epoch: Number of evaluations per epoch. + steps_per_loop: Number of steps per graph-mode loop. In order to reduce + communication in eager context, training logs are printed every + steps_per_loop. + epochs: Number of epochs to train. + eval_input_fn: Function that returns evaluation dataset. If none, + evaluation is skipped. + eval_steps: Number of steps to run evaluation. Required if `eval_input_fn` + is not none. + metric_fn: A metrics function that returns either a Keras Metric object or + a list of Keras Metric objects to record evaluation result using + evaluation dataset or with training dataset after every epoch. + init_checkpoint: Optional checkpoint to load to `sub_model` returned by + `model_fn`. + custom_callbacks: A list of Keras Callbacks objects to run during + training. More specifically, `on_train_begin(), on_train_end(), + on_batch_begin()`, `on_batch_end()`, `on_epoch_begin()`, + `on_epoch_end()` methods are invoked during training. Note that some + metrics may be missing from `logs`. + run_eagerly: Whether to run model training in pure eager execution. This + should be disable for TPUStrategy. + sub_model_export_name: If not None, will export `sub_model` returned by + `model_fn` into checkpoint files. The name of intermediate checkpoint + file is {sub_model_export_name}_step_{step}.ckpt and the last + checkpint's name is {sub_model_export_name}.ckpt; if None, `sub_model` + will not be exported as checkpoint. + explicit_allreduce: Whether to explicitly perform gradient allreduce, + instead of relying on implicit allreduce in optimizer.apply_gradients(). + default is False. For now, if training using FP16 mixed precision, + explicit allreduce will aggregate gradients in FP16 format. For TPU and + GPU training using FP32, explicit allreduce will aggregate gradients in + FP32 format. + pre_allreduce_callbacks: A list of callback functions that takes gradients + and model variables pairs as input, manipulate them, and returns a new + gradients and model variables paris. The callback functions will be + invoked in the list order and before gradients are allreduced. With + mixed precision training, the pre_allreduce_allbacks will be applied on + scaled_gradients. Default is no callbacks. Only used when + explicit_allreduce=True. + post_allreduce_callbacks: A list of callback functions that takes + gradients and model variables pairs as input, manipulate them, and + returns a new gradients and model variables paris. The callback + functions will be invoked in the list order and right before gradients + are applied to variables for updates. Default is no callbacks. Only used + when explicit_allreduce=True. + train_summary_interval: Step interval for training summaries. If the value + is a negative number, then training summaries are not enabled. + allreduce_bytes_per_pack: A non-negative integer. Breaks collective + operations into packs of certain size. If it's zero, all gradients are + in one pack. Breaking gradient into packs could enable overlap between + allreduce and backprop computation. This flag only takes effect when + explicit_allreduce is set to True.' + + Returns: + Trained model. + + Raises: + ValueError: (1) When model returned by `model_fn` does not have optimizer + attribute or when required parameters are set to none. (2) eval args are + not specified correctly. (3) metric_fn must be a callable if specified. + (4) sub_model_checkpoint_name is specified, but `sub_model` returned + by `model_fn` is None. + """ + + if _sentinel is not None: + raise ValueError('only call `run_customized_training_loop()` ' + 'with named arguments.') + + required_arguments = [ + strategy, model_fn, loss_fn, model_dir, steps_per_epoch, train_input_fn + ] + + steps_between_evals = int(steps_per_epoch / num_eval_per_epoch) + if [arg for arg in required_arguments if arg is None]: + raise ValueError('`strategy`, `model_fn`, `loss_fn`, `model_dir`, ' + '`steps_per_epoch` and `train_input_fn` are required ' + 'parameters.') + if not steps_per_loop: + if tf.config.list_logical_devices('TPU'): + # One can't fully utilize a TPU with steps_per_loop=1, so in this case + # default users to a more useful value. + steps_per_loop = min(1000, steps_between_evals) + else: + steps_per_loop = 1 + logging.info('steps_per_loop not specified. Using steps_per_loop=%d', + steps_per_loop) + if steps_per_loop > steps_between_evals: + logging.warning( + 'steps_per_loop: %d is specified to be greater than ' + ' steps_between_evals: %d, we will use steps_between_evals as' + ' steps_per_loop.', steps_per_loop, steps_between_evals) + steps_per_loop = steps_between_evals + assert tf.executing_eagerly() + + if run_eagerly: + if isinstance( + strategy, + (tf.distribute.TPUStrategy, tf.distribute.experimental.TPUStrategy)): + raise ValueError( + 'TPUStrategy should not run eagerly as it heavily relies on graph' + ' optimization for the distributed system.') + + if eval_input_fn and eval_steps is None: + raise ValueError( + '`eval_step` is required when `eval_input_fn ` is not none.') + if metric_fn and not callable(metric_fn): + raise ValueError( + 'if `metric_fn` is specified, metric_fn must be a callable.') + + total_training_steps = steps_per_epoch * epochs + train_iterator = _get_input_iterator(train_input_fn, strategy) + eval_loss_metric = tf.keras.metrics.Mean('training_loss', dtype=tf.float32) + + with distribute_utils.get_strategy_scope(strategy): + # To correctly place the model weights on accelerators, + # model and optimizer should be created in scope. + model, sub_model = model_fn() + if not hasattr(model, 'optimizer'): + raise ValueError('User should set optimizer attribute to model ' + 'inside `model_fn`.') + if sub_model_export_name and sub_model is None: + raise ValueError('sub_model_export_name is specified as %s, but ' + 'sub_model is None.' % sub_model_export_name) + + callback_list = tf.keras.callbacks.CallbackList( + callbacks=custom_callbacks, model=model) + + optimizer = model.optimizer + + if init_checkpoint: + logging.info( + 'Checkpoint file %s found and restoring from ' + 'initial checkpoint for core model.', init_checkpoint) + checkpoint = tf.train.Checkpoint(model=sub_model, encoder=sub_model) + checkpoint.read(init_checkpoint).assert_existing_objects_matched() + logging.info('Loading from checkpoint file completed') + + train_loss_metric = tf.keras.metrics.Mean('training_loss', dtype=tf.float32) + eval_metrics = metric_fn() if metric_fn else [] + if not isinstance(eval_metrics, list): + eval_metrics = [eval_metrics] + # If evaluation is required, make a copy of metric as it will be used by + # both train and evaluation. + train_metrics = [ + metric.__class__.from_config(metric.get_config()) + for metric in eval_metrics + ] + + # Create summary writers + if _should_export_summary(strategy): + summary_dir = os.path.join(model_dir, 'summaries') + else: + # In multi worker training we need every worker to write summary, because + # variables can trigger synchronization on read and synchronization needs + # all workers to participate. + summary_dir = tempfile.mkdtemp() + eval_summary_writer = tf.summary.create_file_writer( + os.path.join(summary_dir, 'eval')) + last_summary_step = 0 + if steps_per_loop >= _MIN_SUMMARY_STEPS and train_summary_interval >= 0: + # Only writes summary when the stats are collected sufficiently over + # enough steps. + train_summary_writer = tf.summary.create_file_writer( + os.path.join(summary_dir, 'train')) + else: + train_summary_writer = tf.summary.create_noop_writer() + + # Collects training variables. + training_vars = model.trainable_variables + + def _replicated_step(inputs): + """Replicated training step.""" + + inputs, labels = inputs + with tf.GradientTape() as tape: + model_outputs = model(inputs, training=True) + loss = loss_fn(labels, model_outputs) + # Raw loss is used for reporting in metrics/logs. + raw_loss = loss + if scale_loss: + # Scales down the loss for gradients to be invariant from replicas. + loss = loss / strategy.num_replicas_in_sync + + if explicit_allreduce: + grad_utils.minimize_using_explicit_allreduce(tape, optimizer, loss, + training_vars, + pre_allreduce_callbacks, + post_allreduce_callbacks, + allreduce_bytes_per_pack) + else: + if isinstance(optimizer, tf.keras.mixed_precision.LossScaleOptimizer): + with tape: + scaled_loss = optimizer.get_scaled_loss(loss) + scaled_grads = tape.gradient(scaled_loss, training_vars) + grads = optimizer.get_unscaled_gradients(scaled_grads) + else: + grads = tape.gradient(loss, training_vars) + optimizer.apply_gradients(zip(grads, training_vars)) + # For reporting, the metric takes the mean of losses. + train_loss_metric.update_state(raw_loss) + for metric in train_metrics: + metric.update_state(labels, model_outputs) + + @tf.function + def train_steps(iterator, steps): + """Performs distributed training steps in a loop. + + Args: + iterator: the distributed iterator of training datasets. + steps: an tf.int32 integer tensor to specify number of steps to run + inside host training loop. + + Raises: + ValueError: Any of the arguments or tensor shapes are invalid. + """ + if not isinstance(steps, tf.Tensor): + raise ValueError('steps should be an Tensor. Python object may cause ' + 'retracing.') + + for _ in tf.range(steps): + strategy.run(_replicated_step, args=(next(iterator),)) + + def train_single_step(iterator): + """Performs a distributed training step. + + Args: + iterator: the distributed iterator of training datasets. + + Raises: + ValueError: Any of the arguments or tensor shapes are invalid. + """ + strategy.run(_replicated_step, args=(next(iterator),)) + + def test_step(iterator): + """Calculates evaluation metrics on distributed devices.""" + + def _test_step_fn(inputs): + """Replicated accuracy calculation.""" + + inputs, labels = inputs + model_outputs = model(inputs, training=False) + for metric in eval_metrics: + metric.update_state(labels, model_outputs) + return model_outputs, labels + + outputs, labels = strategy.run(_test_step_fn, args=(next(iterator),)) + outputs = tf.nest.map_structure(strategy.experimental_local_results, + outputs) + labels = tf.nest.map_structure(strategy.experimental_local_results, + labels) + return outputs, labels + + if not run_eagerly: + train_single_step = tf.function(train_single_step) + test_step = tf.function(test_step) + + def _run_evaluation(current_training_step, test_iterator): + """Runs validation steps and aggregate metrics. + + Args: + current_training_step: tf.int32 tensor containing the current step. + test_iterator: distributed iterator of test datasets. + + Returns: + A dict of metic names and values. + """ + # The last batch of the evaluation is often smaller than previous ones. + # Moreover, in some distributed pieces it might even be empty. Therefore, + # different from the way training_loss is calculated, it is needed to + # gather all the logits and labels here to calculate the evaluation loss + # outside. + loss_list, loss_weights = list(), list() + for _ in range(eval_steps): + outputs, labels = test_step(test_iterator) + for cur_logits, cur_labels in zip(outputs, labels): + # This is to handle cases when cur_labels is not a single tensor, + # but a dict of tensors. + cur_weight = tf.shape(tf.nest.flatten(cur_labels)[0])[0] + if cur_weight != 0: + loss_list.append(loss_fn(cur_labels, cur_logits).numpy()) + loss_weights.append(cur_weight) + # The sample_weights are the actual number of examples in each batch, + # a summation of numbers of examples in each replica if using + # distributed training. + eval_loss_metric.update_state(loss_list, sample_weight=loss_weights) + + logs = {} + with eval_summary_writer.as_default(): + for metric in [eval_loss_metric] + eval_metrics + model.metrics: + metric_value = _float_metric_value(metric) + logs[metric.name] = metric_value + logging.info('Step: [%d] Validation %s = %f', current_training_step, + metric.name, metric_value) + tf.summary.scalar( + metric.name, metric_value, step=current_training_step) + eval_summary_writer.flush() + + return logs + + # Training loop starts here. + checkpoint = tf.train.Checkpoint( + model=model, optimizer=optimizer, global_step=optimizer.iterations) + sub_model_checkpoint = tf.train.Checkpoint( + model=sub_model, + global_step=optimizer.iterations) if sub_model_export_name else None + + latest_checkpoint_file = tf.train.latest_checkpoint(model_dir) + if latest_checkpoint_file: + logging.info('Checkpoint file %s found and restoring from ' + 'checkpoint', latest_checkpoint_file) + checkpoint.restore(latest_checkpoint_file) + logging.info('Loading from checkpoint file completed') + + current_step = optimizer.iterations.numpy() + checkpoint_name = 'ctl_step_{step}.ckpt' + + logs = {} + callback_list.on_train_begin() + while current_step < total_training_steps and not model.stop_training: + if current_step % steps_per_epoch == 0: + callback_list.on_epoch_begin(int(current_step / steps_per_epoch) + 1) + + # Training loss/metric are taking average over steps inside micro + # training loop. We reset the their values before each round. + train_loss_metric.reset_states() + for metric in train_metrics + model.metrics: + metric.reset_states() + + callback_list.on_batch_begin(current_step) + # Runs several steps in the host while loop. + steps = steps_to_run(current_step, steps_between_evals, steps_per_loop) + + if tf.config.list_physical_devices('GPU'): + # TODO(zongweiz): merge with train_steps once tf.while_loop + # GPU performance bugs are fixed. + for _ in range(steps): + train_single_step(train_iterator) + else: + # Converts steps to a Tensor to avoid tf.function retracing. + train_steps(train_iterator, tf.convert_to_tensor(steps, dtype=tf.int32)) + train_loss = _float_metric_value(train_loss_metric) + current_step += steps + + # Updates training logging. + training_status = 'Train Step: %d/%d / loss = %s' % ( + current_step, total_training_steps, train_loss) + + if current_step >= last_summary_step + train_summary_interval: + summary_writer = train_summary_writer + last_summary_step = current_step + else: + summary_writer = tf.summary.create_noop_writer() + + with summary_writer.as_default(): + if callable(optimizer.learning_rate): + tf.summary.scalar( + 'learning_rate', + optimizer.learning_rate(current_step), + step=current_step) + tf.summary.scalar(train_loss_metric.name, train_loss, step=current_step) + for metric in train_metrics + model.metrics: + metric_value = _float_metric_value(metric) + training_status += ' %s = %f' % (metric.name, metric_value) + tf.summary.scalar(metric.name, metric_value, step=current_step) + summary_writer.flush() + logging.info(training_status) + + # If no need for evaluation, we only call on_batch_end with train_loss, + # this is to ensure we get granular global_step/sec on Tensorboard. + if current_step % steps_between_evals: + callback_list.on_batch_end(current_step - 1, {'loss': train_loss}) + else: + # Save a submodel with the step in the file name after each epoch. + if sub_model_export_name: + _save_checkpoint( + strategy, sub_model_checkpoint, model_dir, + '%s_step_%d.ckpt' % (sub_model_export_name, current_step)) + + # Save model checkpoints and run validation steps after each epoch + # (with the exception of the final epoch which is handled after the + # training loop). + if current_step < total_training_steps: + _save_checkpoint(strategy, checkpoint, model_dir, + checkpoint_name.format(step=current_step)) + if eval_input_fn: + # Re-initialize evaluation metric. + eval_loss_metric.reset_states() + for metric in eval_metrics + model.metrics: + metric.reset_states() + + logging.info('Running evaluation after step: %s.', current_step) + logs = _run_evaluation(current_step, + _get_input_iterator(eval_input_fn, strategy)) + # We add train_loss here rather than call on_batch_end twice to make + # sure that no duplicated values are generated. + logs['loss'] = train_loss + callback_list.on_batch_end(current_step - 1, logs) + + # Calls on_epoch_end after each real epoch ends to prevent mis-calculation + # of training steps. + if current_step % steps_per_epoch == 0: + callback_list.on_epoch_end(int(current_step / steps_per_epoch), logs) + + if sub_model_export_name: + _save_checkpoint(strategy, sub_model_checkpoint, model_dir, + '%s.ckpt' % sub_model_export_name) + + _save_checkpoint(strategy, checkpoint, model_dir, + checkpoint_name.format(step=current_step)) + if eval_input_fn: + # Re-initialize evaluation metric. + eval_loss_metric.reset_states() + for metric in eval_metrics + model.metrics: + metric.reset_states() + + logging.info('Running final evaluation after training is complete.') + logs = _run_evaluation(current_step, + _get_input_iterator(eval_input_fn, strategy)) + callback_list.on_epoch_end(int(current_step / steps_per_epoch), logs) + training_summary = { + 'total_training_steps': total_training_steps, + 'train_loss': _float_metric_value(train_loss_metric), + } + for metric in model.metrics: + training_summary[metric.name] = _float_metric_value(metric) + if eval_metrics: + training_summary['last_train_metrics'] = _float_metric_value( + train_metrics[0]) + training_summary['eval_metrics'] = _float_metric_value(eval_metrics[0]) + + write_txt_summary(training_summary, summary_dir) + + if not _should_export_summary(strategy): + tf.io.gfile.rmtree(summary_dir) + + callback_list.on_train_end() + + return model diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/model_training_utils_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/model_training_utils_test.py new file mode 100644 index 0000000000000000000000000000000000000000..544b66834002d09dfabd90169e6f53fa9f2bbaf3 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/model_training_utils_test.py @@ -0,0 +1,306 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for official.modeling.training.model_training_utils.""" + +import os + +from absl import logging +from absl.testing import flagsaver +from absl.testing import parameterized +from absl.testing.absltest import mock +import numpy as np +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.nlp.bert import common_flags +from official.nlp.bert import model_training_utils + + +common_flags.define_common_bert_flags() + + +def eager_strategy_combinations(): + return combinations.combine( + distribution=[ + strategy_combinations.default_strategy, + strategy_combinations.cloud_tpu_strategy, + strategy_combinations.one_device_strategy_gpu, + strategy_combinations.mirrored_strategy_with_gpu_and_cpu, + strategy_combinations.mirrored_strategy_with_two_gpus, + ],) + + +def eager_gpu_strategy_combinations(): + return combinations.combine( + distribution=[ + strategy_combinations.default_strategy, + strategy_combinations.one_device_strategy_gpu, + strategy_combinations.mirrored_strategy_with_gpu_and_cpu, + strategy_combinations.mirrored_strategy_with_two_gpus, + ],) + + +def create_fake_data_input_fn(batch_size, features_shape, num_classes): + """Creates a dummy input function with the given feature and label shapes. + + Args: + batch_size: integer. + features_shape: list[int]. Feature shape for an individual example. + num_classes: integer. Number of labels. + + Returns: + An input function that is usable in the executor. + """ + + def _dataset_fn(input_context=None): + """An input function for generating fake data.""" + local_batch_size = input_context.get_per_replica_batch_size(batch_size) + features = np.random.rand(64, *features_shape) + labels = np.random.randint(2, size=[64, num_classes]) + # Convert the inputs to a Dataset. + dataset = tf.data.Dataset.from_tensor_slices((features, labels)) + dataset = dataset.shard(input_context.num_input_pipelines, + input_context.input_pipeline_id) + + def _assign_dtype(features, labels): + features = tf.cast(features, tf.float32) + labels = tf.cast(labels, tf.float32) + return features, labels + + # Shuffle, repeat, and batch the examples. + dataset = dataset.map(_assign_dtype) + dataset = dataset.shuffle(64).repeat() + dataset = dataset.batch(local_batch_size, drop_remainder=True) + dataset = dataset.prefetch(buffer_size=64) + return dataset + + return _dataset_fn + + +def create_model_fn(input_shape, num_classes, use_float16=False): + + def _model_fn(): + """A one-layer softmax model suitable for testing.""" + input_layer = tf.keras.layers.Input(shape=input_shape) + x = tf.keras.layers.Dense(num_classes, activation='relu')(input_layer) + output_layer = tf.keras.layers.Dense(num_classes, activation='softmax')(x) + sub_model = tf.keras.models.Model(input_layer, x, name='sub_model') + model = tf.keras.models.Model(input_layer, output_layer, name='model') + model.add_metric( + tf.reduce_mean(input_layer), name='mean_input', aggregation='mean') + model.optimizer = tf.keras.optimizers.SGD(learning_rate=0.1, momentum=0.9) + if use_float16: + model.optimizer = tf.keras.mixed_precision.LossScaleOptimizer( + model.optimizer) + return model, sub_model + + return _model_fn + + +def metric_fn(): + """Gets a tf.keras metric object.""" + return tf.keras.metrics.CategoricalAccuracy(name='accuracy', dtype=tf.float32) + + +def summaries_with_matching_keyword(keyword, summary_dir): + """Yields summary protos matching given keyword from event file.""" + event_paths = tf.io.gfile.glob(os.path.join(summary_dir, 'events*')) + for event in tf.compat.v1.train.summary_iterator(event_paths[-1]): + if event.summary is not None: + for value in event.summary.value: + if keyword in value.tag: + logging.error(event) + yield event.summary + + +def check_eventfile_for_keyword(keyword, summary_dir): + """Checks event files for the keyword.""" + return any(summaries_with_matching_keyword(keyword, summary_dir)) + + +class RecordingCallback(tf.keras.callbacks.Callback): + + def __init__(self): + self.batch_begin = [] # (batch, logs) + self.batch_end = [] # (batch, logs) + self.epoch_begin = [] # (epoch, logs) + self.epoch_end = [] # (epoch, logs) + + def on_batch_begin(self, batch, logs=None): + self.batch_begin.append((batch, logs)) + + def on_batch_end(self, batch, logs=None): + self.batch_end.append((batch, logs)) + + def on_epoch_begin(self, epoch, logs=None): + self.epoch_begin.append((epoch, logs)) + + def on_epoch_end(self, epoch, logs=None): + self.epoch_end.append((epoch, logs)) + + +class ModelTrainingUtilsTest(tf.test.TestCase, parameterized.TestCase): + + def setUp(self): + super(ModelTrainingUtilsTest, self).setUp() + self._model_fn = create_model_fn(input_shape=[128], num_classes=3) + + @flagsaver.flagsaver + def run_training(self, strategy, model_dir, steps_per_loop, run_eagerly): + input_fn = create_fake_data_input_fn( + batch_size=8, features_shape=[128], num_classes=3) + model_training_utils.run_customized_training_loop( + strategy=strategy, + model_fn=self._model_fn, + loss_fn=tf.keras.losses.categorical_crossentropy, + model_dir=model_dir, + steps_per_epoch=20, + steps_per_loop=steps_per_loop, + epochs=2, + train_input_fn=input_fn, + eval_input_fn=input_fn, + eval_steps=10, + init_checkpoint=None, + sub_model_export_name='my_submodel_name', + metric_fn=metric_fn, + custom_callbacks=None, + run_eagerly=run_eagerly) + + @combinations.generate(eager_strategy_combinations()) + def test_train_eager_single_step(self, distribution): + model_dir = self.create_tempdir().full_path + if isinstance( + distribution, + (tf.distribute.TPUStrategy, tf.distribute.experimental.TPUStrategy)): + with self.assertRaises(ValueError): + self.run_training( + distribution, model_dir, steps_per_loop=1, run_eagerly=True) + else: + self.run_training( + distribution, model_dir, steps_per_loop=1, run_eagerly=True) + + @combinations.generate(eager_gpu_strategy_combinations()) + def test_train_eager_mixed_precision(self, distribution): + model_dir = self.create_tempdir().full_path + tf.keras.mixed_precision.set_global_policy('mixed_float16') + self._model_fn = create_model_fn( + input_shape=[128], num_classes=3, use_float16=True) + self.run_training( + distribution, model_dir, steps_per_loop=1, run_eagerly=True) + + @combinations.generate(eager_strategy_combinations()) + def test_train_check_artifacts(self, distribution): + model_dir = self.create_tempdir().full_path + self.run_training( + distribution, model_dir, steps_per_loop=10, run_eagerly=False) + + # Two checkpoints should be saved after two epochs. + files = map(os.path.basename, + tf.io.gfile.glob(os.path.join(model_dir, 'ctl_step_*index'))) + self.assertCountEqual( + ['ctl_step_20.ckpt-1.index', 'ctl_step_40.ckpt-2.index'], files) + + # Three submodel checkpoints should be saved after two epochs (one after + # each epoch plus one final). + files = map( + os.path.basename, + tf.io.gfile.glob(os.path.join(model_dir, 'my_submodel_name*index'))) + self.assertCountEqual([ + 'my_submodel_name.ckpt-3.index', + 'my_submodel_name_step_20.ckpt-1.index', + 'my_submodel_name_step_40.ckpt-2.index' + ], files) + + self.assertNotEmpty( + tf.io.gfile.glob( + os.path.join(model_dir, 'summaries/training_summary*'))) + + # Loss and accuracy values should be written into summaries. + self.assertTrue( + check_eventfile_for_keyword('loss', + os.path.join(model_dir, 'summaries/train'))) + self.assertTrue( + check_eventfile_for_keyword('accuracy', + os.path.join(model_dir, 'summaries/train'))) + self.assertTrue( + check_eventfile_for_keyword('mean_input', + os.path.join(model_dir, 'summaries/train'))) + self.assertTrue( + check_eventfile_for_keyword('accuracy', + os.path.join(model_dir, 'summaries/eval'))) + self.assertTrue( + check_eventfile_for_keyword('mean_input', + os.path.join(model_dir, 'summaries/eval'))) + + @combinations.generate(eager_strategy_combinations()) + def test_train_check_callbacks(self, distribution): + model_dir = self.create_tempdir().full_path + callback = RecordingCallback() + callbacks = [callback] + input_fn = create_fake_data_input_fn( + batch_size=8, features_shape=[128], num_classes=3) + model_training_utils.run_customized_training_loop( + strategy=distribution, + model_fn=self._model_fn, + loss_fn=tf.keras.losses.categorical_crossentropy, + model_dir=model_dir, + steps_per_epoch=20, + num_eval_per_epoch=4, + steps_per_loop=10, + epochs=2, + train_input_fn=input_fn, + eval_input_fn=input_fn, + eval_steps=10, + init_checkpoint=None, + metric_fn=metric_fn, + custom_callbacks=callbacks, + run_eagerly=False) + self.assertEqual(callback.epoch_begin, [(1, {}), (2, {})]) + epoch_ends, epoch_end_infos = zip(*callback.epoch_end) + self.assertEqual(list(epoch_ends), [1, 2, 2]) + for info in epoch_end_infos: + self.assertIn('accuracy', info) + + self.assertEqual(callback.batch_begin, [(0, {}), (5, {}), (10, {}), + (15, {}), (20, {}), (25, {}), + (30, {}), (35, {})]) + batch_ends, batch_end_infos = zip(*callback.batch_end) + self.assertEqual(list(batch_ends), [4, 9, 14, 19, 24, 29, 34, 39]) + for info in batch_end_infos: + self.assertIn('loss', info) + + @combinations.generate( + combinations.combine( + distribution=[ + strategy_combinations.one_device_strategy_gpu, + ],)) + def test_train_check_artifacts_non_chief(self, distribution): + # We shouldn't export artifacts on non-chief workers. Since there's no easy + # way to test with real MultiWorkerMirroredStrategy, we patch the strategy + # to make it as if it's MultiWorkerMirroredStrategy on non-chief workers. + extended = distribution.extended + with mock.patch.object(extended.__class__, 'should_checkpoint', + new_callable=mock.PropertyMock, return_value=False), \ + mock.patch.object(extended.__class__, 'should_save_summary', + new_callable=mock.PropertyMock, return_value=False): + model_dir = self.create_tempdir().full_path + self.run_training( + distribution, model_dir, steps_per_loop=10, run_eagerly=False) + self.assertEmpty(tf.io.gfile.listdir(model_dir)) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/run_classifier.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/run_classifier.py new file mode 100644 index 0000000000000000000000000000000000000000..1979af5fb3706350953c3cf7542c026719d5d908 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/run_classifier.py @@ -0,0 +1,516 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""BERT classification or regression finetuning runner in TF 2.x.""" + +import functools +import json +import math +import os + +# Import libraries +from absl import app +from absl import flags +from absl import logging +import gin +import tensorflow as tf +from official.common import distribute_utils +from official.modeling import performance +from official.nlp import optimization +from official.nlp.bert import bert_models +from official.nlp.bert import common_flags +from official.nlp.bert import configs as bert_configs +from official.nlp.bert import input_pipeline +from official.nlp.bert import model_saving_utils +from official.utils.misc import keras_utils + +flags.DEFINE_enum( + 'mode', 'train_and_eval', ['train_and_eval', 'export_only', 'predict'], + 'One of {"train_and_eval", "export_only", "predict"}. `train_and_eval`: ' + 'trains the model and evaluates in the meantime. ' + '`export_only`: will take the latest checkpoint inside ' + 'model_dir and export a `SavedModel`. `predict`: takes a checkpoint and ' + 'restores the model to output predictions on the test set.') +flags.DEFINE_string('train_data_path', None, + 'Path to training data for BERT classifier.') +flags.DEFINE_string('eval_data_path', None, + 'Path to evaluation data for BERT classifier.') +flags.DEFINE_string( + 'input_meta_data_path', None, + 'Path to file that contains meta data about input ' + 'to be used for training and evaluation.') +flags.DEFINE_integer('train_data_size', None, 'Number of training samples ' + 'to use. If None, uses the full train data. ' + '(default: None).') +flags.DEFINE_string('predict_checkpoint_path', None, + 'Path to the checkpoint for predictions.') +flags.DEFINE_integer( + 'num_eval_per_epoch', 1, + 'Number of evaluations per epoch. The purpose of this flag is to provide ' + 'more granular evaluation scores and checkpoints. For example, if original ' + 'data has N samples and num_eval_per_epoch is n, then each epoch will be ' + 'evaluated every N/n samples.') +flags.DEFINE_integer('train_batch_size', 32, 'Batch size for training.') +flags.DEFINE_integer('eval_batch_size', 32, 'Batch size for evaluation.') + +common_flags.define_common_bert_flags() + +FLAGS = flags.FLAGS + +LABEL_TYPES_MAP = {'int': tf.int64, 'float': tf.float32} + + +def get_loss_fn(num_classes): + """Gets the classification loss function.""" + + def classification_loss_fn(labels, logits): + """Classification loss.""" + labels = tf.squeeze(labels) + log_probs = tf.nn.log_softmax(logits, axis=-1) + one_hot_labels = tf.one_hot( + tf.cast(labels, dtype=tf.int32), depth=num_classes, dtype=tf.float32) + per_example_loss = -tf.reduce_sum( + tf.cast(one_hot_labels, dtype=tf.float32) * log_probs, axis=-1) + return tf.reduce_mean(per_example_loss) + + return classification_loss_fn + + +def get_dataset_fn(input_file_pattern, + max_seq_length, + global_batch_size, + is_training, + label_type=tf.int64, + include_sample_weights=False, + num_samples=None): + """Gets a closure to create a dataset.""" + + def _dataset_fn(ctx=None): + """Returns tf.data.Dataset for distributed BERT pretraining.""" + batch_size = ctx.get_per_replica_batch_size( + global_batch_size) if ctx else global_batch_size + dataset = input_pipeline.create_classifier_dataset( + tf.io.gfile.glob(input_file_pattern), + max_seq_length, + batch_size, + is_training=is_training, + input_pipeline_context=ctx, + label_type=label_type, + include_sample_weights=include_sample_weights, + num_samples=num_samples) + return dataset + + return _dataset_fn + + +def run_bert_classifier(strategy, + bert_config, + input_meta_data, + model_dir, + epochs, + steps_per_epoch, + steps_per_loop, + eval_steps, + warmup_steps, + initial_lr, + init_checkpoint, + train_input_fn, + eval_input_fn, + training_callbacks=True, + custom_callbacks=None, + custom_metrics=None): + """Run BERT classifier training using low-level API.""" + max_seq_length = input_meta_data['max_seq_length'] + num_classes = input_meta_data.get('num_labels', 1) + is_regression = num_classes == 1 + + def _get_classifier_model(): + """Gets a classifier model.""" + classifier_model, core_model = ( + bert_models.classifier_model( + bert_config, + num_classes, + max_seq_length, + hub_module_url=FLAGS.hub_module_url, + hub_module_trainable=FLAGS.hub_module_trainable)) + optimizer = optimization.create_optimizer(initial_lr, + steps_per_epoch * epochs, + warmup_steps, FLAGS.end_lr, + FLAGS.optimizer_type) + classifier_model.optimizer = performance.configure_optimizer( + optimizer, + use_float16=common_flags.use_float16(), + use_graph_rewrite=common_flags.use_graph_rewrite()) + return classifier_model, core_model + + # tf.keras.losses objects accept optional sample_weight arguments (eg. coming + # from the dataset) to compute weighted loss, as used for the regression + # tasks. The classification tasks, using the custom get_loss_fn don't accept + # sample weights though. + loss_fn = (tf.keras.losses.MeanSquaredError() if is_regression + else get_loss_fn(num_classes)) + + # Defines evaluation metrics function, which will create metrics in the + # correct device and strategy scope. + if custom_metrics: + metric_fn = custom_metrics + elif is_regression: + metric_fn = functools.partial( + tf.keras.metrics.MeanSquaredError, + 'mean_squared_error', + dtype=tf.float32) + else: + metric_fn = functools.partial( + tf.keras.metrics.SparseCategoricalAccuracy, + 'accuracy', + dtype=tf.float32) + + # Start training using Keras compile/fit API. + logging.info('Training using TF 2.x Keras compile/fit API with ' + 'distribution strategy.') + return run_keras_compile_fit( + model_dir, + strategy, + _get_classifier_model, + train_input_fn, + eval_input_fn, + loss_fn, + metric_fn, + init_checkpoint, + epochs, + steps_per_epoch, + steps_per_loop, + eval_steps, + training_callbacks=training_callbacks, + custom_callbacks=custom_callbacks) + + +def run_keras_compile_fit(model_dir, + strategy, + model_fn, + train_input_fn, + eval_input_fn, + loss_fn, + metric_fn, + init_checkpoint, + epochs, + steps_per_epoch, + steps_per_loop, + eval_steps, + training_callbacks=True, + custom_callbacks=None): + """Runs BERT classifier model using Keras compile/fit API.""" + + with strategy.scope(): + training_dataset = train_input_fn() + evaluation_dataset = eval_input_fn() if eval_input_fn else None + bert_model, sub_model = model_fn() + optimizer = bert_model.optimizer + + if init_checkpoint: + checkpoint = tf.train.Checkpoint(model=sub_model, encoder=sub_model) + checkpoint.read(init_checkpoint).assert_existing_objects_matched() + + if not isinstance(metric_fn, (list, tuple)): + metric_fn = [metric_fn] + bert_model.compile( + optimizer=optimizer, + loss=loss_fn, + metrics=[fn() for fn in metric_fn], + steps_per_execution=steps_per_loop) + + summary_dir = os.path.join(model_dir, 'summaries') + summary_callback = tf.keras.callbacks.TensorBoard(summary_dir) + checkpoint = tf.train.Checkpoint(model=bert_model, optimizer=optimizer) + checkpoint_manager = tf.train.CheckpointManager( + checkpoint, + directory=model_dir, + max_to_keep=None, + step_counter=optimizer.iterations, + checkpoint_interval=0) + checkpoint_callback = keras_utils.SimpleCheckpoint(checkpoint_manager) + + if training_callbacks: + if custom_callbacks is not None: + custom_callbacks += [summary_callback, checkpoint_callback] + else: + custom_callbacks = [summary_callback, checkpoint_callback] + + history = bert_model.fit( + x=training_dataset, + validation_data=evaluation_dataset, + steps_per_epoch=steps_per_epoch, + epochs=epochs, + validation_steps=eval_steps, + callbacks=custom_callbacks) + stats = {'total_training_steps': steps_per_epoch * epochs} + if 'loss' in history.history: + stats['train_loss'] = history.history['loss'][-1] + if 'val_accuracy' in history.history: + stats['eval_metrics'] = history.history['val_accuracy'][-1] + return bert_model, stats + + +def get_predictions_and_labels(strategy, + trained_model, + eval_input_fn, + is_regression=False, + return_probs=False): + """Obtains predictions of trained model on evaluation data. + + Note that list of labels is returned along with the predictions because the + order changes on distributing dataset over TPU pods. + + Args: + strategy: Distribution strategy. + trained_model: Trained model with preloaded weights. + eval_input_fn: Input function for evaluation data. + is_regression: Whether it is a regression task. + return_probs: Whether to return probabilities of classes. + + Returns: + predictions: List of predictions. + labels: List of gold labels corresponding to predictions. + """ + + @tf.function + def test_step(iterator): + """Computes predictions on distributed devices.""" + + def _test_step_fn(inputs): + """Replicated predictions.""" + inputs, labels = inputs + logits = trained_model(inputs, training=False) + if not is_regression: + probabilities = tf.nn.softmax(logits) + return probabilities, labels + else: + return logits, labels + + outputs, labels = strategy.run(_test_step_fn, args=(next(iterator),)) + # outputs: current batch logits as a tuple of shard logits + outputs = tf.nest.map_structure(strategy.experimental_local_results, + outputs) + labels = tf.nest.map_structure(strategy.experimental_local_results, labels) + return outputs, labels + + def _run_evaluation(test_iterator): + """Runs evaluation steps.""" + preds, golds = list(), list() + try: + with tf.experimental.async_scope(): + while True: + probabilities, labels = test_step(test_iterator) + for cur_probs, cur_labels in zip(probabilities, labels): + if return_probs: + preds.extend(cur_probs.numpy().tolist()) + else: + preds.extend(tf.math.argmax(cur_probs, axis=1).numpy()) + golds.extend(cur_labels.numpy().tolist()) + except (StopIteration, tf.errors.OutOfRangeError): + tf.experimental.async_clear_error() + return preds, golds + + test_iter = iter(strategy.distribute_datasets_from_function(eval_input_fn)) + predictions, labels = _run_evaluation(test_iter) + + return predictions, labels + + +def export_classifier(model_export_path, input_meta_data, bert_config, + model_dir): + """Exports a trained model as a `SavedModel` for inference. + + Args: + model_export_path: a string specifying the path to the SavedModel directory. + input_meta_data: dictionary containing meta data about input and model. + bert_config: Bert configuration file to define core bert layers. + model_dir: The directory where the model weights and training/evaluation + summaries are stored. + + Raises: + Export path is not specified, got an empty string or None. + """ + if not model_export_path: + raise ValueError('Export path is not specified: %s' % model_export_path) + if not model_dir: + raise ValueError('Export path is not specified: %s' % model_dir) + + # Export uses float32 for now, even if training uses mixed precision. + tf.keras.mixed_precision.set_global_policy('float32') + classifier_model = bert_models.classifier_model( + bert_config, + input_meta_data.get('num_labels', 1), + hub_module_url=FLAGS.hub_module_url, + hub_module_trainable=False)[0] + + model_saving_utils.export_bert_model( + model_export_path, model=classifier_model, checkpoint_dir=model_dir) + + +def run_bert(strategy, + input_meta_data, + model_config, + train_input_fn=None, + eval_input_fn=None, + init_checkpoint=None, + custom_callbacks=None, + custom_metrics=None): + """Run BERT training.""" + # Enables XLA in Session Config. Should not be set for TPU. + keras_utils.set_session_config(FLAGS.enable_xla) + performance.set_mixed_precision_policy(common_flags.dtype()) + + epochs = FLAGS.num_train_epochs * FLAGS.num_eval_per_epoch + train_data_size = ( + input_meta_data['train_data_size'] // FLAGS.num_eval_per_epoch) + if FLAGS.train_data_size: + train_data_size = min(train_data_size, FLAGS.train_data_size) + logging.info('Updated train_data_size: %s', train_data_size) + steps_per_epoch = int(train_data_size / FLAGS.train_batch_size) + warmup_steps = int(epochs * train_data_size * 0.1 / FLAGS.train_batch_size) + eval_steps = int( + math.ceil(input_meta_data['eval_data_size'] / FLAGS.eval_batch_size)) + + if not strategy: + raise ValueError('Distribution strategy has not been specified.') + + if not custom_callbacks: + custom_callbacks = [] + + if FLAGS.log_steps: + custom_callbacks.append( + keras_utils.TimeHistory( + batch_size=FLAGS.train_batch_size, + log_steps=FLAGS.log_steps, + logdir=FLAGS.model_dir)) + + trained_model, _ = run_bert_classifier( + strategy, + model_config, + input_meta_data, + FLAGS.model_dir, + epochs, + steps_per_epoch, + FLAGS.steps_per_loop, + eval_steps, + warmup_steps, + FLAGS.learning_rate, + init_checkpoint or FLAGS.init_checkpoint, + train_input_fn, + eval_input_fn, + custom_callbacks=custom_callbacks, + custom_metrics=custom_metrics) + + if FLAGS.model_export_path: + model_saving_utils.export_bert_model( + FLAGS.model_export_path, model=trained_model) + return trained_model + + +def custom_main(custom_callbacks=None, custom_metrics=None): + """Run classification or regression. + + Args: + custom_callbacks: list of tf.keras.Callbacks passed to training loop. + custom_metrics: list of metrics passed to the training loop. + """ + gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_param) + + with tf.io.gfile.GFile(FLAGS.input_meta_data_path, 'rb') as reader: + input_meta_data = json.loads(reader.read().decode('utf-8')) + label_type = LABEL_TYPES_MAP[input_meta_data.get('label_type', 'int')] + include_sample_weights = input_meta_data.get('has_sample_weights', False) + + if not FLAGS.model_dir: + FLAGS.model_dir = '/tmp/bert20/' + + bert_config = bert_configs.BertConfig.from_json_file(FLAGS.bert_config_file) + + if FLAGS.mode == 'export_only': + export_classifier(FLAGS.model_export_path, input_meta_data, bert_config, + FLAGS.model_dir) + return + + strategy = distribute_utils.get_distribution_strategy( + distribution_strategy=FLAGS.distribution_strategy, + num_gpus=FLAGS.num_gpus, + tpu_address=FLAGS.tpu) + eval_input_fn = get_dataset_fn( + FLAGS.eval_data_path, + input_meta_data['max_seq_length'], + FLAGS.eval_batch_size, + is_training=False, + label_type=label_type, + include_sample_weights=include_sample_weights) + + if FLAGS.mode == 'predict': + num_labels = input_meta_data.get('num_labels', 1) + with strategy.scope(): + classifier_model = bert_models.classifier_model( + bert_config, num_labels)[0] + checkpoint = tf.train.Checkpoint(model=classifier_model) + latest_checkpoint_file = ( + FLAGS.predict_checkpoint_path or + tf.train.latest_checkpoint(FLAGS.model_dir)) + assert latest_checkpoint_file + logging.info('Checkpoint file %s found and restoring from ' + 'checkpoint', latest_checkpoint_file) + checkpoint.restore( + latest_checkpoint_file).assert_existing_objects_matched() + preds, _ = get_predictions_and_labels( + strategy, + classifier_model, + eval_input_fn, + is_regression=(num_labels == 1), + return_probs=True) + output_predict_file = os.path.join(FLAGS.model_dir, 'test_results.tsv') + with tf.io.gfile.GFile(output_predict_file, 'w') as writer: + logging.info('***** Predict results *****') + for probabilities in preds: + output_line = '\t'.join( + str(class_probability) + for class_probability in probabilities) + '\n' + writer.write(output_line) + return + + if FLAGS.mode != 'train_and_eval': + raise ValueError('Unsupported mode is specified: %s' % FLAGS.mode) + train_input_fn = get_dataset_fn( + FLAGS.train_data_path, + input_meta_data['max_seq_length'], + FLAGS.train_batch_size, + is_training=True, + label_type=label_type, + include_sample_weights=include_sample_weights, + num_samples=FLAGS.train_data_size) + run_bert( + strategy, + input_meta_data, + bert_config, + train_input_fn, + eval_input_fn, + custom_callbacks=custom_callbacks, + custom_metrics=custom_metrics) + + +def main(_): + custom_main(custom_callbacks=None, custom_metrics=None) + + +if __name__ == '__main__': + flags.mark_flag_as_required('bert_config_file') + flags.mark_flag_as_required('input_meta_data_path') + flags.mark_flag_as_required('model_dir') + app.run(main) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/run_pretraining.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/run_pretraining.py new file mode 100644 index 0000000000000000000000000000000000000000..6e21b991d7d41a017c34ae9dc139c772f004d91d --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/run_pretraining.py @@ -0,0 +1,218 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run masked LM/next sentence pre-training for BERT in TF 2.x.""" + +# Import libraries +from absl import app +from absl import flags +from absl import logging +import gin +import tensorflow as tf +from official.common import distribute_utils +from official.modeling import performance +from official.nlp import optimization +from official.nlp.bert import bert_models +from official.nlp.bert import common_flags +from official.nlp.bert import configs +from official.nlp.bert import input_pipeline +from official.nlp.bert import model_training_utils + + +flags.DEFINE_string('input_files', None, + 'File path to retrieve training data for pre-training.') +# Model training specific flags. +flags.DEFINE_integer( + 'max_seq_length', 128, + 'The maximum total input sequence length after WordPiece tokenization. ' + 'Sequences longer than this will be truncated, and sequences shorter ' + 'than this will be padded.') +flags.DEFINE_integer('max_predictions_per_seq', 20, + 'Maximum predictions per sequence_output.') +flags.DEFINE_integer('train_batch_size', 32, 'Total batch size for training.') +flags.DEFINE_integer('num_steps_per_epoch', 1000, + 'Total number of training steps to run per epoch.') +flags.DEFINE_float('warmup_steps', 10000, + 'Warmup steps for Adam weight decay optimizer.') +flags.DEFINE_bool('use_next_sentence_label', True, + 'Whether to use next sentence label to compute final loss.') +flags.DEFINE_bool('train_summary_interval', 0, 'Step interval for training ' + 'summaries. If the value is a negative number, ' + 'then training summaries are not enabled.') + +common_flags.define_common_bert_flags() + +FLAGS = flags.FLAGS + + +def get_pretrain_dataset_fn(input_file_pattern, seq_length, + max_predictions_per_seq, global_batch_size, + use_next_sentence_label=True): + """Returns input dataset from input file string.""" + def _dataset_fn(ctx=None): + """Returns tf.data.Dataset for distributed BERT pretraining.""" + input_patterns = input_file_pattern.split(',') + batch_size = ctx.get_per_replica_batch_size(global_batch_size) + train_dataset = input_pipeline.create_pretrain_dataset( + input_patterns, + seq_length, + max_predictions_per_seq, + batch_size, + is_training=True, + input_pipeline_context=ctx, + use_next_sentence_label=use_next_sentence_label) + return train_dataset + + return _dataset_fn + + +def get_loss_fn(): + """Returns loss function for BERT pretraining.""" + + def _bert_pretrain_loss_fn(unused_labels, losses, **unused_args): + return tf.reduce_mean(losses) + + return _bert_pretrain_loss_fn + + +def run_customized_training(strategy, + bert_config, + init_checkpoint, + max_seq_length, + max_predictions_per_seq, + model_dir, + steps_per_epoch, + steps_per_loop, + epochs, + initial_lr, + warmup_steps, + end_lr, + optimizer_type, + input_files, + train_batch_size, + use_next_sentence_label=True, + train_summary_interval=0, + custom_callbacks=None, + explicit_allreduce=False, + pre_allreduce_callbacks=None, + post_allreduce_callbacks=None, + allreduce_bytes_per_pack=0): + """Run BERT pretrain model training using low-level API.""" + + train_input_fn = get_pretrain_dataset_fn(input_files, max_seq_length, + max_predictions_per_seq, + train_batch_size, + use_next_sentence_label) + + def _get_pretrain_model(): + """Gets a pretraining model.""" + pretrain_model, core_model = bert_models.pretrain_model( + bert_config, max_seq_length, max_predictions_per_seq, + use_next_sentence_label=use_next_sentence_label) + optimizer = optimization.create_optimizer( + initial_lr, steps_per_epoch * epochs, warmup_steps, + end_lr, optimizer_type) + pretrain_model.optimizer = performance.configure_optimizer( + optimizer, + use_float16=common_flags.use_float16(), + use_graph_rewrite=common_flags.use_graph_rewrite()) + return pretrain_model, core_model + + trained_model = model_training_utils.run_customized_training_loop( + strategy=strategy, + model_fn=_get_pretrain_model, + loss_fn=get_loss_fn(), + scale_loss=FLAGS.scale_loss, + model_dir=model_dir, + init_checkpoint=init_checkpoint, + train_input_fn=train_input_fn, + steps_per_epoch=steps_per_epoch, + steps_per_loop=steps_per_loop, + epochs=epochs, + sub_model_export_name='pretrained/bert_model', + explicit_allreduce=explicit_allreduce, + pre_allreduce_callbacks=pre_allreduce_callbacks, + post_allreduce_callbacks=post_allreduce_callbacks, + allreduce_bytes_per_pack=allreduce_bytes_per_pack, + train_summary_interval=train_summary_interval, + custom_callbacks=custom_callbacks) + + return trained_model + + +def run_bert_pretrain(strategy, custom_callbacks=None): + """Runs BERT pre-training.""" + + bert_config = configs.BertConfig.from_json_file(FLAGS.bert_config_file) + if not strategy: + raise ValueError('Distribution strategy is not specified.') + + # Runs customized training loop. + logging.info('Training using customized training loop TF 2.0 with distributed' + 'strategy.') + + performance.set_mixed_precision_policy(common_flags.dtype()) + + # Only when explicit_allreduce = True, post_allreduce_callbacks and + # allreduce_bytes_per_pack will take effect. optimizer.apply_gradients() no + # longer implicitly allreduce gradients, users manually allreduce gradient and + # pass the allreduced grads_and_vars to apply_gradients(). + # With explicit_allreduce = True, clip_by_global_norm is moved to after + # allreduce. + return run_customized_training( + strategy, + bert_config, + FLAGS.init_checkpoint, # Used to initialize only the BERT submodel. + FLAGS.max_seq_length, + FLAGS.max_predictions_per_seq, + FLAGS.model_dir, + FLAGS.num_steps_per_epoch, + FLAGS.steps_per_loop, + FLAGS.num_train_epochs, + FLAGS.learning_rate, + FLAGS.warmup_steps, + FLAGS.end_lr, + FLAGS.optimizer_type, + FLAGS.input_files, + FLAGS.train_batch_size, + FLAGS.use_next_sentence_label, + FLAGS.train_summary_interval, + custom_callbacks=custom_callbacks, + explicit_allreduce=FLAGS.explicit_allreduce, + pre_allreduce_callbacks=[ + model_training_utils.clip_by_global_norm_callback + ], + allreduce_bytes_per_pack=FLAGS.allreduce_bytes_per_pack) + + +def main(_): + gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_param) + if not FLAGS.model_dir: + FLAGS.model_dir = '/tmp/bert20/' + # Configures cluster spec for multi-worker distribution strategy. + if FLAGS.num_gpus > 0: + _ = distribute_utils.configure_cluster(FLAGS.worker_hosts, FLAGS.task_index) + strategy = distribute_utils.get_distribution_strategy( + distribution_strategy=FLAGS.distribution_strategy, + num_gpus=FLAGS.num_gpus, + all_reduce_alg=FLAGS.all_reduce_alg, + tpu_address=FLAGS.tpu) + if strategy: + print('***** Number of cores used : ', strategy.num_replicas_in_sync) + + run_bert_pretrain(strategy) + + +if __name__ == '__main__': + app.run(main) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/run_squad.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/run_squad.py new file mode 100644 index 0000000000000000000000000000000000000000..8cafb917620abe6d969fecb563c3794bc78afc00 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/run_squad.py @@ -0,0 +1,148 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run BERT on SQuAD 1.1 and SQuAD 2.0 in TF 2.x.""" + +import json +import os +import time + +# Import libraries +from absl import app +from absl import flags +from absl import logging +import gin +import tensorflow as tf +from official.common import distribute_utils +from official.nlp.bert import configs as bert_configs +from official.nlp.bert import run_squad_helper +from official.nlp.bert import tokenization +from official.nlp.data import squad_lib as squad_lib_wp +from official.utils.misc import keras_utils + + +flags.DEFINE_string('vocab_file', None, + 'The vocabulary file that the BERT model was trained on.') + +# More flags can be found in run_squad_helper. +run_squad_helper.define_common_squad_flags() + +FLAGS = flags.FLAGS + + +def train_squad(strategy, + input_meta_data, + custom_callbacks=None, + run_eagerly=False, + init_checkpoint=None, + sub_model_export_name=None): + """Run bert squad training.""" + bert_config = bert_configs.BertConfig.from_json_file(FLAGS.bert_config_file) + init_checkpoint = init_checkpoint or FLAGS.init_checkpoint + run_squad_helper.train_squad(strategy, input_meta_data, bert_config, + custom_callbacks, run_eagerly, init_checkpoint, + sub_model_export_name=sub_model_export_name) + + +def predict_squad(strategy, input_meta_data): + """Makes predictions for the squad dataset.""" + bert_config = bert_configs.BertConfig.from_json_file(FLAGS.bert_config_file) + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + run_squad_helper.predict_squad( + strategy, input_meta_data, tokenizer, bert_config, squad_lib_wp) + + +def eval_squad(strategy, input_meta_data): + """Evaluate on the squad dataset.""" + bert_config = bert_configs.BertConfig.from_json_file(FLAGS.bert_config_file) + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + eval_metrics = run_squad_helper.eval_squad( + strategy, input_meta_data, tokenizer, bert_config, squad_lib_wp) + return eval_metrics + + +def export_squad(model_export_path, input_meta_data): + """Exports a trained model as a `SavedModel` for inference. + + Args: + model_export_path: a string specifying the path to the SavedModel directory. + input_meta_data: dictionary containing meta data about input and model. + + Raises: + Export path is not specified, got an empty string or None. + """ + bert_config = bert_configs.BertConfig.from_json_file(FLAGS.bert_config_file) + run_squad_helper.export_squad(model_export_path, input_meta_data, bert_config) + + +def main(_): + gin.parse_config_files_and_bindings(FLAGS.gin_file, FLAGS.gin_param) + + with tf.io.gfile.GFile(FLAGS.input_meta_data_path, 'rb') as reader: + input_meta_data = json.loads(reader.read().decode('utf-8')) + + if FLAGS.mode == 'export_only': + export_squad(FLAGS.model_export_path, input_meta_data) + return + + # Configures cluster spec for multi-worker distribution strategy. + if FLAGS.num_gpus > 0: + _ = distribute_utils.configure_cluster(FLAGS.worker_hosts, FLAGS.task_index) + strategy = distribute_utils.get_distribution_strategy( + distribution_strategy=FLAGS.distribution_strategy, + num_gpus=FLAGS.num_gpus, + all_reduce_alg=FLAGS.all_reduce_alg, + tpu_address=FLAGS.tpu) + + if 'train' in FLAGS.mode: + if FLAGS.log_steps: + custom_callbacks = [keras_utils.TimeHistory( + batch_size=FLAGS.train_batch_size, + log_steps=FLAGS.log_steps, + logdir=FLAGS.model_dir, + )] + else: + custom_callbacks = None + + train_squad( + strategy, + input_meta_data, + custom_callbacks=custom_callbacks, + run_eagerly=FLAGS.run_eagerly, + sub_model_export_name=FLAGS.sub_model_export_name, + ) + if 'predict' in FLAGS.mode: + predict_squad(strategy, input_meta_data) + if 'eval' in FLAGS.mode: + eval_metrics = eval_squad(strategy, input_meta_data) + f1_score = eval_metrics['final_f1'] + logging.info('SQuAD eval F1-score: %f', f1_score) + summary_dir = os.path.join(FLAGS.model_dir, 'summaries', 'eval') + summary_writer = tf.summary.create_file_writer(summary_dir) + with summary_writer.as_default(): + # TODO(lehou): write to the correct step number. + tf.summary.scalar('F1-score', f1_score, step=0) + summary_writer.flush() + # Also write eval_metrics to json file. + squad_lib_wp.write_to_json_files( + eval_metrics, os.path.join(summary_dir, 'eval_metrics.json')) + time.sleep(60) + + +if __name__ == '__main__': + flags.mark_flag_as_required('bert_config_file') + flags.mark_flag_as_required('model_dir') + app.run(main) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/run_squad_helper.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/run_squad_helper.py new file mode 100644 index 0000000000000000000000000000000000000000..8736de9b053314fd77733a653131cd8179f1fa54 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/run_squad_helper.py @@ -0,0 +1,472 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Library for running BERT family models on SQuAD 1.1/2.0 in TF 2.x.""" + +import collections +import json +import os + +from absl import flags +from absl import logging +import tensorflow as tf +from official.modeling import performance +from official.nlp import optimization +from official.nlp.bert import bert_models +from official.nlp.bert import common_flags +from official.nlp.bert import input_pipeline +from official.nlp.bert import model_saving_utils +from official.nlp.bert import model_training_utils +from official.nlp.bert import squad_evaluate_v1_1 +from official.nlp.bert import squad_evaluate_v2_0 +from official.nlp.data import squad_lib_sp +from official.utils.misc import keras_utils + + +def define_common_squad_flags(): + """Defines common flags used by SQuAD tasks.""" + flags.DEFINE_enum( + 'mode', 'train_and_eval', [ + 'train_and_eval', 'train_and_predict', 'train', 'eval', 'predict', + 'export_only' + ], 'One of {"train_and_eval", "train_and_predict", ' + '"train", "eval", "predict", "export_only"}. ' + '`train_and_eval`: train & predict to json files & compute eval metrics. ' + '`train_and_predict`: train & predict to json files. ' + '`train`: only trains the model. ' + '`eval`: predict answers from squad json file & compute eval metrics. ' + '`predict`: predict answers from the squad json file. ' + '`export_only`: will take the latest checkpoint inside ' + 'model_dir and export a `SavedModel`.') + flags.DEFINE_string('train_data_path', '', + 'Training data path with train tfrecords.') + flags.DEFINE_string( + 'input_meta_data_path', None, + 'Path to file that contains meta data about input ' + 'to be used for training and evaluation.') + # Model training specific flags. + flags.DEFINE_integer('train_batch_size', 32, 'Total batch size for training.') + # Predict processing related. + flags.DEFINE_string( + 'predict_file', None, 'SQuAD prediction json file path. ' + '`predict` mode supports multiple files: one can use ' + 'wildcard to specify multiple files and it can also be ' + 'multiple file patterns separated by comma. Note that ' + '`eval` mode only supports a single predict file.') + flags.DEFINE_bool( + 'do_lower_case', True, + 'Whether to lower case the input text. Should be True for uncased ' + 'models and False for cased models.') + flags.DEFINE_float( + 'null_score_diff_threshold', 0.0, + 'If null_score - best_non_null is greater than the threshold, ' + 'predict null. This is only used for SQuAD v2.') + flags.DEFINE_bool( + 'verbose_logging', False, + 'If true, all of the warnings related to data processing will be ' + 'printed. A number of warnings are expected for a normal SQuAD ' + 'evaluation.') + flags.DEFINE_integer('predict_batch_size', 8, + 'Total batch size for prediction.') + flags.DEFINE_integer( + 'n_best_size', 20, + 'The total number of n-best predictions to generate in the ' + 'nbest_predictions.json output file.') + flags.DEFINE_integer( + 'max_answer_length', 30, + 'The maximum length of an answer that can be generated. This is needed ' + 'because the start and end predictions are not conditioned on one ' + 'another.') + + common_flags.define_common_bert_flags() + + +FLAGS = flags.FLAGS + + +def squad_loss_fn(start_positions, end_positions, start_logits, end_logits): + """Returns sparse categorical crossentropy for start/end logits.""" + start_loss = tf.keras.losses.sparse_categorical_crossentropy( + start_positions, start_logits, from_logits=True) + end_loss = tf.keras.losses.sparse_categorical_crossentropy( + end_positions, end_logits, from_logits=True) + + total_loss = (tf.reduce_mean(start_loss) + tf.reduce_mean(end_loss)) / 2 + return total_loss + + +def get_loss_fn(): + """Gets a loss function for squad task.""" + + def _loss_fn(labels, model_outputs): + start_positions = labels['start_positions'] + end_positions = labels['end_positions'] + start_logits, end_logits = model_outputs + return squad_loss_fn(start_positions, end_positions, start_logits, + end_logits) + + return _loss_fn + + +RawResult = collections.namedtuple('RawResult', + ['unique_id', 'start_logits', 'end_logits']) + + +def get_raw_results(predictions): + """Converts multi-replica predictions to RawResult.""" + for unique_ids, start_logits, end_logits in zip(predictions['unique_ids'], + predictions['start_logits'], + predictions['end_logits']): + for values in zip(unique_ids.numpy(), start_logits.numpy(), + end_logits.numpy()): + yield RawResult( + unique_id=values[0], + start_logits=values[1].tolist(), + end_logits=values[2].tolist()) + + +def get_dataset_fn(input_file_pattern, max_seq_length, global_batch_size, + is_training): + """Gets a closure to create a dataset..""" + + def _dataset_fn(ctx=None): + """Returns tf.data.Dataset for distributed BERT pretraining.""" + batch_size = ctx.get_per_replica_batch_size( + global_batch_size) if ctx else global_batch_size + dataset = input_pipeline.create_squad_dataset( + input_file_pattern, + max_seq_length, + batch_size, + is_training=is_training, + input_pipeline_context=ctx) + return dataset + + return _dataset_fn + + +def get_squad_model_to_predict(strategy, bert_config, checkpoint_path, + input_meta_data): + """Gets a squad model to make predictions.""" + with strategy.scope(): + # Prediction always uses float32, even if training uses mixed precision. + tf.keras.mixed_precision.set_global_policy('float32') + squad_model, _ = bert_models.squad_model( + bert_config, + input_meta_data['max_seq_length'], + hub_module_url=FLAGS.hub_module_url) + + if checkpoint_path is None: + checkpoint_path = tf.train.latest_checkpoint(FLAGS.model_dir) + logging.info('Restoring checkpoints from %s', checkpoint_path) + checkpoint = tf.train.Checkpoint(model=squad_model) + checkpoint.restore(checkpoint_path).expect_partial() + return squad_model + + +def predict_squad_customized(strategy, input_meta_data, predict_tfrecord_path, + num_steps, squad_model): + """Make predictions using a Bert-based squad model.""" + predict_dataset_fn = get_dataset_fn( + predict_tfrecord_path, + input_meta_data['max_seq_length'], + FLAGS.predict_batch_size, + is_training=False) + predict_iterator = iter( + strategy.distribute_datasets_from_function(predict_dataset_fn)) + + @tf.function + def predict_step(iterator): + """Predicts on distributed devices.""" + + def _replicated_step(inputs): + """Replicated prediction calculation.""" + x, _ = inputs + unique_ids = x.pop('unique_ids') + start_logits, end_logits = squad_model(x, training=False) + return dict( + unique_ids=unique_ids, + start_logits=start_logits, + end_logits=end_logits) + + outputs = strategy.run(_replicated_step, args=(next(iterator),)) + return tf.nest.map_structure(strategy.experimental_local_results, outputs) + + all_results = [] + for _ in range(num_steps): + predictions = predict_step(predict_iterator) + for result in get_raw_results(predictions): + all_results.append(result) + if len(all_results) % 100 == 0: + logging.info('Made predictions for %d records.', len(all_results)) + return all_results + + +def train_squad(strategy, + input_meta_data, + bert_config, + custom_callbacks=None, + run_eagerly=False, + init_checkpoint=None, + sub_model_export_name=None): + """Run bert squad training.""" + if strategy: + logging.info('Training using customized training loop with distribution' + ' strategy.') + # Enables XLA in Session Config. Should not be set for TPU. + keras_utils.set_session_config(FLAGS.enable_xla) + performance.set_mixed_precision_policy(common_flags.dtype()) + + epochs = FLAGS.num_train_epochs + num_train_examples = input_meta_data['train_data_size'] + max_seq_length = input_meta_data['max_seq_length'] + steps_per_epoch = int(num_train_examples / FLAGS.train_batch_size) + warmup_steps = int(epochs * num_train_examples * 0.1 / FLAGS.train_batch_size) + train_input_fn = get_dataset_fn( + FLAGS.train_data_path, + max_seq_length, + FLAGS.train_batch_size, + is_training=True) + + def _get_squad_model(): + """Get Squad model and optimizer.""" + squad_model, core_model = bert_models.squad_model( + bert_config, + max_seq_length, + hub_module_url=FLAGS.hub_module_url, + hub_module_trainable=FLAGS.hub_module_trainable) + optimizer = optimization.create_optimizer(FLAGS.learning_rate, + steps_per_epoch * epochs, + warmup_steps, FLAGS.end_lr, + FLAGS.optimizer_type) + + squad_model.optimizer = performance.configure_optimizer( + optimizer, + use_float16=common_flags.use_float16(), + use_graph_rewrite=common_flags.use_graph_rewrite()) + return squad_model, core_model + + # Only when explicit_allreduce = True, post_allreduce_callbacks and + # allreduce_bytes_per_pack will take effect. optimizer.apply_gradients() no + # longer implicitly allreduce gradients, users manually allreduce gradient and + # pass the allreduced grads_and_vars to apply_gradients(). + # With explicit_allreduce = True, clip_by_global_norm is moved to after + # allreduce. + model_training_utils.run_customized_training_loop( + strategy=strategy, + model_fn=_get_squad_model, + loss_fn=get_loss_fn(), + model_dir=FLAGS.model_dir, + steps_per_epoch=steps_per_epoch, + steps_per_loop=FLAGS.steps_per_loop, + epochs=epochs, + train_input_fn=train_input_fn, + init_checkpoint=init_checkpoint or FLAGS.init_checkpoint, + sub_model_export_name=sub_model_export_name, + run_eagerly=run_eagerly, + custom_callbacks=custom_callbacks, + explicit_allreduce=FLAGS.explicit_allreduce, + pre_allreduce_callbacks=[ + model_training_utils.clip_by_global_norm_callback + ], + allreduce_bytes_per_pack=FLAGS.allreduce_bytes_per_pack) + + +def prediction_output_squad(strategy, input_meta_data, tokenizer, squad_lib, + predict_file, squad_model): + """Makes predictions for a squad dataset.""" + doc_stride = input_meta_data['doc_stride'] + max_query_length = input_meta_data['max_query_length'] + # Whether data should be in Ver 2.0 format. + version_2_with_negative = input_meta_data.get('version_2_with_negative', + False) + eval_examples = squad_lib.read_squad_examples( + input_file=predict_file, + is_training=False, + version_2_with_negative=version_2_with_negative) + + eval_writer = squad_lib.FeatureWriter( + filename=os.path.join(FLAGS.model_dir, 'eval.tf_record'), + is_training=False) + eval_features = [] + + def _append_feature(feature, is_padding): + if not is_padding: + eval_features.append(feature) + eval_writer.process_feature(feature) + + # TPU requires a fixed batch size for all batches, therefore the number + # of examples must be a multiple of the batch size, or else examples + # will get dropped. So we pad with fake examples which are ignored + # later on. + kwargs = dict( + examples=eval_examples, + tokenizer=tokenizer, + max_seq_length=input_meta_data['max_seq_length'], + doc_stride=doc_stride, + max_query_length=max_query_length, + is_training=False, + output_fn=_append_feature, + batch_size=FLAGS.predict_batch_size) + + # squad_lib_sp requires one more argument 'do_lower_case'. + if squad_lib == squad_lib_sp: + kwargs['do_lower_case'] = FLAGS.do_lower_case + dataset_size = squad_lib.convert_examples_to_features(**kwargs) + eval_writer.close() + + logging.info('***** Running predictions *****') + logging.info(' Num orig examples = %d', len(eval_examples)) + logging.info(' Num split examples = %d', len(eval_features)) + logging.info(' Batch size = %d', FLAGS.predict_batch_size) + + num_steps = int(dataset_size / FLAGS.predict_batch_size) + all_results = predict_squad_customized(strategy, input_meta_data, + eval_writer.filename, num_steps, + squad_model) + + all_predictions, all_nbest_json, scores_diff_json = ( + squad_lib.postprocess_output( + eval_examples, + eval_features, + all_results, + FLAGS.n_best_size, + FLAGS.max_answer_length, + FLAGS.do_lower_case, + version_2_with_negative=version_2_with_negative, + null_score_diff_threshold=FLAGS.null_score_diff_threshold, + verbose=FLAGS.verbose_logging)) + + return all_predictions, all_nbest_json, scores_diff_json + + +def dump_to_files(all_predictions, + all_nbest_json, + scores_diff_json, + squad_lib, + version_2_with_negative, + file_prefix=''): + """Save output to json files.""" + output_prediction_file = os.path.join(FLAGS.model_dir, + '%spredictions.json' % file_prefix) + output_nbest_file = os.path.join(FLAGS.model_dir, + '%snbest_predictions.json' % file_prefix) + output_null_log_odds_file = os.path.join(FLAGS.model_dir, file_prefix, + '%snull_odds.json' % file_prefix) + logging.info('Writing predictions to: %s', (output_prediction_file)) + logging.info('Writing nbest to: %s', (output_nbest_file)) + + squad_lib.write_to_json_files(all_predictions, output_prediction_file) + squad_lib.write_to_json_files(all_nbest_json, output_nbest_file) + if version_2_with_negative: + squad_lib.write_to_json_files(scores_diff_json, output_null_log_odds_file) + + +def _get_matched_files(input_path): + """Returns all files that matches the input_path.""" + input_patterns = input_path.strip().split(',') + all_matched_files = [] + for input_pattern in input_patterns: + input_pattern = input_pattern.strip() + if not input_pattern: + continue + matched_files = tf.io.gfile.glob(input_pattern) + if not matched_files: + raise ValueError('%s does not match any files.' % input_pattern) + else: + all_matched_files.extend(matched_files) + return sorted(all_matched_files) + + +def predict_squad(strategy, + input_meta_data, + tokenizer, + bert_config, + squad_lib, + init_checkpoint=None): + """Get prediction results and evaluate them to hard drive.""" + if init_checkpoint is None: + init_checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir) + + all_predict_files = _get_matched_files(FLAGS.predict_file) + squad_model = get_squad_model_to_predict(strategy, bert_config, + init_checkpoint, input_meta_data) + for idx, predict_file in enumerate(all_predict_files): + all_predictions, all_nbest_json, scores_diff_json = prediction_output_squad( + strategy, input_meta_data, tokenizer, squad_lib, predict_file, + squad_model) + if len(all_predict_files) == 1: + file_prefix = '' + else: + # if predict_file is /path/xquad.ar.json, the `file_prefix` may be + # "xquad.ar-0-" + file_prefix = '%s-' % os.path.splitext( + os.path.basename(all_predict_files[idx]))[0] + dump_to_files(all_predictions, all_nbest_json, scores_diff_json, squad_lib, + input_meta_data.get('version_2_with_negative', False), + file_prefix) + + +def eval_squad(strategy, + input_meta_data, + tokenizer, + bert_config, + squad_lib, + init_checkpoint=None): + """Get prediction results and evaluate them against ground truth.""" + if init_checkpoint is None: + init_checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir) + + all_predict_files = _get_matched_files(FLAGS.predict_file) + if len(all_predict_files) != 1: + raise ValueError('`eval_squad` only supports one predict file, ' + 'but got %s' % all_predict_files) + + squad_model = get_squad_model_to_predict(strategy, bert_config, + init_checkpoint, input_meta_data) + all_predictions, all_nbest_json, scores_diff_json = prediction_output_squad( + strategy, input_meta_data, tokenizer, squad_lib, all_predict_files[0], + squad_model) + dump_to_files(all_predictions, all_nbest_json, scores_diff_json, squad_lib, + input_meta_data.get('version_2_with_negative', False)) + + with tf.io.gfile.GFile(FLAGS.predict_file, 'r') as reader: + dataset_json = json.load(reader) + pred_dataset = dataset_json['data'] + if input_meta_data.get('version_2_with_negative', False): + eval_metrics = squad_evaluate_v2_0.evaluate(pred_dataset, all_predictions, + scores_diff_json) + else: + eval_metrics = squad_evaluate_v1_1.evaluate(pred_dataset, all_predictions) + return eval_metrics + + +def export_squad(model_export_path, input_meta_data, bert_config): + """Exports a trained model as a `SavedModel` for inference. + + Args: + model_export_path: a string specifying the path to the SavedModel directory. + input_meta_data: dictionary containing meta data about input and model. + bert_config: Bert configuration file to define core bert layers. + + Raises: + Export path is not specified, got an empty string or None. + """ + if not model_export_path: + raise ValueError('Export path is not specified: %s' % model_export_path) + # Export uses float32 for now, even if training uses mixed precision. + tf.keras.mixed_precision.set_global_policy('float32') + squad_model, _ = bert_models.squad_model(bert_config, + input_meta_data['max_seq_length']) + model_saving_utils.export_bert_model( + model_export_path, model=squad_model, checkpoint_dir=FLAGS.model_dir) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/serving.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/serving.py new file mode 100644 index 0000000000000000000000000000000000000000..7e27869c74b30ae5ce1a8a9b75760d0d8013640a --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/serving.py @@ -0,0 +1,133 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Examples of SavedModel export for tf-serving.""" + +from absl import app +from absl import flags +import tensorflow as tf + +from official.nlp.bert import bert_models +from official.nlp.bert import configs + +flags.DEFINE_integer( + "sequence_length", None, "Sequence length to parse the tf.Example. If " + "sequence_length > 0, add a signature for serialized " + "tf.Example and define the parsing specification by the " + "sequence_length.") +flags.DEFINE_string("bert_config_file", None, + "Bert configuration file to define core bert layers.") +flags.DEFINE_string("model_checkpoint_path", None, + "File path to TF model checkpoint.") +flags.DEFINE_string("export_path", None, + "Destination folder to export the serving SavedModel.") + +FLAGS = flags.FLAGS + + +class BertServing(tf.keras.Model): + """Bert transformer encoder model for serving.""" + + def __init__(self, bert_config, name_to_features=None, name="serving_model"): + super(BertServing, self).__init__(name=name) + self.encoder = bert_models.get_transformer_encoder( + bert_config, sequence_length=None) + self.name_to_features = name_to_features + + def call(self, inputs): + input_word_ids = inputs["input_ids"] + input_mask = inputs["input_mask"] + input_type_ids = inputs["segment_ids"] + + encoder_outputs, _ = self.encoder( + [input_word_ids, input_mask, input_type_ids]) + return encoder_outputs + + def serve_body(self, input_ids, input_mask=None, segment_ids=None): + if segment_ids is None: + # Requires CLS token is the first token of inputs. + segment_ids = tf.zeros_like(input_ids) + if input_mask is None: + # The mask has 1 for real tokens and 0 for padding tokens. + input_mask = tf.where( + tf.equal(input_ids, 0), tf.zeros_like(input_ids), + tf.ones_like(input_ids)) + + inputs = dict( + input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids) + return self.call(inputs) + + @tf.function + def serve(self, input_ids, input_mask=None, segment_ids=None): + outputs = self.serve_body(input_ids, input_mask, segment_ids) + # Returns a dictionary to control SignatureDef output signature. + return {"outputs": outputs[-1]} + + @tf.function + def serve_examples(self, inputs): + features = tf.io.parse_example(inputs, self.name_to_features) + for key in list(features.keys()): + t = features[key] + if t.dtype == tf.int64: + t = tf.cast(t, tf.int32) + features[key] = t + return self.serve( + features["input_ids"], + input_mask=features["input_mask"] if "input_mask" in features else None, + segment_ids=features["segment_ids"] + if "segment_ids" in features else None) + + @classmethod + def export(cls, model, export_dir): + if not isinstance(model, cls): + raise ValueError("Invalid model instance: %s, it should be a %s" % + (model, cls)) + + signatures = { + "serving_default": + model.serve.get_concrete_function( + input_ids=tf.TensorSpec( + shape=[None, None], dtype=tf.int32, name="inputs")), + } + if model.name_to_features: + signatures[ + "serving_examples"] = model.serve_examples.get_concrete_function( + tf.TensorSpec(shape=[None], dtype=tf.string, name="examples")) + tf.saved_model.save(model, export_dir=export_dir, signatures=signatures) + + +def main(_): + sequence_length = FLAGS.sequence_length + if sequence_length is not None and sequence_length > 0: + name_to_features = { + "input_ids": tf.io.FixedLenFeature([sequence_length], tf.int64), + "input_mask": tf.io.FixedLenFeature([sequence_length], tf.int64), + "segment_ids": tf.io.FixedLenFeature([sequence_length], tf.int64), + } + else: + name_to_features = None + bert_config = configs.BertConfig.from_json_file(FLAGS.bert_config_file) + serving_model = BertServing( + bert_config=bert_config, name_to_features=name_to_features) + checkpoint = tf.train.Checkpoint(model=serving_model.encoder) + checkpoint.restore(FLAGS.model_checkpoint_path + ).assert_existing_objects_matched().run_restore_ops() + BertServing.export(serving_model, FLAGS.export_path) + + +if __name__ == "__main__": + flags.mark_flag_as_required("bert_config_file") + flags.mark_flag_as_required("model_checkpoint_path") + flags.mark_flag_as_required("export_path") + app.run(main) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/squad_evaluate_v1_1.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/squad_evaluate_v1_1.py new file mode 100644 index 0000000000000000000000000000000000000000..a39f571c37b002ab10cfe36a1454827d91512945 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/squad_evaluate_v1_1.py @@ -0,0 +1,106 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Evaluation of SQuAD predictions (version 1.1). + +The functions are copied from +https://worksheets.codalab.org/rest/bundles/0xbcd57bee090b421c982906709c8c27e1/contents/blob/. + +The SQuAD dataset is described in this paper: +SQuAD: 100,000+ Questions for Machine Comprehension of Text +Pranav Rajpurkar, Jian Zhang, Konstantin Lopyrev, Percy Liang +https://nlp.stanford.edu/pubs/rajpurkar2016squad.pdf +""" + +import collections +import re +import string + +# pylint: disable=g-bad-import-order + +from absl import logging +# pylint: enable=g-bad-import-order + + +def _normalize_answer(s): + """Lowers text and remove punctuation, articles and extra whitespace.""" + + def remove_articles(text): + return re.sub(r"\b(a|an|the)\b", " ", text) + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return "".join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def _f1_score(prediction, ground_truth): + """Computes F1 score by comparing prediction to ground truth.""" + prediction_tokens = _normalize_answer(prediction).split() + ground_truth_tokens = _normalize_answer(ground_truth).split() + prediction_counter = collections.Counter(prediction_tokens) + ground_truth_counter = collections.Counter(ground_truth_tokens) + common = prediction_counter & ground_truth_counter + num_same = sum(common.values()) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(prediction_tokens) + recall = 1.0 * num_same / len(ground_truth_tokens) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def _exact_match_score(prediction, ground_truth): + """Checks if predicted answer exactly matches ground truth answer.""" + return _normalize_answer(prediction) == _normalize_answer(ground_truth) + + +def _metric_max_over_ground_truths(metric_fn, prediction, ground_truths): + """Computes the max over all metric scores.""" + scores_for_ground_truths = [] + for ground_truth in ground_truths: + score = metric_fn(prediction, ground_truth) + scores_for_ground_truths.append(score) + return max(scores_for_ground_truths) + + +def evaluate(dataset, predictions): + """Evaluates predictions for a dataset.""" + f1 = exact_match = total = 0 + for article in dataset: + for paragraph in article["paragraphs"]: + for qa in paragraph["qas"]: + total += 1 + if qa["id"] not in predictions: + message = "Unanswered question " + qa["id"] + " will receive score 0." + logging.error(message) + continue + ground_truths = [entry["text"] for entry in qa["answers"]] + prediction = predictions[qa["id"]] + exact_match += _metric_max_over_ground_truths(_exact_match_score, + prediction, ground_truths) + f1 += _metric_max_over_ground_truths(_f1_score, prediction, + ground_truths) + + exact_match = exact_match / total + f1 = f1 / total + + return {"exact_match": exact_match, "final_f1": f1} diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/squad_evaluate_v2_0.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/squad_evaluate_v2_0.py new file mode 100644 index 0000000000000000000000000000000000000000..12c5a7e3d6b406e45e4f91580f8b4198733db37c --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/squad_evaluate_v2_0.py @@ -0,0 +1,249 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Evaluation script for SQuAD version 2.0. + +The functions are copied and modified from +https://raw.githubusercontent.com/white127/SQUAD-2.0-bidaf/master/evaluate-v2.0.py + +In addition to basic functionality, we also compute additional statistics and +plot precision-recall curves if an additional na_prob.json file is provided. +This file is expected to map question ID's to the model's predicted probability +that a question is unanswerable. +""" + +import collections +import re +import string + +from absl import logging + + +def _make_qid_to_has_ans(dataset): + qid_to_has_ans = {} + for article in dataset: + for p in article['paragraphs']: + for qa in p['qas']: + qid_to_has_ans[qa['id']] = bool(qa['answers']) + return qid_to_has_ans + + +def _normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + def remove_articles(text): + regex = re.compile(r'\b(a|an|the)\b', re.UNICODE) + return re.sub(regex, ' ', text) + def white_space_fix(text): + return ' '.join(text.split()) + def remove_punc(text): + exclude = set(string.punctuation) + return ''.join(ch for ch in text if ch not in exclude) + def lower(text): + return text.lower() + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def _get_tokens(s): + if not s: return [] + return _normalize_answer(s).split() + + +def _compute_exact(a_gold, a_pred): + return int(_normalize_answer(a_gold) == _normalize_answer(a_pred)) + + +def _compute_f1(a_gold, a_pred): + """Compute F1-score.""" + gold_toks = _get_tokens(a_gold) + pred_toks = _get_tokens(a_pred) + common = collections.Counter(gold_toks) & collections.Counter(pred_toks) + num_same = sum(common.values()) + if not gold_toks or not pred_toks: + # If either is no-answer, then F1 is 1 if they agree, 0 otherwise + return int(gold_toks == pred_toks) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(pred_toks) + recall = 1.0 * num_same / len(gold_toks) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def _get_raw_scores(dataset, predictions): + """Compute raw scores.""" + exact_scores = {} + f1_scores = {} + for article in dataset: + for p in article['paragraphs']: + for qa in p['qas']: + qid = qa['id'] + gold_answers = [a['text'] for a in qa['answers'] + if _normalize_answer(a['text'])] + if not gold_answers: + # For unanswerable questions, only correct answer is empty string + gold_answers = [''] + if qid not in predictions: + logging.error('Missing prediction for %s', qid) + continue + a_pred = predictions[qid] + # Take max over all gold answers + exact_scores[qid] = max(_compute_exact(a, a_pred) for a in gold_answers) + f1_scores[qid] = max(_compute_f1(a, a_pred) for a in gold_answers) + return exact_scores, f1_scores + + +def _apply_no_ans_threshold( + scores, na_probs, qid_to_has_ans, na_prob_thresh=1.0): + new_scores = {} + for qid, s in scores.items(): + pred_na = na_probs[qid] > na_prob_thresh + if pred_na: + new_scores[qid] = float(not qid_to_has_ans[qid]) + else: + new_scores[qid] = s + return new_scores + + +def _make_eval_dict(exact_scores, f1_scores, qid_list=None): + """Make evaluation result dictionary.""" + if not qid_list: + total = len(exact_scores) + return collections.OrderedDict([ + ('exact', 100.0 * sum(exact_scores.values()) / total), + ('f1', 100.0 * sum(f1_scores.values()) / total), + ('total', total), + ]) + else: + total = len(qid_list) + return collections.OrderedDict([ + ('exact', 100.0 * sum(exact_scores[k] for k in qid_list) / total), + ('f1', 100.0 * sum(f1_scores[k] for k in qid_list) / total), + ('total', total), + ]) + + +def _merge_eval(main_eval, new_eval, prefix): + for k in new_eval: + main_eval['%s_%s' % (prefix, k)] = new_eval[k] + + +def _make_precision_recall_eval(scores, na_probs, num_true_pos, qid_to_has_ans): + """Make evaluation dictionary containing average recision recall.""" + qid_list = sorted(na_probs, key=lambda k: na_probs[k]) + true_pos = 0.0 + cur_p = 1.0 + cur_r = 0.0 + precisions = [1.0] + recalls = [0.0] + avg_prec = 0.0 + for i, qid in enumerate(qid_list): + if qid_to_has_ans[qid]: + true_pos += scores[qid] + cur_p = true_pos / float(i+1) + cur_r = true_pos / float(num_true_pos) + if i == len(qid_list) - 1 or na_probs[qid] != na_probs[qid_list[i+1]]: + # i.e., if we can put a threshold after this point + avg_prec += cur_p * (cur_r - recalls[-1]) + precisions.append(cur_p) + recalls.append(cur_r) + return {'ap': 100.0 * avg_prec} + + +def _run_precision_recall_analysis( + main_eval, exact_raw, f1_raw, na_probs, qid_to_has_ans): + """Run precision recall analysis and return result dictionary.""" + num_true_pos = sum(1 for v in qid_to_has_ans.values() if v) + if num_true_pos == 0: + return + pr_exact = _make_precision_recall_eval( + exact_raw, na_probs, num_true_pos, qid_to_has_ans) + pr_f1 = _make_precision_recall_eval( + f1_raw, na_probs, num_true_pos, qid_to_has_ans) + oracle_scores = {k: float(v) for k, v in qid_to_has_ans.items()} + pr_oracle = _make_precision_recall_eval( + oracle_scores, na_probs, num_true_pos, qid_to_has_ans) + _merge_eval(main_eval, pr_exact, 'pr_exact') + _merge_eval(main_eval, pr_f1, 'pr_f1') + _merge_eval(main_eval, pr_oracle, 'pr_oracle') + + +def _find_best_thresh(predictions, scores, na_probs, qid_to_has_ans): + """Find the best threshold for no answer probability.""" + num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) + cur_score = num_no_ans + best_score = cur_score + best_thresh = 0.0 + qid_list = sorted(na_probs, key=lambda k: na_probs[k]) + for qid in qid_list: + if qid not in scores: continue + if qid_to_has_ans[qid]: + diff = scores[qid] + else: + if predictions[qid]: + diff = -1 + else: + diff = 0 + cur_score += diff + if cur_score > best_score: + best_score = cur_score + best_thresh = na_probs[qid] + return 100.0 * best_score / len(scores), best_thresh + + +def _find_all_best_thresh( + main_eval, predictions, exact_raw, f1_raw, na_probs, qid_to_has_ans): + best_exact, exact_thresh = _find_best_thresh( + predictions, exact_raw, na_probs, qid_to_has_ans) + best_f1, f1_thresh = _find_best_thresh( + predictions, f1_raw, na_probs, qid_to_has_ans) + main_eval['final_exact'] = best_exact + main_eval['final_exact_thresh'] = exact_thresh + main_eval['final_f1'] = best_f1 + main_eval['final_f1_thresh'] = f1_thresh + + +def evaluate(dataset, predictions, na_probs=None): + """Evaluate prediction results.""" + new_orig_data = [] + for article in dataset: + for p in article['paragraphs']: + for qa in p['qas']: + if qa['id'] in predictions: + new_para = {'qas': [qa]} + new_article = {'paragraphs': [new_para]} + new_orig_data.append(new_article) + dataset = new_orig_data + + if na_probs is None: + na_probs = {k: 0.0 for k in predictions} + qid_to_has_ans = _make_qid_to_has_ans(dataset) # maps qid to True/False + has_ans_qids = [k for k, v in qid_to_has_ans.items() if v] + no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v] + exact_raw, f1_raw = _get_raw_scores(dataset, predictions) + exact_thresh = _apply_no_ans_threshold(exact_raw, na_probs, qid_to_has_ans) + f1_thresh = _apply_no_ans_threshold(f1_raw, na_probs, qid_to_has_ans) + out_eval = _make_eval_dict(exact_thresh, f1_thresh) + if has_ans_qids: + has_ans_eval = _make_eval_dict( + exact_thresh, f1_thresh, qid_list=has_ans_qids) + _merge_eval(out_eval, has_ans_eval, 'HasAns') + if no_ans_qids: + no_ans_eval = _make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids) + _merge_eval(out_eval, no_ans_eval, 'NoAns') + + _find_all_best_thresh( + out_eval, predictions, exact_raw, f1_raw, na_probs, qid_to_has_ans) + _run_precision_recall_analysis( + out_eval, exact_raw, f1_raw, na_probs, qid_to_has_ans) + return out_eval diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/tf1_checkpoint_converter_lib.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/tf1_checkpoint_converter_lib.py new file mode 100644 index 0000000000000000000000000000000000000000..035a694385abfede7314188e38ab6801b6fef70a --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/tf1_checkpoint_converter_lib.py @@ -0,0 +1,201 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +r"""Convert checkpoints created by Estimator (tf1) to be Keras compatible.""" + +import numpy as np +import tensorflow.compat.v1 as tf # TF 1.x + +# Mapping between old <=> new names. The source pattern in original variable +# name will be replaced by destination pattern. +BERT_NAME_REPLACEMENTS = ( + ("bert", "bert_model"), + ("embeddings/word_embeddings", "word_embeddings/embeddings"), + ("embeddings/token_type_embeddings", + "embedding_postprocessor/type_embeddings"), + ("embeddings/position_embeddings", + "embedding_postprocessor/position_embeddings"), + ("embeddings/LayerNorm", "embedding_postprocessor/layer_norm"), + ("attention/self", "self_attention"), + ("attention/output/dense", "self_attention_output"), + ("attention/output/LayerNorm", "self_attention_layer_norm"), + ("intermediate/dense", "intermediate"), + ("output/dense", "output"), + ("output/LayerNorm", "output_layer_norm"), + ("pooler/dense", "pooler_transform"), +) + +BERT_V2_NAME_REPLACEMENTS = ( + ("bert/", ""), + ("encoder", "transformer"), + ("embeddings/word_embeddings", "word_embeddings/embeddings"), + ("embeddings/token_type_embeddings", "type_embeddings/embeddings"), + ("embeddings/position_embeddings", "position_embedding/embeddings"), + ("embeddings/LayerNorm", "embeddings/layer_norm"), + ("attention/self", "self_attention"), + ("attention/output/dense", "self_attention/attention_output"), + ("attention/output/LayerNorm", "self_attention_layer_norm"), + ("intermediate/dense", "intermediate"), + ("output/dense", "output"), + ("output/LayerNorm", "output_layer_norm"), + ("pooler/dense", "pooler_transform"), + ("cls/predictions", "bert/cls/predictions"), + ("cls/predictions/output_bias", "cls/predictions/output_bias/bias"), + ("cls/seq_relationship/output_bias", "predictions/transform/logits/bias"), + ("cls/seq_relationship/output_weights", + "predictions/transform/logits/kernel"), +) + +BERT_PERMUTATIONS = () + +BERT_V2_PERMUTATIONS = (("cls/seq_relationship/output_weights", (1, 0)),) + + +def _bert_name_replacement(var_name, name_replacements): + """Gets the variable name replacement.""" + for src_pattern, tgt_pattern in name_replacements: + if src_pattern in var_name: + old_var_name = var_name + var_name = var_name.replace(src_pattern, tgt_pattern) + tf.logging.info("Converted: %s --> %s", old_var_name, var_name) + return var_name + + +def _has_exclude_patterns(name, exclude_patterns): + """Checks if a string contains substrings that match patterns to exclude.""" + for p in exclude_patterns: + if p in name: + return True + return False + + +def _get_permutation(name, permutations): + """Checks whether a variable requires transposition by pattern matching.""" + for src_pattern, permutation in permutations: + if src_pattern in name: + tf.logging.info("Permuted: %s --> %s", name, permutation) + return permutation + + return None + + +def _get_new_shape(name, shape, num_heads): + """Checks whether a variable requires reshape by pattern matching.""" + if "self_attention/attention_output/kernel" in name: + return tuple([num_heads, shape[0] // num_heads, shape[1]]) + if "self_attention/attention_output/bias" in name: + return shape + + patterns = [ + "self_attention/query", "self_attention/value", "self_attention/key" + ] + for pattern in patterns: + if pattern in name: + if "kernel" in name: + return tuple([shape[0], num_heads, shape[1] // num_heads]) + if "bias" in name: + return tuple([num_heads, shape[0] // num_heads]) + return None + + +def create_v2_checkpoint(model, + src_checkpoint, + output_path, + checkpoint_model_name="model"): + """Converts a name-based matched TF V1 checkpoint to TF V2 checkpoint.""" + # Uses streaming-restore in eager model to read V1 name-based checkpoints. + model.load_weights(src_checkpoint).assert_existing_objects_matched() + if hasattr(model, "checkpoint_items"): + checkpoint_items = model.checkpoint_items + else: + checkpoint_items = {} + + checkpoint_items[checkpoint_model_name] = model + checkpoint = tf.train.Checkpoint(**checkpoint_items) + checkpoint.save(output_path) + + +def convert(checkpoint_from_path, + checkpoint_to_path, + num_heads, + name_replacements, + permutations, + exclude_patterns=None): + """Migrates the names of variables within a checkpoint. + + Args: + checkpoint_from_path: Path to source checkpoint to be read in. + checkpoint_to_path: Path to checkpoint to be written out. + num_heads: The number of heads of the model. + name_replacements: A list of tuples of the form (match_str, replace_str) + describing variable names to adjust. + permutations: A list of tuples of the form (match_str, permutation) + describing permutations to apply to given variables. Note that match_str + should match the original variable name, not the replaced one. + exclude_patterns: A list of string patterns to exclude variables from + checkpoint conversion. + + Returns: + A dictionary that maps the new variable names to the Variable objects. + A dictionary that maps the old variable names to the new variable names. + """ + with tf.Graph().as_default(): + tf.logging.info("Reading checkpoint_from_path %s", checkpoint_from_path) + reader = tf.train.NewCheckpointReader(checkpoint_from_path) + name_shape_map = reader.get_variable_to_shape_map() + new_variable_map = {} + conversion_map = {} + for var_name in name_shape_map: + if exclude_patterns and _has_exclude_patterns(var_name, exclude_patterns): + continue + # Get the original tensor data. + tensor = reader.get_tensor(var_name) + + # Look up the new variable name, if any. + new_var_name = _bert_name_replacement(var_name, name_replacements) + + # See if we need to reshape the underlying tensor. + new_shape = None + if num_heads > 0: + new_shape = _get_new_shape(new_var_name, tensor.shape, num_heads) + if new_shape: + tf.logging.info("Veriable %s has a shape change from %s to %s", + var_name, tensor.shape, new_shape) + tensor = np.reshape(tensor, new_shape) + + # See if we need to permute the underlying tensor. + permutation = _get_permutation(var_name, permutations) + if permutation: + tensor = np.transpose(tensor, permutation) + + # Create a new variable with the possibly-reshaped or transposed tensor. + var = tf.Variable(tensor, name=var_name) + + # Save the variable into the new variable map. + new_variable_map[new_var_name] = var + + # Keep a list of converter variables for sanity checking. + if new_var_name != var_name: + conversion_map[var_name] = new_var_name + + saver = tf.train.Saver(new_variable_map) + + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + tf.logging.info("Writing checkpoint_to_path %s", checkpoint_to_path) + saver.save(sess, checkpoint_to_path, write_meta_graph=False) + + tf.logging.info("Summary:") + tf.logging.info(" Converted %d variable name(s).", len(new_variable_map)) + tf.logging.info(" Converted: %s", str(conversion_map)) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/tf2_encoder_checkpoint_converter.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/tf2_encoder_checkpoint_converter.py new file mode 100644 index 0000000000000000000000000000000000000000..9fced5daee95479c28cffd2b63dffcf6f2d90408 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/tf2_encoder_checkpoint_converter.py @@ -0,0 +1,160 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A converter from a V1 BERT encoder checkpoint to a V2 encoder checkpoint. + +The conversion will yield an object-oriented checkpoint that can be used +to restore a BertEncoder or BertPretrainerV2 object (see the `converted_model` +FLAG below). +""" + +import os + +from absl import app +from absl import flags + +import tensorflow as tf +from official.modeling import tf_utils +from official.nlp.bert import configs +from official.nlp.bert import tf1_checkpoint_converter_lib +from official.nlp.modeling import models +from official.nlp.modeling import networks + +FLAGS = flags.FLAGS + +flags.DEFINE_string("bert_config_file", None, + "Bert configuration file to define core bert layers.") +flags.DEFINE_string( + "checkpoint_to_convert", None, + "Initial checkpoint from a pretrained BERT model core (that is, only the " + "BertModel, with no task heads.)") +flags.DEFINE_string("converted_checkpoint_path", None, + "Name for the created object-based V2 checkpoint.") +flags.DEFINE_string("checkpoint_model_name", "encoder", + "The name of the model when saving the checkpoint, i.e., " + "the checkpoint will be saved using: " + "tf.train.Checkpoint(FLAGS.checkpoint_model_name=model).") +flags.DEFINE_enum( + "converted_model", "encoder", ["encoder", "pretrainer"], + "Whether to convert the checkpoint to a `BertEncoder` model or a " + "`BertPretrainerV2` model (with mlm but without classification heads).") + + +def _create_bert_model(cfg): + """Creates a BERT keras core model from BERT configuration. + + Args: + cfg: A `BertConfig` to create the core model. + + Returns: + A BertEncoder network. + """ + bert_encoder = networks.BertEncoder( + vocab_size=cfg.vocab_size, + hidden_size=cfg.hidden_size, + num_layers=cfg.num_hidden_layers, + num_attention_heads=cfg.num_attention_heads, + intermediate_size=cfg.intermediate_size, + activation=tf_utils.get_activation(cfg.hidden_act), + dropout_rate=cfg.hidden_dropout_prob, + attention_dropout_rate=cfg.attention_probs_dropout_prob, + max_sequence_length=cfg.max_position_embeddings, + type_vocab_size=cfg.type_vocab_size, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=cfg.initializer_range), + embedding_width=cfg.embedding_size) + + return bert_encoder + + +def _create_bert_pretrainer_model(cfg): + """Creates a BERT keras core model from BERT configuration. + + Args: + cfg: A `BertConfig` to create the core model. + + Returns: + A BertPretrainerV2 model. + """ + bert_encoder = _create_bert_model(cfg) + pretrainer = models.BertPretrainerV2( + encoder_network=bert_encoder, + mlm_activation=tf_utils.get_activation(cfg.hidden_act), + mlm_initializer=tf.keras.initializers.TruncatedNormal( + stddev=cfg.initializer_range)) + # Makes sure the pretrainer variables are created. + _ = pretrainer(pretrainer.inputs) + return pretrainer + + +def convert_checkpoint(bert_config, + output_path, + v1_checkpoint, + checkpoint_model_name="model", + converted_model="encoder"): + """Converts a V1 checkpoint into an OO V2 checkpoint.""" + output_dir, _ = os.path.split(output_path) + tf.io.gfile.makedirs(output_dir) + + # Create a temporary V1 name-converted checkpoint in the output directory. + temporary_checkpoint_dir = os.path.join(output_dir, "temp_v1") + temporary_checkpoint = os.path.join(temporary_checkpoint_dir, "ckpt") + + tf1_checkpoint_converter_lib.convert( + checkpoint_from_path=v1_checkpoint, + checkpoint_to_path=temporary_checkpoint, + num_heads=bert_config.num_attention_heads, + name_replacements=tf1_checkpoint_converter_lib.BERT_V2_NAME_REPLACEMENTS, + permutations=tf1_checkpoint_converter_lib.BERT_V2_PERMUTATIONS, + exclude_patterns=["adam", "Adam"]) + + if converted_model == "encoder": + model = _create_bert_model(bert_config) + elif converted_model == "pretrainer": + model = _create_bert_pretrainer_model(bert_config) + else: + raise ValueError("Unsupported converted_model: %s" % converted_model) + + # Create a V2 checkpoint from the temporary checkpoint. + tf1_checkpoint_converter_lib.create_v2_checkpoint(model, temporary_checkpoint, + output_path, + checkpoint_model_name) + + # Clean up the temporary checkpoint, if it exists. + try: + tf.io.gfile.rmtree(temporary_checkpoint_dir) + except tf.errors.OpError: + # If it doesn't exist, we don't need to clean it up; continue. + pass + + +def main(argv): + if len(argv) > 1: + raise app.UsageError("Too many command-line arguments.") + + output_path = FLAGS.converted_checkpoint_path + v1_checkpoint = FLAGS.checkpoint_to_convert + checkpoint_model_name = FLAGS.checkpoint_model_name + converted_model = FLAGS.converted_model + bert_config = configs.BertConfig.from_json_file(FLAGS.bert_config_file) + convert_checkpoint( + bert_config=bert_config, + output_path=output_path, + v1_checkpoint=v1_checkpoint, + checkpoint_model_name=checkpoint_model_name, + converted_model=converted_model) + + +if __name__ == "__main__": + app.run(main) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/tokenization.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/tokenization.py new file mode 100644 index 0000000000000000000000000000000000000000..ea1546e3c29f33c593c64a4341366254da328b86 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/tokenization.py @@ -0,0 +1,541 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# coding=utf-8 +"""Tokenization classes implementation. + +The file is forked from: +https://github.com/google-research/bert/blob/master/tokenization.py. +""" + +import collections +import re +import unicodedata + +import six +import tensorflow as tf + +import sentencepiece as spm + +SPIECE_UNDERLINE = "▁" + + +def validate_case_matches_checkpoint(do_lower_case, init_checkpoint): + """Checks whether the casing config is consistent with the checkpoint name.""" + + # The casing has to be passed in by the user and there is no explicit check + # as to whether it matches the checkpoint. The casing information probably + # should have been stored in the bert_config.json file, but it's not, so + # we have to heuristically detect it to validate. + + if not init_checkpoint: + return + + m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint) + if m is None: + return + + model_name = m.group(1) + + lower_models = [ + "uncased_L-24_H-1024_A-16", "uncased_L-12_H-768_A-12", + "multilingual_L-12_H-768_A-12", "chinese_L-12_H-768_A-12" + ] + + cased_models = [ + "cased_L-12_H-768_A-12", "cased_L-24_H-1024_A-16", + "multi_cased_L-12_H-768_A-12" + ] + + is_bad_config = False + if model_name in lower_models and not do_lower_case: + is_bad_config = True + actual_flag = "False" + case_name = "lowercased" + opposite_flag = "True" + + if model_name in cased_models and do_lower_case: + is_bad_config = True + actual_flag = "True" + case_name = "cased" + opposite_flag = "False" + + if is_bad_config: + raise ValueError( + "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. " + "However, `%s` seems to be a %s model, so you " + "should pass in `--do_lower_case=%s` so that the fine-tuning matches " + "how the model was pre-training. If this error is wrong, please " + "just comment out this check." % + (actual_flag, init_checkpoint, model_name, case_name, opposite_flag)) + + +def convert_to_unicode(text): + """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text.decode("utf-8", "ignore") + elif isinstance(text, unicode): + return text + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def printable_text(text): + """Returns text encoded in a way suitable for print or `tf.logging`.""" + + # These functions want `str` for both Python2 and Python3, but in one case + # it's a Unicode string and in the other it's a byte string. + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text + elif isinstance(text, unicode): + return text.encode("utf-8") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + index = 0 + with tf.io.gfile.GFile(vocab_file, "r") as reader: + while True: + token = convert_to_unicode(reader.readline()) + if not token: + break + token = token.strip() + vocab[token] = index + index += 1 + return vocab + + +def convert_by_vocab(vocab, items): + """Converts a sequence of [tokens|ids] using the vocab.""" + output = [] + for item in items: + output.append(vocab[item]) + return output + + +def convert_tokens_to_ids(vocab, tokens): + return convert_by_vocab(vocab, tokens) + + +def convert_ids_to_tokens(inv_vocab, ids): + return convert_by_vocab(inv_vocab, ids) + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class FullTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True, split_on_punc=True): + self.vocab = load_vocab(vocab_file) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.basic_tokenizer = BasicTokenizer( + do_lower_case=do_lower_case, split_on_punc=split_on_punc) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + return convert_by_vocab(self.inv_vocab, ids) + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True, split_on_punc=True): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + split_on_punc: Whether to apply split on punctuations. By default BERT + starts a new token for punctuations. This makes detokenization difficult + for tasks like seq2seq decoding. + """ + self.do_lower_case = do_lower_case + self.split_on_punc = split_on_punc + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = convert_to_unicode(text) + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case: + token = token.lower() + token = self._run_strip_accents(token) + if self.split_on_punc: + split_tokens.extend(self._run_split_on_punc(token)) + else: + split_tokens.append(token) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ((cp >= 0x4E00 and cp <= 0x9FFF) or # + (cp >= 0x3400 and cp <= 0x4DBF) or # + (cp >= 0x20000 and cp <= 0x2A6DF) or # + (cp >= 0x2A700 and cp <= 0x2B73F) or # + (cp >= 0x2B740 and cp <= 0x2B81F) or # + (cp >= 0x2B820 and cp <= 0x2CEAF) or + (cp >= 0xF900 and cp <= 0xFAFF) or # + (cp >= 0x2F800 and cp <= 0x2FA1F)): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xfffd or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenziation.""" + + def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=400): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer. + + Returns: + A list of wordpiece tokens. + """ + + text = convert_to_unicode(text) + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + if substr in self.vocab: + cur_substr = substr + break + end -= 1 + if cur_substr is None: + is_bad = True + break + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically control characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat in ("Cc", "Cf"): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or + (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False + + +def preprocess_text(inputs, remove_space=True, lower=False): + """Preprocesses data by removing extra space and normalize data. + + This method is used together with sentence piece tokenizer and is forked from: + https://github.com/google-research/google-research/blob/e1f6fa00/albert/tokenization.py + + Args: + inputs: The input text. + remove_space: Whether to remove the extra space. + lower: Whether to lowercase the text. + + Returns: + The preprocessed text. + + """ + outputs = inputs + if remove_space: + outputs = " ".join(inputs.strip().split()) + + if six.PY2 and isinstance(outputs, str): + try: + outputs = six.ensure_text(outputs, "utf-8") + except UnicodeDecodeError: + outputs = six.ensure_text(outputs, "latin-1") + + outputs = unicodedata.normalize("NFKD", outputs) + outputs = "".join([c for c in outputs if not unicodedata.combining(c)]) + if lower: + outputs = outputs.lower() + + return outputs + + +def encode_pieces(sp_model, text, sample=False): + """Segements text into pieces. + + This method is used together with sentence piece tokenizer and is forked from: + https://github.com/google-research/google-research/blob/e1f6fa00/albert/tokenization.py + + + Args: + sp_model: A spm.SentencePieceProcessor object. + text: The input text to be segemented. + sample: Whether to randomly sample a segmentation output or return a + deterministic one. + + Returns: + A list of token pieces. + """ + if six.PY2 and isinstance(text, six.text_type): + text = six.ensure_binary(text, "utf-8") + + if not sample: + pieces = sp_model.EncodeAsPieces(text) + else: + pieces = sp_model.SampleEncodeAsPieces(text, 64, 0.1) + new_pieces = [] + for piece in pieces: + piece = printable_text(piece) + if len(piece) > 1 and piece[-1] == "," and piece[-2].isdigit(): + cur_pieces = sp_model.EncodeAsPieces(piece[:-1].replace( + SPIECE_UNDERLINE, "")) + if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE: + if len(cur_pieces[0]) == 1: + cur_pieces = cur_pieces[1:] + else: + cur_pieces[0] = cur_pieces[0][1:] + cur_pieces.append(piece[-1]) + new_pieces.extend(cur_pieces) + else: + new_pieces.append(piece) + + return new_pieces + + +def encode_ids(sp_model, text, sample=False): + """Segments text and return token ids. + + This method is used together with sentence piece tokenizer and is forked from: + https://github.com/google-research/google-research/blob/e1f6fa00/albert/tokenization.py + + Args: + sp_model: A spm.SentencePieceProcessor object. + text: The input text to be segemented. + sample: Whether to randomly sample a segmentation output or return a + deterministic one. + + Returns: + A list of token ids. + """ + pieces = encode_pieces(sp_model, text, sample=sample) + ids = [sp_model.PieceToId(piece) for piece in pieces] + return ids + + +class FullSentencePieceTokenizer(object): + """Runs end-to-end sentence piece tokenization. + + The interface of this class is intended to keep the same as above + `FullTokenizer` class for easier usage. + """ + + def __init__(self, sp_model_file): + """Inits FullSentencePieceTokenizer. + + Args: + sp_model_file: The path to the sentence piece model file. + """ + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(sp_model_file) + self.vocab = { + self.sp_model.IdToPiece(i): i + for i in six.moves.range(self.sp_model.GetPieceSize()) + } + + def tokenize(self, text): + """Tokenizes text into pieces.""" + return encode_pieces(self.sp_model, text) + + def convert_tokens_to_ids(self, tokens): + """Converts a list of tokens to a list of ids.""" + return [self.sp_model.PieceToId(printable_text(token)) for token in tokens] + + def convert_ids_to_tokens(self, ids): + """Converts a list of ids ot a list of tokens.""" + return [self.sp_model.IdToPiece(id_) for id_ in ids] diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/tokenization_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/tokenization_test.py new file mode 100644 index 0000000000000000000000000000000000000000..07759de20b7c6eaf1a964c110da645215c10753a --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/bert/tokenization_test.py @@ -0,0 +1,156 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile + +import six +import tensorflow as tf + +from official.nlp.bert import tokenization + + +class TokenizationTest(tf.test.TestCase): + """Tokenization test. + + The implementation is forked from + https://github.com/google-research/bert/blob/master/tokenization_test.py." + """ + + def test_full_tokenizer(self): + vocab_tokens = [ + "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", + "##ing", "," + ] + with tempfile.NamedTemporaryFile(delete=False) as vocab_writer: + if six.PY2: + vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) + else: + vocab_writer.write("".join([x + "\n" for x in vocab_tokens + ]).encode("utf-8")) + + vocab_file = vocab_writer.name + + tokenizer = tokenization.FullTokenizer(vocab_file) + os.unlink(vocab_file) + + tokens = tokenizer.tokenize(u"UNwant\u00E9d,running") + self.assertAllEqual(tokens, ["un", "##want", "##ed", ",", "runn", "##ing"]) + + self.assertAllEqual( + tokenizer.convert_tokens_to_ids(tokens), [7, 4, 5, 10, 8, 9]) + + def test_chinese(self): + tokenizer = tokenization.BasicTokenizer() + + self.assertAllEqual( + tokenizer.tokenize(u"ah\u535A\u63A8zz"), + [u"ah", u"\u535A", u"\u63A8", u"zz"]) + + def test_basic_tokenizer_lower(self): + tokenizer = tokenization.BasicTokenizer(do_lower_case=True) + + self.assertAllEqual( + tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), + ["hello", "!", "how", "are", "you", "?"]) + self.assertAllEqual(tokenizer.tokenize(u"H\u00E9llo"), ["hello"]) + + def test_basic_tokenizer_no_lower(self): + tokenizer = tokenization.BasicTokenizer(do_lower_case=False) + + self.assertAllEqual( + tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), + ["HeLLo", "!", "how", "Are", "yoU", "?"]) + + def test_basic_tokenizer_no_split_on_punc(self): + tokenizer = tokenization.BasicTokenizer( + do_lower_case=True, split_on_punc=False) + + self.assertAllEqual( + tokenizer.tokenize(u" \tHeLLo!how \n Are yoU? "), + ["hello!how", "are", "you?"]) + + def test_wordpiece_tokenizer(self): + vocab_tokens = [ + "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", + "##ing", "##!", "!" + ] + + vocab = {} + for (i, token) in enumerate(vocab_tokens): + vocab[token] = i + tokenizer = tokenization.WordpieceTokenizer(vocab=vocab) + + self.assertAllEqual(tokenizer.tokenize(""), []) + + self.assertAllEqual( + tokenizer.tokenize("unwanted running"), + ["un", "##want", "##ed", "runn", "##ing"]) + + self.assertAllEqual( + tokenizer.tokenize("unwanted running !"), + ["un", "##want", "##ed", "runn", "##ing", "!"]) + + self.assertAllEqual( + tokenizer.tokenize("unwanted running!"), + ["un", "##want", "##ed", "runn", "##ing", "##!"]) + + self.assertAllEqual( + tokenizer.tokenize("unwantedX running"), ["[UNK]", "runn", "##ing"]) + + def test_convert_tokens_to_ids(self): + vocab_tokens = [ + "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", + "##ing" + ] + + vocab = {} + for (i, token) in enumerate(vocab_tokens): + vocab[token] = i + + self.assertAllEqual( + tokenization.convert_tokens_to_ids( + vocab, ["un", "##want", "##ed", "runn", "##ing"]), [7, 4, 5, 8, 9]) + + def test_is_whitespace(self): + self.assertTrue(tokenization._is_whitespace(u" ")) + self.assertTrue(tokenization._is_whitespace(u"\t")) + self.assertTrue(tokenization._is_whitespace(u"\r")) + self.assertTrue(tokenization._is_whitespace(u"\n")) + self.assertTrue(tokenization._is_whitespace(u"\u00A0")) + + self.assertFalse(tokenization._is_whitespace(u"A")) + self.assertFalse(tokenization._is_whitespace(u"-")) + + def test_is_control(self): + self.assertTrue(tokenization._is_control(u"\u0005")) + + self.assertFalse(tokenization._is_control(u"A")) + self.assertFalse(tokenization._is_control(u" ")) + self.assertFalse(tokenization._is_control(u"\t")) + self.assertFalse(tokenization._is_control(u"\r")) + self.assertFalse(tokenization._is_control(u"\U0001F4A9")) + + def test_is_punctuation(self): + self.assertTrue(tokenization._is_punctuation(u"-")) + self.assertTrue(tokenization._is_punctuation(u"$")) + self.assertTrue(tokenization._is_punctuation(u"`")) + self.assertTrue(tokenization._is_punctuation(u".")) + + self.assertFalse(tokenization._is_punctuation(u"A")) + self.assertFalse(tokenization._is_punctuation(u" ")) + + +if __name__ == "__main__": + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/__init__.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e419af524b5f349fe04abfa820c3cb51b777d422 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/bert.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/bert.py new file mode 100644 index 0000000000000000000000000000000000000000..cf78de0388bf76b68cd6df8cc656842bbfc90b64 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/bert.py @@ -0,0 +1,43 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Multi-head BERT encoder network with classification heads. + +Includes configurations and instantiation methods. +""" +from typing import List, Optional, Text + +import dataclasses + +from official.modeling.hyperparams import base_config +from official.nlp.configs import encoders + + +@dataclasses.dataclass +class ClsHeadConfig(base_config.Config): + inner_dim: int = 0 + num_classes: int = 2 + activation: Optional[Text] = "tanh" + dropout_rate: float = 0.0 + cls_token_idx: int = 0 + name: Optional[Text] = None + + +@dataclasses.dataclass +class PretrainerConfig(base_config.Config): + """Pretrainer configuration.""" + encoder: encoders.EncoderConfig = encoders.EncoderConfig() + cls_heads: List[ClsHeadConfig] = dataclasses.field(default_factory=list) + mlm_activation: str = "gelu" + mlm_initializer_range: float = 0.02 diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/electra.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/electra.py new file mode 100644 index 0000000000000000000000000000000000000000..5e62297667a470fd192779d8dc7f5c5117836804 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/electra.py @@ -0,0 +1,36 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ELECTRA model configurations and instantiation methods.""" +from typing import List + +import dataclasses + +from official.modeling.hyperparams import base_config +from official.nlp.configs import bert +from official.nlp.configs import encoders + + +@dataclasses.dataclass +class ElectraPretrainerConfig(base_config.Config): + """ELECTRA pretrainer configuration.""" + num_masked_tokens: int = 76 + sequence_length: int = 512 + num_classes: int = 2 + discriminator_loss_weight: float = 50.0 + tie_embeddings: bool = True + disallow_correct: bool = False + generator_encoder: encoders.EncoderConfig = encoders.EncoderConfig() + discriminator_encoder: encoders.EncoderConfig = encoders.EncoderConfig() + cls_heads: List[bert.ClsHeadConfig] = dataclasses.field(default_factory=list) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/encoders.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/encoders.py new file mode 100644 index 0000000000000000000000000000000000000000..1e720f69df5a4ac595df8bacaa416abda657ed9a --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/encoders.py @@ -0,0 +1,563 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Transformer Encoders. + +Includes configurations and factory methods. +""" +from typing import Optional + +import dataclasses +import gin +import tensorflow as tf + +from official.modeling import hyperparams +from official.modeling import tf_utils +from official.nlp.modeling import layers +from official.nlp.modeling import networks +from official.nlp.projects.bigbird import encoder as bigbird_encoder + + +@dataclasses.dataclass +class BertEncoderConfig(hyperparams.Config): + """BERT encoder configuration.""" + vocab_size: int = 30522 + hidden_size: int = 768 + num_layers: int = 12 + num_attention_heads: int = 12 + hidden_activation: str = "gelu" + intermediate_size: int = 3072 + dropout_rate: float = 0.1 + attention_dropout_rate: float = 0.1 + max_position_embeddings: int = 512 + type_vocab_size: int = 2 + initializer_range: float = 0.02 + embedding_size: Optional[int] = None + output_range: Optional[int] = None + return_all_encoder_outputs: bool = False + # Pre/Post-LN Transformer + norm_first: bool = False + + +@dataclasses.dataclass +class MobileBertEncoderConfig(hyperparams.Config): + """MobileBERT encoder configuration. + + Attributes: + word_vocab_size: number of words in the vocabulary. + word_embed_size: word embedding size. + type_vocab_size: number of word types. + max_sequence_length: maximum length of input sequence. + num_blocks: number of transformer block in the encoder model. + hidden_size: the hidden size for the transformer block. + num_attention_heads: number of attention heads in the transformer block. + intermediate_size: the size of the "intermediate" (a.k.a., feed forward) + layer. + hidden_activation: the non-linear activation function to apply to the + output of the intermediate/feed-forward layer. + hidden_dropout_prob: dropout probability for the hidden layers. + attention_probs_dropout_prob: dropout probability of the attention + probabilities. + intra_bottleneck_size: the size of bottleneck. + initializer_range: The stddev of the truncated_normal_initializer for + initializing all weight matrices. + use_bottleneck_attention: Use attention inputs from the bottleneck + transformation. If true, the following `key_query_shared_bottleneck` + will be ignored. + key_query_shared_bottleneck: whether to share linear transformation for keys + and queries. + num_feedforward_networks: number of stacked feed-forward networks. + normalization_type: the type of normalization_type, only 'no_norm' and + 'layer_norm' are supported. 'no_norm' represents the element-wise linear + transformation for the student model, as suggested by the original + MobileBERT paper. 'layer_norm' is used for the teacher model. + classifier_activation: if using the tanh activation for the final + representation of the [CLS] token in fine-tuning. + """ + word_vocab_size: int = 30522 + word_embed_size: int = 128 + type_vocab_size: int = 2 + max_sequence_length: int = 512 + num_blocks: int = 24 + hidden_size: int = 512 + num_attention_heads: int = 4 + intermediate_size: int = 4096 + hidden_activation: str = "gelu" + hidden_dropout_prob: float = 0.1 + attention_probs_dropout_prob: float = 0.1 + intra_bottleneck_size: int = 1024 + initializer_range: float = 0.02 + use_bottleneck_attention: bool = False + key_query_shared_bottleneck: bool = False + num_feedforward_networks: int = 1 + normalization_type: str = "layer_norm" + classifier_activation: bool = True + input_mask_dtype: str = "int32" + + +@dataclasses.dataclass +class AlbertEncoderConfig(hyperparams.Config): + """ALBERT encoder configuration.""" + vocab_size: int = 30000 + embedding_width: int = 128 + hidden_size: int = 768 + num_layers: int = 12 + num_attention_heads: int = 12 + hidden_activation: str = "gelu" + intermediate_size: int = 3072 + dropout_rate: float = 0.0 + attention_dropout_rate: float = 0.0 + max_position_embeddings: int = 512 + type_vocab_size: int = 2 + initializer_range: float = 0.02 + + +@dataclasses.dataclass +class BigBirdEncoderConfig(hyperparams.Config): + """BigBird encoder configuration.""" + vocab_size: int = 50358 + hidden_size: int = 768 + num_layers: int = 12 + num_attention_heads: int = 12 + hidden_activation: str = "gelu" + intermediate_size: int = 3072 + dropout_rate: float = 0.1 + attention_dropout_rate: float = 0.1 + # Pre/Post-LN Transformer + norm_first: bool = False + max_position_embeddings: int = 4096 + num_rand_blocks: int = 3 + block_size: int = 64 + type_vocab_size: int = 16 + initializer_range: float = 0.02 + embedding_width: Optional[int] = None + use_gradient_checkpointing: bool = False + + +@dataclasses.dataclass +class KernelEncoderConfig(hyperparams.Config): + """Linear encoder configuration.""" + vocab_size: int = 30522 + hidden_size: int = 768 + num_layers: int = 12 + num_attention_heads: int = 12 + hidden_activation: str = "gelu" + intermediate_size: int = 3072 + dropout_rate: float = 0.1 + attention_dropout_rate: float = 0.1 + # Pre/Post-LN Transformer + norm_first: bool = False + max_position_embeddings: int = 512 + type_vocab_size: int = 2 + initializer_range: float = 0.02 + embedding_size: Optional[int] = None + feature_transform: str = "exp" + num_random_features: int = 256 + redraw: bool = False + is_short_seq: bool = False + begin_kernel: int = 0 + scale: Optional[float] = None + + +@dataclasses.dataclass +class ReuseEncoderConfig(hyperparams.Config): + """Reuse encoder configuration.""" + vocab_size: int = 30522 + hidden_size: int = 768 + num_layers: int = 12 + num_attention_heads: int = 12 + hidden_activation: str = "gelu" + intermediate_size: int = 3072 + dropout_rate: float = 0.1 + attention_dropout_rate: float = 0.1 + max_position_embeddings: int = 512 + type_vocab_size: int = 2 + initializer_range: float = 0.02 + embedding_size: Optional[int] = None + output_range: Optional[int] = None + return_all_encoder_outputs: bool = False + # Pre/Post-LN Transformer + norm_first: bool = False + # Reuse transformer + reuse_attention: int = -1 + use_relative_pe: bool = False + pe_max_seq_length: int = 512 + max_reuse_layer_idx: int = 6 + + +@dataclasses.dataclass +class XLNetEncoderConfig(hyperparams.Config): + """XLNet encoder configuration.""" + vocab_size: int = 32000 + num_layers: int = 24 + hidden_size: int = 1024 + num_attention_heads: int = 16 + head_size: int = 64 + inner_size: int = 4096 + inner_activation: str = "gelu" + dropout_rate: float = 0.1 + attention_dropout_rate: float = 0.1 + attention_type: str = "bi" + bi_data: bool = False + tie_attention_biases: bool = False + memory_length: int = 0 + same_length: bool = False + clamp_length: int = -1 + reuse_length: int = 0 + use_cls_mask: bool = False + embedding_width: int = 1024 + initializer_range: float = 0.02 + two_stream: bool = False + + +@dataclasses.dataclass +class EncoderConfig(hyperparams.OneOfConfig): + """Encoder configuration.""" + type: Optional[str] = "bert" + albert: AlbertEncoderConfig = AlbertEncoderConfig() + bert: BertEncoderConfig = BertEncoderConfig() + bert_v2: BertEncoderConfig = BertEncoderConfig() + bigbird: BigBirdEncoderConfig = BigBirdEncoderConfig() + kernel: KernelEncoderConfig = KernelEncoderConfig() + mobilebert: MobileBertEncoderConfig = MobileBertEncoderConfig() + reuse: ReuseEncoderConfig = ReuseEncoderConfig() + teams: BertEncoderConfig = BertEncoderConfig() + xlnet: XLNetEncoderConfig = XLNetEncoderConfig() + + +@gin.configurable +def build_encoder(config: EncoderConfig, + embedding_layer: Optional[tf.keras.layers.Layer] = None, + encoder_cls=None, + bypass_config: bool = False): + """Instantiate a Transformer encoder network from EncoderConfig. + + Args: + config: the one-of encoder config, which provides encoder parameters of a + chosen encoder. + embedding_layer: an external embedding layer passed to the encoder. + encoder_cls: an external encoder cls not included in the supported encoders, + usually used by gin.configurable. + bypass_config: whether to ignore config instance to create the object with + `encoder_cls`. + + Returns: + An encoder instance. + """ + if bypass_config: + return encoder_cls() + encoder_type = config.type + encoder_cfg = config.get() + if encoder_cls and encoder_cls.__name__ == "EncoderScaffold": + embedding_cfg = dict( + vocab_size=encoder_cfg.vocab_size, + type_vocab_size=encoder_cfg.type_vocab_size, + hidden_size=encoder_cfg.hidden_size, + max_seq_length=encoder_cfg.max_position_embeddings, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + dropout_rate=encoder_cfg.dropout_rate, + ) + hidden_cfg = dict( + num_attention_heads=encoder_cfg.num_attention_heads, + intermediate_size=encoder_cfg.intermediate_size, + intermediate_activation=tf_utils.get_activation( + encoder_cfg.hidden_activation), + dropout_rate=encoder_cfg.dropout_rate, + attention_dropout_rate=encoder_cfg.attention_dropout_rate, + kernel_initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + ) + kwargs = dict( + embedding_cfg=embedding_cfg, + hidden_cfg=hidden_cfg, + num_hidden_instances=encoder_cfg.num_layers, + pooled_output_dim=encoder_cfg.hidden_size, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + return_all_layer_outputs=encoder_cfg.return_all_encoder_outputs, + dict_outputs=True) + return encoder_cls(**kwargs) + + if encoder_type == "mobilebert": + return networks.MobileBERTEncoder( + word_vocab_size=encoder_cfg.word_vocab_size, + word_embed_size=encoder_cfg.word_embed_size, + type_vocab_size=encoder_cfg.type_vocab_size, + max_sequence_length=encoder_cfg.max_sequence_length, + num_blocks=encoder_cfg.num_blocks, + hidden_size=encoder_cfg.hidden_size, + num_attention_heads=encoder_cfg.num_attention_heads, + intermediate_size=encoder_cfg.intermediate_size, + intermediate_act_fn=encoder_cfg.hidden_activation, + hidden_dropout_prob=encoder_cfg.hidden_dropout_prob, + attention_probs_dropout_prob=encoder_cfg.attention_probs_dropout_prob, + intra_bottleneck_size=encoder_cfg.intra_bottleneck_size, + initializer_range=encoder_cfg.initializer_range, + use_bottleneck_attention=encoder_cfg.use_bottleneck_attention, + key_query_shared_bottleneck=encoder_cfg.key_query_shared_bottleneck, + num_feedforward_networks=encoder_cfg.num_feedforward_networks, + normalization_type=encoder_cfg.normalization_type, + classifier_activation=encoder_cfg.classifier_activation, + input_mask_dtype=encoder_cfg.input_mask_dtype) + + if encoder_type == "albert": + return networks.AlbertEncoder( + vocab_size=encoder_cfg.vocab_size, + embedding_width=encoder_cfg.embedding_width, + hidden_size=encoder_cfg.hidden_size, + num_layers=encoder_cfg.num_layers, + num_attention_heads=encoder_cfg.num_attention_heads, + max_sequence_length=encoder_cfg.max_position_embeddings, + type_vocab_size=encoder_cfg.type_vocab_size, + intermediate_size=encoder_cfg.intermediate_size, + activation=tf_utils.get_activation(encoder_cfg.hidden_activation), + dropout_rate=encoder_cfg.dropout_rate, + attention_dropout_rate=encoder_cfg.attention_dropout_rate, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + dict_outputs=True) + + if encoder_type == "bigbird": + # TODO(frederickliu): Support use_gradient_checkpointing and update + # experiments to use the EncoderScaffold only. + if encoder_cfg.use_gradient_checkpointing: + return bigbird_encoder.BigBirdEncoder( + vocab_size=encoder_cfg.vocab_size, + hidden_size=encoder_cfg.hidden_size, + num_layers=encoder_cfg.num_layers, + num_attention_heads=encoder_cfg.num_attention_heads, + intermediate_size=encoder_cfg.intermediate_size, + activation=tf_utils.get_activation(encoder_cfg.hidden_activation), + dropout_rate=encoder_cfg.dropout_rate, + attention_dropout_rate=encoder_cfg.attention_dropout_rate, + num_rand_blocks=encoder_cfg.num_rand_blocks, + block_size=encoder_cfg.block_size, + max_position_embeddings=encoder_cfg.max_position_embeddings, + type_vocab_size=encoder_cfg.type_vocab_size, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + embedding_width=encoder_cfg.embedding_width, + use_gradient_checkpointing=encoder_cfg.use_gradient_checkpointing) + embedding_cfg = dict( + vocab_size=encoder_cfg.vocab_size, + type_vocab_size=encoder_cfg.type_vocab_size, + hidden_size=encoder_cfg.hidden_size, + max_seq_length=encoder_cfg.max_position_embeddings, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + dropout_rate=encoder_cfg.dropout_rate) + attention_cfg = dict( + num_heads=encoder_cfg.num_attention_heads, + key_dim=int(encoder_cfg.hidden_size // encoder_cfg.num_attention_heads), + kernel_initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + max_rand_mask_length=encoder_cfg.max_position_embeddings, + num_rand_blocks=encoder_cfg.num_rand_blocks, + from_block_size=encoder_cfg.block_size, + to_block_size=encoder_cfg.block_size, + ) + hidden_cfg = dict( + num_attention_heads=encoder_cfg.num_attention_heads, + intermediate_size=encoder_cfg.intermediate_size, + intermediate_activation=tf_utils.get_activation( + encoder_cfg.hidden_activation), + dropout_rate=encoder_cfg.dropout_rate, + attention_dropout_rate=encoder_cfg.attention_dropout_rate, + norm_first=encoder_cfg.norm_first, + kernel_initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + attention_cls=layers.BigBirdAttention, + attention_cfg=attention_cfg) + kwargs = dict( + embedding_cfg=embedding_cfg, + hidden_cls=layers.TransformerScaffold, + hidden_cfg=hidden_cfg, + num_hidden_instances=encoder_cfg.num_layers, + mask_cls=layers.BigBirdMasks, + mask_cfg=dict(block_size=encoder_cfg.block_size), + pooled_output_dim=encoder_cfg.hidden_size, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + return_all_layer_outputs=False, + dict_outputs=True, + layer_idx_as_attention_seed=True) + return networks.EncoderScaffold(**kwargs) + + if encoder_type == "kernel": + embedding_cfg = dict( + vocab_size=encoder_cfg.vocab_size, + type_vocab_size=encoder_cfg.type_vocab_size, + hidden_size=encoder_cfg.hidden_size, + max_seq_length=encoder_cfg.max_position_embeddings, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + dropout_rate=encoder_cfg.dropout_rate) + attention_cfg = dict( + num_heads=encoder_cfg.num_attention_heads, + key_dim=int(encoder_cfg.hidden_size // encoder_cfg.num_attention_heads), + kernel_initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + feature_transform=encoder_cfg.feature_transform, + num_random_features=encoder_cfg.num_random_features, + redraw=encoder_cfg.redraw, + is_short_seq=encoder_cfg.is_short_seq, + begin_kernel=encoder_cfg.begin_kernel, + scale=encoder_cfg.scale, + ) + hidden_cfg = dict( + num_attention_heads=encoder_cfg.num_attention_heads, + intermediate_size=encoder_cfg.intermediate_size, + intermediate_activation=tf_utils.get_activation( + encoder_cfg.hidden_activation), + dropout_rate=encoder_cfg.dropout_rate, + attention_dropout_rate=encoder_cfg.attention_dropout_rate, + norm_first=encoder_cfg.norm_first, + kernel_initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + attention_cls=layers.KernelAttention, + attention_cfg=attention_cfg) + kwargs = dict( + embedding_cfg=embedding_cfg, + hidden_cls=layers.TransformerScaffold, + hidden_cfg=hidden_cfg, + num_hidden_instances=encoder_cfg.num_layers, + mask_cls=layers.KernelMask, + pooled_output_dim=encoder_cfg.hidden_size, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + return_all_layer_outputs=False, + dict_outputs=True, + layer_idx_as_attention_seed=True) + return networks.EncoderScaffold(**kwargs) + + if encoder_type == "xlnet": + return networks.XLNetBase( + vocab_size=encoder_cfg.vocab_size, + num_layers=encoder_cfg.num_layers, + hidden_size=encoder_cfg.hidden_size, + num_attention_heads=encoder_cfg.num_attention_heads, + head_size=encoder_cfg.head_size, + inner_size=encoder_cfg.inner_size, + dropout_rate=encoder_cfg.dropout_rate, + attention_dropout_rate=encoder_cfg.attention_dropout_rate, + attention_type=encoder_cfg.attention_type, + bi_data=encoder_cfg.bi_data, + two_stream=encoder_cfg.two_stream, + tie_attention_biases=encoder_cfg.tie_attention_biases, + memory_length=encoder_cfg.memory_length, + clamp_length=encoder_cfg.clamp_length, + reuse_length=encoder_cfg.reuse_length, + inner_activation=encoder_cfg.inner_activation, + use_cls_mask=encoder_cfg.use_cls_mask, + embedding_width=encoder_cfg.embedding_width, + initializer=tf.keras.initializers.RandomNormal( + stddev=encoder_cfg.initializer_range)) + + if encoder_type == "teams": + embedding_cfg = dict( + vocab_size=encoder_cfg.vocab_size, + type_vocab_size=encoder_cfg.type_vocab_size, + hidden_size=encoder_cfg.hidden_size, + embedding_width=encoder_cfg.embedding_size, + max_seq_length=encoder_cfg.max_position_embeddings, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + dropout_rate=encoder_cfg.dropout_rate, + ) + embedding_network = networks.PackedSequenceEmbedding(**embedding_cfg) + hidden_cfg = dict( + num_attention_heads=encoder_cfg.num_attention_heads, + intermediate_size=encoder_cfg.intermediate_size, + intermediate_activation=tf_utils.get_activation( + encoder_cfg.hidden_activation), + dropout_rate=encoder_cfg.dropout_rate, + attention_dropout_rate=encoder_cfg.attention_dropout_rate, + kernel_initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + ) + kwargs = dict( + embedding_cfg=embedding_cfg, + embedding_cls=embedding_network, + hidden_cfg=hidden_cfg, + num_hidden_instances=encoder_cfg.num_layers, + pooled_output_dim=encoder_cfg.hidden_size, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + return_all_layer_outputs=encoder_cfg.return_all_encoder_outputs, + dict_outputs=True) + return networks.EncoderScaffold(**kwargs) + + if encoder_type == "reuse": + embedding_cfg = dict( + vocab_size=encoder_cfg.vocab_size, + type_vocab_size=encoder_cfg.type_vocab_size, + hidden_size=encoder_cfg.hidden_size, + max_seq_length=encoder_cfg.max_position_embeddings, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + dropout_rate=encoder_cfg.dropout_rate) + hidden_cfg = dict( + num_attention_heads=encoder_cfg.num_attention_heads, + inner_dim=encoder_cfg.intermediate_size, + inner_activation=tf_utils.get_activation( + encoder_cfg.hidden_activation), + output_dropout=encoder_cfg.dropout_rate, + attention_dropout=encoder_cfg.attention_dropout_rate, + norm_first=encoder_cfg.norm_first, + kernel_initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + reuse_attention=encoder_cfg.reuse_attention, + use_relative_pe=encoder_cfg.use_relative_pe, + pe_max_seq_length=encoder_cfg.pe_max_seq_length, + max_reuse_layer_idx=encoder_cfg.max_reuse_layer_idx) + kwargs = dict( + embedding_cfg=embedding_cfg, + hidden_cls=layers.ReuseTransformer, + hidden_cfg=hidden_cfg, + num_hidden_instances=encoder_cfg.num_layers, + pooled_output_dim=encoder_cfg.hidden_size, + pooler_layer_initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + return_all_layer_outputs=False, + dict_outputs=True, + feed_layer_idx=True, + recursive=True) + return networks.EncoderScaffold(**kwargs) + + bert_encoder_cls = networks.BertEncoder + if encoder_type == "bert_v2": + bert_encoder_cls = networks.BertEncoderV2 + + # Uses the default BERTEncoder configuration schema to create the encoder. + # If it does not match, please add a switch branch by the encoder type. + return bert_encoder_cls( + vocab_size=encoder_cfg.vocab_size, + hidden_size=encoder_cfg.hidden_size, + num_layers=encoder_cfg.num_layers, + num_attention_heads=encoder_cfg.num_attention_heads, + intermediate_size=encoder_cfg.intermediate_size, + activation=tf_utils.get_activation(encoder_cfg.hidden_activation), + dropout_rate=encoder_cfg.dropout_rate, + attention_dropout_rate=encoder_cfg.attention_dropout_rate, + max_sequence_length=encoder_cfg.max_position_embeddings, + type_vocab_size=encoder_cfg.type_vocab_size, + initializer=tf.keras.initializers.TruncatedNormal( + stddev=encoder_cfg.initializer_range), + output_range=encoder_cfg.output_range, + embedding_width=encoder_cfg.embedding_size, + embedding_layer=embedding_layer, + return_all_encoder_outputs=encoder_cfg.return_all_encoder_outputs, + dict_outputs=True, + norm_first=encoder_cfg.norm_first) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/encoders_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/encoders_test.py new file mode 100644 index 0000000000000000000000000000000000000000..1fe3b16e97846cb38c65c58c6f01d8c29cf3f246 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/encoders_test.py @@ -0,0 +1,42 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for official.nlp.configs.encoders.""" +import os + +import tensorflow as tf + +from official.modeling import hyperparams +from official.nlp.configs import encoders + + +class EncodersTest(tf.test.TestCase): + + def test_encoder_from_yaml(self): + config = encoders.EncoderConfig( + type="bert", bert=encoders.BertEncoderConfig(num_layers=1)) + encoder = encoders.build_encoder(config) + ckpt = tf.train.Checkpoint(encoder=encoder) + ckpt_path = ckpt.save(self.get_temp_dir() + "/ckpt") + params_save_path = os.path.join(self.get_temp_dir(), "params.yaml") + hyperparams.save_params_dict_to_yaml(config, params_save_path) + + retored_cfg = encoders.EncoderConfig.from_yaml(params_save_path) + retored_encoder = encoders.build_encoder(retored_cfg) + status = tf.train.Checkpoint(encoder=retored_encoder).restore(ckpt_path) + status.assert_consumed() + + +if __name__ == "__main__": + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/experiment_configs.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/experiment_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..2b52d5b4b7fda4bfd487310b0bd39a255117968a --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/experiment_configs.py @@ -0,0 +1,20 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Experiments definition.""" +# pylint: disable=unused-import +from official.nlp.configs import finetuning_experiments +from official.nlp.configs import pretraining_experiments +from official.nlp.configs import wmt_transformer_experiments +from official.nlp.projects.teams import teams_experiments diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/experiments/glue_mnli_matched.yaml b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/experiments/glue_mnli_matched.yaml new file mode 100644 index 0000000000000000000000000000000000000000..29dfcb68b9c314d309239c321dde4ec4f439da1d --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/experiments/glue_mnli_matched.yaml @@ -0,0 +1,49 @@ +task: + hub_module_url: '' + model: + num_classes: 3 + init_checkpoint: '' + metric_type: 'accuracy' + train_data: + drop_remainder: true + global_batch_size: 32 + input_path: '' + is_training: true + seq_length: 128 + label_type: 'int' + validation_data: + drop_remainder: false + global_batch_size: 32 + input_path: '' + is_training: false + seq_length: 128 + label_type: 'int' +trainer: + checkpoint_interval: 3000 + optimizer_config: + learning_rate: + polynomial: + # 100% of train_steps. + decay_steps: 36813 + end_learning_rate: 0.0 + initial_learning_rate: 3.0e-05 + power: 1.0 + type: polynomial + optimizer: + type: adamw + warmup: + polynomial: + power: 1 + # ~10% of train_steps. + warmup_steps: 3681 + type: polynomial + steps_per_loop: 1000 + summary_interval: 1000 + # Training data size 392,702 examples, 3 epochs. + train_steps: 36813 + validation_interval: 6135 + # Eval data size = 9815 examples. + validation_steps: 307 + best_checkpoint_export_subdir: 'best_ckpt' + best_checkpoint_eval_metric: 'cls_accuracy' + best_checkpoint_metric_comp: 'higher' diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/experiments/squad_v1.yaml b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/experiments/squad_v1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a69710a58f7dfa4e044bceb73c5870701ca39189 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/experiments/squad_v1.yaml @@ -0,0 +1,50 @@ +task: + hub_module_url: '' + max_answer_length: 30 + n_best_size: 20 + null_score_diff_threshold: 0.0 + init_checkpoint: '' + train_data: + drop_remainder: true + global_batch_size: 48 + input_path: '' + is_training: true + seq_length: 384 + validation_data: + do_lower_case: true + doc_stride: 128 + drop_remainder: false + global_batch_size: 48 + input_path: '' + is_training: false + query_length: 64 + seq_length: 384 + tokenization: WordPiece + version_2_with_negative: false + vocab_file: '' +trainer: + checkpoint_interval: 1000 + max_to_keep: 5 + optimizer_config: + learning_rate: + polynomial: + decay_steps: 3699 + end_learning_rate: 0.0 + initial_learning_rate: 8.0e-05 + power: 1.0 + type: polynomial + optimizer: + type: adamw + warmup: + polynomial: + power: 1 + warmup_steps: 370 + type: polynomial + steps_per_loop: 1000 + summary_interval: 1000 + train_steps: 3699 + validation_interval: 1000 + validation_steps: 226 + best_checkpoint_export_subdir: 'best_ckpt' + best_checkpoint_eval_metric: 'final_f1' + best_checkpoint_metric_comp: 'higher' diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/finetuning_experiments.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/finetuning_experiments.py new file mode 100644 index 0000000000000000000000000000000000000000..15ae860b2b5882ca0f56826bf56ea9c2a32a6d04 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/finetuning_experiments.py @@ -0,0 +1,137 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Finetuning experiment configurations.""" +# pylint: disable=g-doc-return-or-yield,line-too-long +from official.core import config_definitions as cfg +from official.core import exp_factory +from official.modeling import optimization +from official.nlp.data import question_answering_dataloader +from official.nlp.data import sentence_prediction_dataloader +from official.nlp.data import tagging_dataloader +from official.nlp.tasks import question_answering +from official.nlp.tasks import sentence_prediction +from official.nlp.tasks import tagging + + +@exp_factory.register_config_factory('bert/sentence_prediction') +def bert_sentence_prediction() -> cfg.ExperimentConfig: + r"""BERT GLUE.""" + config = cfg.ExperimentConfig( + task=sentence_prediction.SentencePredictionConfig( + train_data=sentence_prediction_dataloader + .SentencePredictionDataConfig(), + validation_data=sentence_prediction_dataloader + .SentencePredictionDataConfig( + is_training=False, drop_remainder=False)), + trainer=cfg.TrainerConfig( + optimizer_config=optimization.OptimizationConfig({ + 'optimizer': { + 'type': 'adamw', + 'adamw': { + 'weight_decay_rate': + 0.01, + 'exclude_from_weight_decay': + ['LayerNorm', 'layer_norm', 'bias'], + } + }, + 'learning_rate': { + 'type': 'polynomial', + 'polynomial': { + 'initial_learning_rate': 3e-5, + 'end_learning_rate': 0.0, + } + }, + 'warmup': { + 'type': 'polynomial' + } + })), + restrictions=[ + 'task.train_data.is_training != None', + 'task.validation_data.is_training != None' + ]) + return config + + +@exp_factory.register_config_factory('bert/squad') +def bert_squad() -> cfg.ExperimentConfig: + """BERT Squad V1/V2.""" + config = cfg.ExperimentConfig( + task=question_answering.QuestionAnsweringConfig( + train_data=question_answering_dataloader.QADataConfig(), + validation_data=question_answering_dataloader.QADataConfig()), + trainer=cfg.TrainerConfig( + optimizer_config=optimization.OptimizationConfig({ + 'optimizer': { + 'type': 'adamw', + 'adamw': { + 'weight_decay_rate': + 0.01, + 'exclude_from_weight_decay': + ['LayerNorm', 'layer_norm', 'bias'], + } + }, + 'learning_rate': { + 'type': 'polynomial', + 'polynomial': { + 'initial_learning_rate': 8e-5, + 'end_learning_rate': 0.0, + } + }, + 'warmup': { + 'type': 'polynomial' + } + })), + restrictions=[ + 'task.train_data.is_training != None', + 'task.validation_data.is_training != None' + ]) + return config + + +@exp_factory.register_config_factory('bert/tagging') +def bert_tagging() -> cfg.ExperimentConfig: + """BERT tagging task.""" + config = cfg.ExperimentConfig( + task=tagging.TaggingConfig( + train_data=tagging_dataloader.TaggingDataConfig(), + validation_data=tagging_dataloader.TaggingDataConfig( + is_training=False, drop_remainder=False)), + trainer=cfg.TrainerConfig( + optimizer_config=optimization.OptimizationConfig({ + 'optimizer': { + 'type': 'adamw', + 'adamw': { + 'weight_decay_rate': + 0.01, + 'exclude_from_weight_decay': + ['LayerNorm', 'layer_norm', 'bias'], + } + }, + 'learning_rate': { + 'type': 'polynomial', + 'polynomial': { + 'initial_learning_rate': 8e-5, + 'end_learning_rate': 0.0, + } + }, + 'warmup': { + 'type': 'polynomial' + } + })), + restrictions=[ + 'task.train_data.is_training != None', + 'task.validation_data.is_training != None', + ]) + return config diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/models/albert_base.yaml b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/models/albert_base.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d1317b2f4113de671967f0dc94b848653c544c9 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/models/albert_base.yaml @@ -0,0 +1,16 @@ +task: + model: + encoder: + type: albert + albert: + attention_dropout_rate: 0.0 + dropout_rate: 0.0 + hidden_activation: gelu + hidden_size: 768 + initializer_range: 0.02 + intermediate_size: 3072 + max_position_embeddings: 512 + num_attention_heads: 12 + num_layers: 12 + type_vocab_size: 2 + vocab_size: 30000 diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/models/bert_en_uncased_base.yaml b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/models/bert_en_uncased_base.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e49bc5430ed0135aa6d981421aad623f4f1fac9 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/models/bert_en_uncased_base.yaml @@ -0,0 +1,16 @@ +task: + model: + encoder: + type: bert + bert: + attention_dropout_rate: 0.1 + dropout_rate: 0.1 + hidden_activation: gelu + hidden_size: 768 + initializer_range: 0.02 + intermediate_size: 3072 + max_position_embeddings: 512 + num_attention_heads: 12 + num_layers: 12 + type_vocab_size: 2 + vocab_size: 30522 diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/pretraining_experiments.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/pretraining_experiments.py new file mode 100644 index 0000000000000000000000000000000000000000..024c6fcfb281a467ceaaffa1cdbdf07fdae5a95a --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/pretraining_experiments.py @@ -0,0 +1,84 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pretraining experiment configurations.""" +# pylint: disable=g-doc-return-or-yield,line-too-long +from official.core import config_definitions as cfg +from official.core import exp_factory +from official.modeling import optimization +from official.nlp.data import pretrain_dataloader +from official.nlp.data import pretrain_dynamic_dataloader +from official.nlp.tasks import masked_lm + +_TRAINER = cfg.TrainerConfig( + train_steps=1000000, + optimizer_config=optimization.OptimizationConfig({ + 'optimizer': { + 'type': 'adamw', + 'adamw': { + 'weight_decay_rate': + 0.01, + 'exclude_from_weight_decay': [ + 'LayerNorm', 'layer_norm', 'bias' + ], + } + }, + 'learning_rate': { + 'type': 'polynomial', + 'polynomial': { + 'initial_learning_rate': 1e-4, + 'end_learning_rate': 0.0, + } + }, + 'warmup': { + 'type': 'polynomial' + } + })) + + +@exp_factory.register_config_factory('bert/pretraining') +def bert_pretraining() -> cfg.ExperimentConfig: + """BERT pretraining experiment.""" + config = cfg.ExperimentConfig( + runtime=cfg.RuntimeConfig(enable_xla=True), + task=masked_lm.MaskedLMConfig( + train_data=pretrain_dataloader.BertPretrainDataConfig(), + validation_data=pretrain_dataloader.BertPretrainDataConfig( + is_training=False)), + trainer=_TRAINER, + restrictions=[ + 'task.train_data.is_training != None', + 'task.validation_data.is_training != None' + ]) + return config + + +@exp_factory.register_config_factory('bert/pretraining_dynamic') +def bert_dynamic() -> cfg.ExperimentConfig: + """BERT base with dynamic input sequences. + + TPU needs to run with tf.data service with round-robin behavior. + """ + config = cfg.ExperimentConfig( + runtime=cfg.RuntimeConfig(enable_xla=True), + task=masked_lm.MaskedLMConfig( + train_data=pretrain_dynamic_dataloader.BertPretrainDataConfig(), + validation_data=pretrain_dataloader.BertPretrainDataConfig( + is_training=False)), + trainer=_TRAINER, + restrictions=[ + 'task.train_data.is_training != None', + 'task.validation_data.is_training != None' + ]) + return config diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/wmt_transformer_experiments.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/wmt_transformer_experiments.py new file mode 100644 index 0000000000000000000000000000000000000000..eb85b76c5a94505de9c4e7e2e11a563abce5a645 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/configs/wmt_transformer_experiments.py @@ -0,0 +1,111 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +# pylint: disable=g-doc-return-or-yield,line-too-long +"""WMT translation configurations.""" + +from official.core import config_definitions as cfg +from official.core import exp_factory +from official.modeling import optimization +from official.nlp.data import wmt_dataloader +from official.nlp.tasks import translation + + +@exp_factory.register_config_factory('wmt_transformer/large') +def wmt_transformer_large() -> cfg.ExperimentConfig: + """WMT Transformer Large. + + Please refer to + tensorflow_models/official/nlp/data/train_sentencepiece.py + to generate sentencepiece_model + and pass + --params_override=task.sentencepiece_model_path='YOUR_PATH' + to the train script. + """ + learning_rate = 2.0 + hidden_size = 1024 + learning_rate *= (hidden_size**-0.5) + warmup_steps = 16000 + train_steps = 300000 + token_batch_size = 24576 + encdecoder = translation.EncDecoder( + num_attention_heads=16, intermediate_size=hidden_size * 4) + config = cfg.ExperimentConfig( + runtime=cfg.RuntimeConfig(enable_xla=True), + task=translation.TranslationConfig( + model=translation.ModelConfig( + encoder=encdecoder, + decoder=encdecoder, + embedding_width=hidden_size, + padded_decode=True, + decode_max_length=100), + train_data=wmt_dataloader.WMTDataConfig( + tfds_name='wmt14_translate/de-en', + tfds_split='train', + src_lang='en', + tgt_lang='de', + is_training=True, + global_batch_size=token_batch_size, + static_batch=True, + max_seq_length=64 + ), + validation_data=wmt_dataloader.WMTDataConfig( + tfds_name='wmt14_translate/de-en', + tfds_split='test', + src_lang='en', + tgt_lang='de', + is_training=False, + global_batch_size=32, + static_batch=True, + max_seq_length=100, + ), + sentencepiece_model_path=None, + ), + trainer=cfg.TrainerConfig( + train_steps=train_steps, + validation_steps=-1, + steps_per_loop=1000, + summary_interval=1000, + checkpoint_interval=5000, + validation_interval=5000, + max_to_keep=1, + optimizer_config=optimization.OptimizationConfig({ + 'optimizer': { + 'type': 'adam', + 'adam': { + 'beta_2': 0.997, + 'epsilon': 1e-9, + }, + }, + 'learning_rate': { + 'type': 'power', + 'power': { + 'initial_learning_rate': learning_rate, + 'power': -0.5, + } + }, + 'warmup': { + 'type': 'linear', + 'linear': { + 'warmup_steps': warmup_steps, + 'warmup_learning_rate': 0.0 + } + } + })), + restrictions=[ + 'task.train_data.is_training != None', + 'task.sentencepiece_model_path != None', + ]) + return config diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/continuous_finetune_lib.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/continuous_finetune_lib.py new file mode 100644 index 0000000000000000000000000000000000000000..6fe851741c0f631ea18f80b9bf259c551e7f2561 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/continuous_finetune_lib.py @@ -0,0 +1,217 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TFM continuous finetuning+eval training driver library.""" +import gc +import os +import time +from typing import Any, Mapping, Optional + +from absl import logging +import tensorflow as tf + +from official.common import distribute_utils +from official.core import config_definitions +from official.core import task_factory +from official.core import train_lib +from official.core import train_utils +from official.modeling import performance +from official.modeling.multitask import configs +from official.modeling.multitask import train_lib as multitask_train_lib + + +def _flatten_dict(xs): + """Flatten a nested dictionary. + + The nested keys are flattened to a tuple. + + Example:: + + xs = {'foo': 1, 'bar': {'a': 2, 'b': {}}} + flat_xs = flatten_dict(xs) + print(flat_xs) + # { + # ('foo',): 1, + # ('bar', 'a'): 2, + # } + + Note that empty dictionaries are ignored and + will not be restored by `unflatten_dict`. + + Args: + xs: a nested dictionary + + Returns: + The flattened dictionary. + """ + assert isinstance(xs, dict), 'input is not a dict' + + def _flatten(xs, prefix): + if not isinstance(xs, dict): + return {prefix: xs} + result = {} + for key, value in xs.items(): + path = prefix + (key,) + result.update(_flatten(value, path)) + return result + + return _flatten(xs, ()) + + +def run_continuous_finetune( + mode: str, + params: config_definitions.ExperimentConfig, + model_dir: str, + run_post_eval: bool = False, + pretrain_steps: Optional[int] = None, +) -> Mapping[str, Any]: + """Run modes with continuous training. + + Currently only supports continuous_train_and_eval. + + Args: + mode: A 'str', specifying the mode. continuous_train_and_eval - monitors a + checkpoint directory. Once a new checkpoint is discovered, loads the + checkpoint, finetune the model by training it (probably on another dataset + or with another task), then evaluate the finetuned model. + params: ExperimentConfig instance. + model_dir: A 'str', a path to store model checkpoints and summaries. + run_post_eval: Whether to run post eval once after training, metrics logs + are returned. + pretrain_steps: Optional, the number of total training steps for the + pretraining job. + + Returns: + eval logs: returns eval metrics logs when run_post_eval is set to True, + othewise, returns {}. + """ + + assert mode == 'continuous_train_and_eval', ( + 'Only continuous_train_and_eval is supported by continuous_finetune. ' + 'Got mode: {}'.format(mode)) + + # Sets mixed_precision policy. Using 'mixed_float16' or 'mixed_bfloat16' + # can have significant impact on model speeds by utilizing float16 in case of + # GPUs, and bfloat16 in the case of TPUs. loss_scale takes effect only when + # dtype is float16 + if params.runtime.mixed_precision_dtype: + performance.set_mixed_precision_policy(params.runtime.mixed_precision_dtype) + distribution_strategy = distribute_utils.get_distribution_strategy( + distribution_strategy=params.runtime.distribution_strategy, + all_reduce_alg=params.runtime.all_reduce_alg, + num_gpus=params.runtime.num_gpus, + tpu_address=params.runtime.tpu) + + retry_times = 0 + while not tf.io.gfile.isdir(params.task.init_checkpoint): + # Wait for the init_checkpoint directory to be created. + if retry_times >= 60: + raise ValueError( + 'ExperimentConfig.task.init_checkpoint must be a directory for ' + 'continuous_train_and_eval mode.') + retry_times += 1 + time.sleep(60) + + summary_writer = tf.summary.create_file_writer( + os.path.join(model_dir, 'eval')) + + global_step = 0 + + def timeout_fn(): + if pretrain_steps and global_step < pretrain_steps: + # Keeps waiting for another timeout period. + logging.info( + 'Continue waiting for new checkpoint as current pretrain ' + 'global_step=%d and target is %d.', global_step, pretrain_steps) + return False + # Quits the loop. + return True + + for pretrain_ckpt in tf.train.checkpoints_iterator( + checkpoint_dir=params.task.init_checkpoint, + min_interval_secs=10, + timeout=params.trainer.continuous_eval_timeout, + timeout_fn=timeout_fn): + + # If there are checkpoints, they might be the finetune checkpoint of a + # different pretrained checkpoint. So we just remove all checkpoints. + train_utils.remove_ckpts(model_dir) + + with distribution_strategy.scope(): + global_step = train_utils.read_global_step_from_checkpoint(pretrain_ckpt) + # Replaces params.task.init_checkpoint to make sure that we load + # exactly this pretrain checkpoint. + if params.trainer.best_checkpoint_export_subdir: + best_ckpt_subdir = '{}_{}'.format( + params.trainer.best_checkpoint_export_subdir, global_step) + params_replaced = params.replace( + task={'init_checkpoint': pretrain_ckpt}, + trainer={'best_checkpoint_export_subdir': best_ckpt_subdir}) + else: + params_replaced = params.replace(task={'init_checkpoint': pretrain_ckpt}) + params_replaced.lock() + logging.info('Running finetuning with params: %s', params_replaced) + + with distribution_strategy.scope(): + if isinstance(params, configs.MultiEvalExperimentConfig): + task = task_factory.get_task(params_replaced.task) + eval_tasks = [ + task_factory.get_task(config.task_config, name=config.task_name) + for config in params.eval_tasks + ] + (_, + eval_metrics) = multitask_train_lib.run_experiment_with_multitask_eval( + distribution_strategy=distribution_strategy, + train_task=task, + eval_tasks=eval_tasks, + mode='train_and_eval', + params=params_replaced, + model_dir=model_dir, + run_post_eval=True, + save_summary=False) + else: + task = task_factory.get_task( + params_replaced.task, logging_dir=model_dir) + _, eval_metrics = train_lib.run_experiment( + distribution_strategy=distribution_strategy, + task=task, + mode='train_and_eval', + params=params_replaced, + model_dir=model_dir, + run_post_eval=True, + save_summary=False) + logging.info('Evaluation finished. Pretrain global_step: %d', global_step) + train_utils.write_json_summary(model_dir, global_step, eval_metrics) + + if not os.path.basename(model_dir): # if model_dir.endswith('/') + summary_grp = os.path.dirname(model_dir) + '_' + task.name + else: + summary_grp = os.path.basename(model_dir) + '_' + task.name + summaries = {} + for name, value in _flatten_dict(eval_metrics).items(): + summaries[summary_grp + '/' + '-'.join(name)] = value + train_utils.write_summary(summary_writer, global_step, summaries) + + train_utils.remove_ckpts(model_dir) + # In TF2, the resource life cycle is bound with the python object life + # cycle. Force trigger python garbage collection here so those resources + # can be deallocated in time, so it doesn't cause OOM when allocating new + # objects. + # TODO(b/169178664): Fix cycle reference in Keras model and revisit to see + # if we need gc here. + gc.collect() + + if run_post_eval: + return eval_metrics + return {} diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/continuous_finetune_lib_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/continuous_finetune_lib_test.py new file mode 100644 index 0000000000000000000000000000000000000000..08ee381dce133d73e18e697b938cab92d04f2ff0 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/continuous_finetune_lib_test.py @@ -0,0 +1,98 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from absl import flags +from absl.testing import flagsaver +from absl.testing import parameterized +import tensorflow as tf + +# pylint: disable=unused-import +from official.common import registry_imports +# pylint: enable=unused-import +from official.common import flags as tfm_flags +from official.core import task_factory +from official.core import train_lib +from official.core import train_utils +from official.nlp import continuous_finetune_lib + +FLAGS = flags.FLAGS + +tfm_flags.define_flags() + + +class ContinuousFinetuneTest(tf.test.TestCase, parameterized.TestCase): + + def setUp(self): + super().setUp() + self._model_dir = os.path.join(self.get_temp_dir(), 'model_dir') + + def testContinuousFinetune(self): + pretrain_steps = 1 + src_model_dir = self.get_temp_dir() + flags_dict = dict( + experiment='mock', + mode='continuous_train_and_eval', + model_dir=self._model_dir, + params_override={ + 'task': { + 'init_checkpoint': src_model_dir, + }, + 'trainer': { + 'continuous_eval_timeout': 1, + 'steps_per_loop': 1, + 'train_steps': 1, + 'validation_steps': 1, + 'best_checkpoint_export_subdir': 'best_ckpt', + 'best_checkpoint_eval_metric': 'acc', + 'optimizer_config': { + 'optimizer': { + 'type': 'sgd' + }, + 'learning_rate': { + 'type': 'constant' + } + } + } + }) + + with flagsaver.flagsaver(**flags_dict): + # Train and save some checkpoints. + params = train_utils.parse_configuration(flags.FLAGS) + distribution_strategy = tf.distribute.get_strategy() + with distribution_strategy.scope(): + task = task_factory.get_task(params.task, logging_dir=src_model_dir) + _ = train_lib.run_experiment( + distribution_strategy=distribution_strategy, + task=task, + mode='train', + params=params, + model_dir=src_model_dir) + + params = train_utils.parse_configuration(FLAGS) + eval_metrics = continuous_finetune_lib.run_continuous_finetune( + FLAGS.mode, + params, + FLAGS.model_dir, + run_post_eval=True, + pretrain_steps=pretrain_steps) + self.assertIn('best_acc', eval_metrics) + + self.assertFalse( + tf.io.gfile.exists(os.path.join(FLAGS.model_dir, 'checkpoint'))) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/__init__.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e419af524b5f349fe04abfa820c3cb51b777d422 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/classifier_data_lib.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/classifier_data_lib.py new file mode 100644 index 0000000000000000000000000000000000000000..0ba9dcf9a055ab9a3e5206f251d45d2ea41a2661 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/classifier_data_lib.py @@ -0,0 +1,1592 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""BERT library to process data for classification task.""" + +import collections +import csv +import importlib +import json +import os + +from absl import logging +import tensorflow as tf +import tensorflow_datasets as tfds + +from official.nlp.bert import tokenization + + +class InputExample(object): + """A single training/test example for simple seq regression/classification.""" + + def __init__(self, + guid, + text_a, + text_b=None, + label=None, + weight=None, + example_id=None): + """Constructs a InputExample. + + Args: + guid: Unique id for the example. + text_a: string. The untokenized text of the first sequence. For single + sequence tasks, only this sequence must be specified. + text_b: (Optional) string. The untokenized text of the second sequence. + Only must be specified for sequence pair tasks. + label: (Optional) string for classification, float for regression. The + label of the example. This should be specified for train and dev + examples, but not for test examples. + weight: (Optional) float. The weight of the example to be used during + training. + example_id: (Optional) int. The int identification number of example in + the corpus. + """ + self.guid = guid + self.text_a = text_a + self.text_b = text_b + self.label = label + self.weight = weight + self.example_id = example_id + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, + input_ids, + input_mask, + segment_ids, + label_id, + is_real_example=True, + weight=None, + example_id=None): + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.label_id = label_id + self.is_real_example = is_real_example + self.weight = weight + self.example_id = example_id + + +class DataProcessor(object): + """Base class for converters for seq regression/classification datasets.""" + + def __init__(self, process_text_fn=tokenization.convert_to_unicode): + self.process_text_fn = process_text_fn + self.is_regression = False + self.label_type = None + + def get_train_examples(self, data_dir): + """Gets a collection of `InputExample`s for the train set.""" + raise NotImplementedError() + + def get_dev_examples(self, data_dir): + """Gets a collection of `InputExample`s for the dev set.""" + raise NotImplementedError() + + def get_test_examples(self, data_dir): + """Gets a collection of `InputExample`s for prediction.""" + raise NotImplementedError() + + def get_labels(self): + """Gets the list of labels for this data set.""" + raise NotImplementedError() + + @staticmethod + def get_processor_name(): + """Gets the string identifier of the processor.""" + raise NotImplementedError() + + @classmethod + def _read_tsv(cls, input_file, quotechar=None): + """Reads a tab separated value file.""" + with tf.io.gfile.GFile(input_file, "r") as f: + reader = csv.reader(f, delimiter="\t", quotechar=quotechar) + lines = [] + for line in reader: + lines.append(line) + return lines + + @classmethod + def _read_jsonl(cls, input_file): + """Reads a json line file.""" + with tf.io.gfile.GFile(input_file, "r") as f: + lines = [] + for json_str in f: + lines.append(json.loads(json_str)) + return lines + + def featurize_example(self, *kargs, **kwargs): + """Converts a single `InputExample` into a single `InputFeatures`.""" + return convert_single_example(*kargs, **kwargs) + + +class DefaultGLUEDataProcessor(DataProcessor): + """Processor for the SuperGLUE dataset.""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples_tfds("train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples_tfds("validation") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples_tfds("test") + + def _create_examples_tfds(self, set_type): + """Creates examples for the training/dev/test sets.""" + raise NotImplementedError() + + +class AxProcessor(DataProcessor): + """Processor for the AX dataset (GLUE diagnostics dataset).""" + + def get_train_examples(self, data_dir): + """See base class.""" + train_mnli_dataset = tfds.load( + "glue/mnli", split="train", try_gcs=True).as_numpy_iterator() + return self._create_examples_tfds(train_mnli_dataset, "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + val_mnli_dataset = tfds.load( + "glue/mnli", split="validation_matched", + try_gcs=True).as_numpy_iterator() + return self._create_examples_tfds(val_mnli_dataset, "validation") + + def get_test_examples(self, data_dir): + """See base class.""" + test_ax_dataset = tfds.load( + "glue/ax", split="test", try_gcs=True).as_numpy_iterator() + return self._create_examples_tfds(test_ax_dataset, "test") + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "AX" + + def _create_examples_tfds(self, dataset, set_type): + """Creates examples for the training/dev/test sets.""" + examples = [] + for i, example in enumerate(dataset): + guid = "%s-%s" % (set_type, i) + label = "contradiction" + text_a = self.process_text_fn(example["hypothesis"]) + text_b = self.process_text_fn(example["premise"]) + if set_type != "test": + label = self.get_labels()[example["label"]] + examples.append( + InputExample( + guid=guid, text_a=text_a, text_b=text_b, label=label, + weight=None)) + return examples + + +class ColaProcessor(DefaultGLUEDataProcessor): + """Processor for the CoLA data set (GLUE version).""" + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "COLA" + + def _create_examples_tfds(self, set_type): + """Creates examples for the training/dev/test sets.""" + dataset = tfds.load( + "glue/cola", split=set_type, try_gcs=True).as_numpy_iterator() + examples = [] + for i, example in enumerate(dataset): + guid = "%s-%s" % (set_type, i) + label = "0" + text_a = self.process_text_fn(example["sentence"]) + if set_type != "test": + label = str(example["label"]) + examples.append( + InputExample( + guid=guid, text_a=text_a, text_b=None, label=label, weight=None)) + return examples + + +class ImdbProcessor(DataProcessor): + """Processor for the IMDb dataset.""" + + def get_labels(self): + return ["neg", "pos"] + + def get_train_examples(self, data_dir): + return self._create_examples(os.path.join(data_dir, "train")) + + def get_dev_examples(self, data_dir): + return self._create_examples(os.path.join(data_dir, "test")) + + @staticmethod + def get_processor_name(): + """See base class.""" + return "IMDB" + + def _create_examples(self, data_dir): + """Creates examples.""" + examples = [] + for label in ["neg", "pos"]: + cur_dir = os.path.join(data_dir, label) + for filename in tf.io.gfile.listdir(cur_dir): + if not filename.endswith("txt"): + continue + + if len(examples) % 1000 == 0: + logging.info("Loading dev example %d", len(examples)) + + path = os.path.join(cur_dir, filename) + with tf.io.gfile.GFile(path, "r") as f: + text = f.read().strip().replace("
", " ") + examples.append( + InputExample( + guid="unused_id", text_a=text, text_b=None, label=label)) + return examples + + +class MnliProcessor(DataProcessor): + """Processor for the MultiNLI data set (GLUE version).""" + + def __init__(self, + mnli_type="matched", + process_text_fn=tokenization.convert_to_unicode): + super(MnliProcessor, self).__init__(process_text_fn) + self.dataset = tfds.load("glue/mnli", try_gcs=True) + if mnli_type not in ("matched", "mismatched"): + raise ValueError("Invalid `mnli_type`: %s" % mnli_type) + self.mnli_type = mnli_type + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples_tfds("train") + + def get_dev_examples(self, data_dir): + """See base class.""" + if self.mnli_type == "matched": + return self._create_examples_tfds("validation_matched") + else: + return self._create_examples_tfds("validation_mismatched") + + def get_test_examples(self, data_dir): + """See base class.""" + if self.mnli_type == "matched": + return self._create_examples_tfds("test_matched") + else: + return self._create_examples_tfds("test_mismatched") + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "MNLI" + + def _create_examples_tfds(self, set_type): + """Creates examples for the training/dev/test sets.""" + dataset = tfds.load( + "glue/mnli", split=set_type, try_gcs=True).as_numpy_iterator() + examples = [] + for i, example in enumerate(dataset): + guid = "%s-%s" % (set_type, i) + label = "contradiction" + text_a = self.process_text_fn(example["hypothesis"]) + text_b = self.process_text_fn(example["premise"]) + if set_type != "test": + label = self.get_labels()[example["label"]] + examples.append( + InputExample( + guid=guid, text_a=text_a, text_b=text_b, label=label, + weight=None)) + return examples + + +class MrpcProcessor(DefaultGLUEDataProcessor): + """Processor for the MRPC data set (GLUE version).""" + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "MRPC" + + def _create_examples_tfds(self, set_type): + """Creates examples for the training/dev/test sets.""" + dataset = tfds.load( + "glue/mrpc", split=set_type, try_gcs=True).as_numpy_iterator() + examples = [] + for i, example in enumerate(dataset): + guid = "%s-%s" % (set_type, i) + label = "0" + text_a = self.process_text_fn(example["sentence1"]) + text_b = self.process_text_fn(example["sentence2"]) + if set_type != "test": + label = str(example["label"]) + examples.append( + InputExample( + guid=guid, text_a=text_a, text_b=text_b, label=label, + weight=None)) + return examples + + +class PawsxProcessor(DataProcessor): + """Processor for the PAWS-X data set.""" + supported_languages = ["de", "en", "es", "fr", "ja", "ko", "zh"] + + def __init__(self, + language="en", + process_text_fn=tokenization.convert_to_unicode): + super(PawsxProcessor, self).__init__(process_text_fn) + if language == "all": + self.languages = PawsxProcessor.supported_languages + elif language not in PawsxProcessor.supported_languages: + raise ValueError("language %s is not supported for PAWS-X task." % + language) + else: + self.languages = [language] + + def get_train_examples(self, data_dir): + """See base class.""" + lines = [] + for language in self.languages: + if language == "en": + train_tsv = "train.tsv" + else: + train_tsv = "translated_train.tsv" + # Skips the header. + lines.extend( + self._read_tsv(os.path.join(data_dir, language, train_tsv))[1:]) + + examples = [] + for i, line in enumerate(lines): + guid = "train-%d" % i + text_a = self.process_text_fn(line[1]) + text_b = self.process_text_fn(line[2]) + label = self.process_text_fn(line[3]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_dev_examples(self, data_dir): + """See base class.""" + lines = [] + for lang in PawsxProcessor.supported_languages: + lines.extend( + self._read_tsv(os.path.join(data_dir, lang, "dev_2k.tsv"))[1:]) + + examples = [] + for i, line in enumerate(lines): + guid = "dev-%d" % i + text_a = self.process_text_fn(line[1]) + text_b = self.process_text_fn(line[2]) + label = self.process_text_fn(line[3]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_test_examples(self, data_dir): + """See base class.""" + examples_by_lang = {k: [] for k in self.supported_languages} + for lang in self.supported_languages: + lines = self._read_tsv(os.path.join(data_dir, lang, "test_2k.tsv"))[1:] + for i, line in enumerate(lines): + guid = "test-%d" % i + text_a = self.process_text_fn(line[1]) + text_b = self.process_text_fn(line[2]) + label = self.process_text_fn(line[3]) + examples_by_lang[lang].append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples_by_lang + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "XTREME-PAWS-X" + + +class QnliProcessor(DefaultGLUEDataProcessor): + """Processor for the QNLI data set (GLUE version).""" + + def get_labels(self): + """See base class.""" + return ["entailment", "not_entailment"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "QNLI" + + def _create_examples_tfds(self, set_type): + """Creates examples for the training/dev/test sets.""" + dataset = tfds.load( + "glue/qnli", split=set_type, try_gcs=True).as_numpy_iterator() + examples = [] + for i, example in enumerate(dataset): + guid = "%s-%s" % (set_type, i) + label = "entailment" + text_a = self.process_text_fn(example["question"]) + text_b = self.process_text_fn(example["sentence"]) + if set_type != "test": + label = self.get_labels()[example["label"]] + examples.append( + InputExample( + guid=guid, text_a=text_a, text_b=text_b, label=label, + weight=None)) + return examples + + +class QqpProcessor(DefaultGLUEDataProcessor): + """Processor for the QQP data set (GLUE version).""" + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "QQP" + + def _create_examples_tfds(self, set_type): + """Creates examples for the training/dev/test sets.""" + dataset = tfds.load( + "glue/qqp", split=set_type, try_gcs=True).as_numpy_iterator() + examples = [] + for i, example in enumerate(dataset): + guid = "%s-%s" % (set_type, i) + label = "0" + text_a = self.process_text_fn(example["question1"]) + text_b = self.process_text_fn(example["question2"]) + if set_type != "test": + label = str(example["label"]) + examples.append( + InputExample( + guid=guid, text_a=text_a, text_b=text_b, label=label, + weight=None)) + return examples + + +class RteProcessor(DefaultGLUEDataProcessor): + """Processor for the RTE data set (GLUE version).""" + + def get_labels(self): + """See base class.""" + # All datasets are converted to 2-class split, where for 3-class datasets we + # collapse neutral and contradiction into not_entailment. + return ["entailment", "not_entailment"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "RTE" + + def _create_examples_tfds(self, set_type): + """Creates examples for the training/dev/test sets.""" + dataset = tfds.load( + "glue/rte", split=set_type, try_gcs=True).as_numpy_iterator() + examples = [] + for i, example in enumerate(dataset): + guid = "%s-%s" % (set_type, i) + label = "entailment" + text_a = self.process_text_fn(example["sentence1"]) + text_b = self.process_text_fn(example["sentence2"]) + if set_type != "test": + label = self.get_labels()[example["label"]] + examples.append( + InputExample( + guid=guid, text_a=text_a, text_b=text_b, label=label, + weight=None)) + return examples + + +class SstProcessor(DefaultGLUEDataProcessor): + """Processor for the SST-2 data set (GLUE version).""" + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "SST-2" + + def _create_examples_tfds(self, set_type): + """Creates examples for the training/dev/test sets.""" + dataset = tfds.load( + "glue/sst2", split=set_type, try_gcs=True).as_numpy_iterator() + examples = [] + for i, example in enumerate(dataset): + guid = "%s-%s" % (set_type, i) + label = "0" + text_a = self.process_text_fn(example["sentence"]) + if set_type != "test": + label = str(example["label"]) + examples.append( + InputExample( + guid=guid, text_a=text_a, text_b=None, label=label, weight=None)) + return examples + + +class StsBProcessor(DefaultGLUEDataProcessor): + """Processor for the STS-B data set (GLUE version).""" + + def __init__(self, process_text_fn=tokenization.convert_to_unicode): + super(StsBProcessor, self).__init__(process_text_fn=process_text_fn) + self.is_regression = True + self.label_type = float + self._labels = None + + def _create_examples_tfds(self, set_type): + """Creates examples for the training/dev/test sets.""" + dataset = tfds.load( + "glue/stsb", split=set_type, try_gcs=True).as_numpy_iterator() + examples = [] + for i, example in enumerate(dataset): + guid = "%s-%s" % (set_type, i) + label = 0.0 + text_a = self.process_text_fn(example["sentence1"]) + text_b = self.process_text_fn(example["sentence2"]) + if set_type != "test": + label = self.label_type(example["label"]) + examples.append( + InputExample( + guid=guid, text_a=text_a, text_b=text_b, label=label, + weight=None)) + return examples + + def get_labels(self): + """See base class.""" + return self._labels + + @staticmethod + def get_processor_name(): + """See base class.""" + return "STS-B" + + +class TfdsProcessor(DataProcessor): + """Processor for generic text classification and regression TFDS data set. + + The TFDS parameters are expected to be provided in the tfds_params string, in + a comma-separated list of parameter assignments. + Examples: + tfds_params="dataset=scicite,text_key=string" + tfds_params="dataset=imdb_reviews,test_split=,dev_split=test" + tfds_params="dataset=glue/cola,text_key=sentence" + tfds_params="dataset=glue/sst2,text_key=sentence" + tfds_params="dataset=glue/qnli,text_key=question,text_b_key=sentence" + tfds_params="dataset=glue/mrpc,text_key=sentence1,text_b_key=sentence2" + tfds_params="dataset=glue/stsb,text_key=sentence1,text_b_key=sentence2," + "is_regression=true,label_type=float" + tfds_params="dataset=snli,text_key=premise,text_b_key=hypothesis," + "skip_label=-1" + Possible parameters (please refer to the documentation of Tensorflow Datasets + (TFDS) for the meaning of individual parameters): + dataset: Required dataset name (potentially with subset and version number). + data_dir: Optional TFDS source root directory. + module_import: Optional Dataset module to import. + train_split: Name of the train split (defaults to `train`). + dev_split: Name of the dev split (defaults to `validation`). + test_split: Name of the test split (defaults to `test`). + text_key: Key of the text_a feature (defaults to `text`). + text_b_key: Key of the second text feature if available. + label_key: Key of the label feature (defaults to `label`). + test_text_key: Key of the text feature to use in test set. + test_text_b_key: Key of the second text feature to use in test set. + test_label: String to be used as the label for all test examples. + label_type: Type of the label key (defaults to `int`). + weight_key: Key of the float sample weight (is not used if not provided). + is_regression: Whether the task is a regression problem (defaults to False). + skip_label: Skip examples with given label (defaults to None). + """ + + def __init__(self, + tfds_params, + process_text_fn=tokenization.convert_to_unicode): + super(TfdsProcessor, self).__init__(process_text_fn) + self._process_tfds_params_str(tfds_params) + if self.module_import: + importlib.import_module(self.module_import) + + self.dataset, info = tfds.load( + self.dataset_name, data_dir=self.data_dir, with_info=True) + if self.is_regression: + self._labels = None + else: + self._labels = list(range(info.features[self.label_key].num_classes)) + + def _process_tfds_params_str(self, params_str): + """Extracts TFDS parameters from a comma-separated assignements string.""" + dtype_map = {"int": int, "float": float} + cast_str_to_bool = lambda s: s.lower() not in ["false", "0"] + + tuples = [x.split("=") for x in params_str.split(",")] + d = {k.strip(): v.strip() for k, v in tuples} + self.dataset_name = d["dataset"] # Required. + self.data_dir = d.get("data_dir", None) + self.module_import = d.get("module_import", None) + self.train_split = d.get("train_split", "train") + self.dev_split = d.get("dev_split", "validation") + self.test_split = d.get("test_split", "test") + self.text_key = d.get("text_key", "text") + self.text_b_key = d.get("text_b_key", None) + self.label_key = d.get("label_key", "label") + self.test_text_key = d.get("test_text_key", self.text_key) + self.test_text_b_key = d.get("test_text_b_key", self.text_b_key) + self.test_label = d.get("test_label", "test_example") + self.label_type = dtype_map[d.get("label_type", "int")] + self.is_regression = cast_str_to_bool(d.get("is_regression", "False")) + self.weight_key = d.get("weight_key", None) + self.skip_label = d.get("skip_label", None) + if self.skip_label is not None: + self.skip_label = self.label_type(self.skip_label) + + def get_train_examples(self, data_dir): + assert data_dir is None + return self._create_examples(self.train_split, "train") + + def get_dev_examples(self, data_dir): + assert data_dir is None + return self._create_examples(self.dev_split, "dev") + + def get_test_examples(self, data_dir): + assert data_dir is None + return self._create_examples(self.test_split, "test") + + def get_labels(self): + return self._labels + + def get_processor_name(self): + return "TFDS_" + self.dataset_name + + def _create_examples(self, split_name, set_type): + """Creates examples for the training/dev/test sets.""" + if split_name not in self.dataset: + raise ValueError("Split {} not available.".format(split_name)) + dataset = self.dataset[split_name].as_numpy_iterator() + examples = [] + text_b, weight = None, None + for i, example in enumerate(dataset): + guid = "%s-%s" % (set_type, i) + if set_type == "test": + text_a = self.process_text_fn(example[self.test_text_key]) + if self.test_text_b_key: + text_b = self.process_text_fn(example[self.test_text_b_key]) + label = self.test_label + else: + text_a = self.process_text_fn(example[self.text_key]) + if self.text_b_key: + text_b = self.process_text_fn(example[self.text_b_key]) + label = self.label_type(example[self.label_key]) + if self.skip_label is not None and label == self.skip_label: + continue + if self.weight_key: + weight = float(example[self.weight_key]) + examples.append( + InputExample( + guid=guid, + text_a=text_a, + text_b=text_b, + label=label, + weight=weight)) + return examples + + +class WnliProcessor(DefaultGLUEDataProcessor): + """Processor for the WNLI data set (GLUE version).""" + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "WNLI" + + def _create_examples_tfds(self, set_type): + """Creates examples for the training/dev/test sets.""" + dataset = tfds.load( + "glue/wnli", split=set_type, try_gcs=True).as_numpy_iterator() + examples = [] + for i, example in enumerate(dataset): + guid = "%s-%s" % (set_type, i) + label = "0" + text_a = self.process_text_fn(example["sentence1"]) + text_b = self.process_text_fn(example["sentence2"]) + if set_type != "test": + label = str(example["label"]) + examples.append( + InputExample( + guid=guid, text_a=text_a, text_b=text_b, label=label, + weight=None)) + return examples + + +class XnliProcessor(DataProcessor): + """Processor for the XNLI data set.""" + supported_languages = [ + "ar", "bg", "de", "el", "en", "es", "fr", "hi", "ru", "sw", "th", "tr", + "ur", "vi", "zh" + ] + + def __init__(self, + language="en", + process_text_fn=tokenization.convert_to_unicode): + super(XnliProcessor, self).__init__(process_text_fn) + if language == "all": + self.languages = XnliProcessor.supported_languages + elif language not in XnliProcessor.supported_languages: + raise ValueError("language %s is not supported for XNLI task." % language) + else: + self.languages = [language] + + def get_train_examples(self, data_dir): + """See base class.""" + lines = [] + for language in self.languages: + # Skips the header. + lines.extend( + self._read_tsv( + os.path.join(data_dir, "multinli", + "multinli.train.%s.tsv" % language))[1:]) + + examples = [] + for i, line in enumerate(lines): + guid = "train-%d" % i + text_a = self.process_text_fn(line[0]) + text_b = self.process_text_fn(line[1]) + label = self.process_text_fn(line[2]) + if label == self.process_text_fn("contradictory"): + label = self.process_text_fn("contradiction") + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_dev_examples(self, data_dir): + """See base class.""" + lines = self._read_tsv(os.path.join(data_dir, "xnli.dev.tsv")) + examples = [] + for i, line in enumerate(lines): + if i == 0: + continue + guid = "dev-%d" % i + text_a = self.process_text_fn(line[6]) + text_b = self.process_text_fn(line[7]) + label = self.process_text_fn(line[1]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_test_examples(self, data_dir): + """See base class.""" + lines = self._read_tsv(os.path.join(data_dir, "xnli.test.tsv")) + examples_by_lang = {k: [] for k in XnliProcessor.supported_languages} + for i, line in enumerate(lines): + if i == 0: + continue + guid = "test-%d" % i + language = self.process_text_fn(line[0]) + text_a = self.process_text_fn(line[6]) + text_b = self.process_text_fn(line[7]) + label = self.process_text_fn(line[1]) + examples_by_lang[language].append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples_by_lang + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "XNLI" + + +class XtremePawsxProcessor(DataProcessor): + """Processor for the XTREME PAWS-X data set.""" + supported_languages = ["de", "en", "es", "fr", "ja", "ko", "zh"] + + def __init__(self, + process_text_fn=tokenization.convert_to_unicode, + translated_data_dir=None, + only_use_en_dev=True): + """See base class. + + Args: + process_text_fn: See base class. + translated_data_dir: If specified, will also include translated data in + the training and testing data. + only_use_en_dev: If True, only use english dev data. Otherwise, use dev + data from all languages. + """ + super(XtremePawsxProcessor, self).__init__(process_text_fn) + self.translated_data_dir = translated_data_dir + self.only_use_en_dev = only_use_en_dev + + def get_train_examples(self, data_dir): + """See base class.""" + examples = [] + if self.translated_data_dir is None: + lines = self._read_tsv(os.path.join(data_dir, "train-en.tsv")) + for i, line in enumerate(lines): + guid = "train-%d" % i + text_a = self.process_text_fn(line[0]) + text_b = self.process_text_fn(line[1]) + label = self.process_text_fn(line[2]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + else: + for lang in self.supported_languages: + lines = self._read_tsv( + os.path.join(self.translated_data_dir, "translate-train", + f"en-{lang}-translated.tsv")) + for i, line in enumerate(lines): + guid = f"train-{lang}-{i}" + text_a = self.process_text_fn(line[2]) + text_b = self.process_text_fn(line[3]) + label = self.process_text_fn(line[4]) + examples.append( + InputExample( + guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_dev_examples(self, data_dir): + """See base class.""" + examples = [] + if self.only_use_en_dev: + lines = self._read_tsv(os.path.join(data_dir, "dev-en.tsv")) + for i, line in enumerate(lines): + guid = "dev-%d" % i + text_a = self.process_text_fn(line[0]) + text_b = self.process_text_fn(line[1]) + label = self.process_text_fn(line[2]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + else: + for lang in self.supported_languages: + lines = self._read_tsv(os.path.join(data_dir, f"dev-{lang}.tsv")) + for i, line in enumerate(lines): + guid = f"dev-{lang}-{i}" + text_a = self.process_text_fn(line[0]) + text_b = self.process_text_fn(line[1]) + label = self.process_text_fn(line[2]) + examples.append( + InputExample( + guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_test_examples(self, data_dir): + """See base class.""" + examples_by_lang = {} + for lang in self.supported_languages: + examples_by_lang[lang] = [] + lines = self._read_tsv(os.path.join(data_dir, f"test-{lang}.tsv")) + for i, line in enumerate(lines): + guid = f"test-{lang}-{i}" + text_a = self.process_text_fn(line[0]) + text_b = self.process_text_fn(line[1]) + label = "0" + examples_by_lang[lang].append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + if self.translated_data_dir is not None: + for lang in self.supported_languages: + if lang == "en": + continue + examples_by_lang[f"{lang}-en"] = [] + lines = self._read_tsv( + os.path.join(self.translated_data_dir, "translate-test", + f"test-{lang}-en-translated.tsv")) + for i, line in enumerate(lines): + guid = f"test-{lang}-en-{i}" + text_a = self.process_text_fn(line[2]) + text_b = self.process_text_fn(line[3]) + label = "0" + examples_by_lang[f"{lang}-en"].append( + InputExample( + guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples_by_lang + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "XTREME-PAWS-X" + + +class XtremeXnliProcessor(DataProcessor): + """Processor for the XTREME XNLI data set.""" + supported_languages = [ + "ar", "bg", "de", "el", "en", "es", "fr", "hi", "ru", "sw", "th", "tr", + "ur", "vi", "zh" + ] + + def __init__(self, + process_text_fn=tokenization.convert_to_unicode, + translated_data_dir=None, + only_use_en_dev=True): + """See base class. + + Args: + process_text_fn: See base class. + translated_data_dir: If specified, will also include translated data in + the training data. + only_use_en_dev: If True, only use english dev data. Otherwise, use dev + data from all languages. + """ + super(XtremeXnliProcessor, self).__init__(process_text_fn) + self.translated_data_dir = translated_data_dir + self.only_use_en_dev = only_use_en_dev + + def get_train_examples(self, data_dir): + """See base class.""" + lines = self._read_tsv(os.path.join(data_dir, "train-en.tsv")) + + examples = [] + if self.translated_data_dir is None: + for i, line in enumerate(lines): + guid = "train-%d" % i + text_a = self.process_text_fn(line[0]) + text_b = self.process_text_fn(line[1]) + label = self.process_text_fn(line[2]) + if label == self.process_text_fn("contradictory"): + label = self.process_text_fn("contradiction") + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + else: + for lang in self.supported_languages: + lines = self._read_tsv( + os.path.join(self.translated_data_dir, "translate-train", + f"en-{lang}-translated.tsv")) + for i, line in enumerate(lines): + guid = f"train-{lang}-{i}" + text_a = self.process_text_fn(line[2]) + text_b = self.process_text_fn(line[3]) + label = self.process_text_fn(line[4]) + if label == self.process_text_fn("contradictory"): + label = self.process_text_fn("contradiction") + examples.append( + InputExample( + guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_dev_examples(self, data_dir): + """See base class.""" + examples = [] + if self.only_use_en_dev: + lines = self._read_tsv(os.path.join(data_dir, "dev-en.tsv")) + for i, line in enumerate(lines): + guid = "dev-%d" % i + text_a = self.process_text_fn(line[0]) + text_b = self.process_text_fn(line[1]) + label = self.process_text_fn(line[2]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + else: + for lang in self.supported_languages: + lines = self._read_tsv(os.path.join(data_dir, f"dev-{lang}.tsv")) + for i, line in enumerate(lines): + guid = f"dev-{lang}-{i}" + text_a = self.process_text_fn(line[0]) + text_b = self.process_text_fn(line[1]) + label = self.process_text_fn(line[2]) + if label == self.process_text_fn("contradictory"): + label = self.process_text_fn("contradiction") + examples.append( + InputExample( + guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_test_examples(self, data_dir): + """See base class.""" + examples_by_lang = {} + for lang in self.supported_languages: + examples_by_lang[lang] = [] + lines = self._read_tsv(os.path.join(data_dir, f"test-{lang}.tsv")) + for i, line in enumerate(lines): + guid = f"test-{lang}-{i}" + text_a = self.process_text_fn(line[0]) + text_b = self.process_text_fn(line[1]) + label = "contradiction" + examples_by_lang[lang].append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + if self.translated_data_dir is not None: + for lang in self.supported_languages: + if lang == "en": + continue + examples_by_lang[f"{lang}-en"] = [] + lines = self._read_tsv( + os.path.join(self.translated_data_dir, "translate-test", + f"test-{lang}-en-translated.tsv")) + for i, line in enumerate(lines): + guid = f"test-{lang}-en-{i}" + text_a = self.process_text_fn(line[2]) + text_b = self.process_text_fn(line[3]) + label = "contradiction" + examples_by_lang[f"{lang}-en"].append( + InputExample( + guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples_by_lang + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "XTREME-XNLI" + + +def convert_single_example(ex_index, example, label_list, max_seq_length, + tokenizer): + """Converts a single `InputExample` into a single `InputFeatures`.""" + label_map = {} + if label_list: + for (i, label) in enumerate(label_list): + label_map[label] = i + + tokens_a = tokenizer.tokenize(example.text_a) + tokens_b = None + if example.text_b: + tokens_b = tokenizer.tokenize(example.text_b) + + if tokens_b: + # Modifies `tokens_a` and `tokens_b` in place so that the total + # length is less than the specified length. + # Account for [CLS], [SEP], [SEP] with "- 3" + _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3) + else: + # Account for [CLS] and [SEP] with "- 2" + if len(tokens_a) > max_seq_length - 2: + tokens_a = tokens_a[0:(max_seq_length - 2)] + + seg_id_a = 0 + seg_id_b = 1 + seg_id_cls = 0 + seg_id_pad = 0 + + # The convention in BERT is: + # (a) For sequence pairs: + # tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP] + # type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1 + # (b) For single sequences: + # tokens: [CLS] the dog is hairy . [SEP] + # type_ids: 0 0 0 0 0 0 0 + # + # Where "type_ids" are used to indicate whether this is the first + # sequence or the second sequence. The embedding vectors for `type=0` and + # `type=1` were learned during pre-training and are added to the wordpiece + # embedding vector (and position vector). This is not *strictly* necessary + # since the [SEP] token unambiguously separates the sequences, but it makes + # it easier for the model to learn the concept of sequences. + # + # For classification tasks, the first vector (corresponding to [CLS]) is + # used as the "sentence vector". Note that this only makes sense because + # the entire model is fine-tuned. + tokens = [] + segment_ids = [] + tokens.append("[CLS]") + segment_ids.append(seg_id_cls) + for token in tokens_a: + tokens.append(token) + segment_ids.append(seg_id_a) + tokens.append("[SEP]") + segment_ids.append(seg_id_a) + + if tokens_b: + for token in tokens_b: + tokens.append(token) + segment_ids.append(seg_id_b) + tokens.append("[SEP]") + segment_ids.append(seg_id_b) + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < max_seq_length: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(seg_id_pad) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + label_id = label_map[example.label] if label_map else example.label + if ex_index < 5: + logging.info("*** Example ***") + logging.info("guid: %s", (example.guid)) + logging.info("tokens: %s", + " ".join([tokenization.printable_text(x) for x in tokens])) + logging.info("input_ids: %s", " ".join([str(x) for x in input_ids])) + logging.info("input_mask: %s", " ".join([str(x) for x in input_mask])) + logging.info("segment_ids: %s", " ".join([str(x) for x in segment_ids])) + logging.info("label: %s (id = %s)", example.label, str(label_id)) + logging.info("weight: %s", example.weight) + logging.info("example_id: %s", example.example_id) + + feature = InputFeatures( + input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + label_id=label_id, + is_real_example=True, + weight=example.weight, + example_id=example.example_id) + + return feature + + +class AXgProcessor(DataProcessor): + """Processor for the AXg dataset (SuperGLUE diagnostics dataset).""" + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_jsonl(os.path.join(data_dir, "AX-g.jsonl")), "test") + + def get_labels(self): + """See base class.""" + return ["entailment", "not_entailment"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "AXg" + + def _create_examples(self, lines, set_type): + """Creates examples for the training/dev/test sets.""" + examples = [] + for line in lines: + guid = "%s-%s" % (set_type, self.process_text_fn(str(line["idx"]))) + text_a = self.process_text_fn(line["premise"]) + text_b = self.process_text_fn(line["hypothesis"]) + label = self.process_text_fn(line["label"]) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class BoolQProcessor(DefaultGLUEDataProcessor): + """Processor for the BoolQ dataset (SuperGLUE diagnostics dataset).""" + + def get_labels(self): + """See base class.""" + return ["True", "False"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "BoolQ" + + def _create_examples_tfds(self, set_type): + """Creates examples for the training/dev/test sets.""" + dataset = tfds.load( + "super_glue/boolq", split=set_type, try_gcs=True).as_numpy_iterator() + examples = [] + for example in dataset: + guid = "%s-%s" % (set_type, self.process_text_fn(str(example["idx"]))) + text_a = self.process_text_fn(example["question"]) + text_b = self.process_text_fn(example["passage"]) + label = "False" + if set_type != "test": + label = self.get_labels()[example["label"]] + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class CBProcessor(DefaultGLUEDataProcessor): + """Processor for the CB dataset (SuperGLUE diagnostics dataset).""" + + def get_labels(self): + """See base class.""" + return ["entailment", "neutral", "contradiction"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "CB" + + def _create_examples_tfds(self, set_type): + """Creates examples for the training/dev/test sets.""" + dataset = tfds.load( + "super_glue/cb", split=set_type, try_gcs=True).as_numpy_iterator() + examples = [] + for example in dataset: + guid = "%s-%s" % (set_type, self.process_text_fn(str(example["idx"]))) + text_a = self.process_text_fn(example["premise"]) + text_b = self.process_text_fn(example["hypothesis"]) + label = "entailment" + if set_type != "test": + label = self.get_labels()[example["label"]] + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class SuperGLUERTEProcessor(DefaultGLUEDataProcessor): + """Processor for the RTE dataset (SuperGLUE version).""" + + def get_labels(self): + """See base class.""" + # All datasets are converted to 2-class split, where for 3-class datasets we + # collapse neutral and contradiction into not_entailment. + return ["entailment", "not_entailment"] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "RTESuperGLUE" + + def _create_examples_tfds(self, set_type): + """Creates examples for the training/dev/test sets.""" + examples = [] + dataset = tfds.load( + "super_glue/rte", split=set_type, try_gcs=True).as_numpy_iterator() + for example in dataset: + guid = "%s-%s" % (set_type, self.process_text_fn(str(example["idx"]))) + text_a = self.process_text_fn(example["premise"]) + text_b = self.process_text_fn(example["hypothesis"]) + label = "entailment" + if set_type != "test": + label = self.get_labels()[example["label"]] + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class WiCInputExample(InputExample): + """Processor for the WiC dataset (SuperGLUE version).""" + + def __init__(self, + guid, + text_a, + text_b=None, + label=None, + word=None, + weight=None, + example_id=None): + """A single training/test example for simple seq regression/classification.""" + super(WiCInputExample, self).__init__(guid, text_a, text_b, label, weight, + example_id) + self.word = word + + +class WiCProcessor(DefaultGLUEDataProcessor): + """Processor for the RTE dataset (SuperGLUE version).""" + + def get_labels(self): + """Not used.""" + return [] + + @staticmethod + def get_processor_name(): + """See base class.""" + return "RTESuperGLUE" + + def _create_examples_tfds(self, set_type): + """Creates examples for the training/dev/test sets.""" + examples = [] + dataset = tfds.load( + "super_glue/wic", split=set_type, try_gcs=True).as_numpy_iterator() + for example in dataset: + guid = "%s-%s" % (set_type, self.process_text_fn(str(example["idx"]))) + text_a = self.process_text_fn(example["sentence1"]) + text_b = self.process_text_fn(example["sentence2"]) + word = self.process_text_fn(example["word"]) + label = 0 + if set_type != "test": + label = example["label"] + examples.append( + WiCInputExample( + guid=guid, text_a=text_a, text_b=text_b, word=word, label=label)) + return examples + + def featurize_example(self, ex_index, example, label_list, max_seq_length, + tokenizer): + """Here we concate sentence1, sentence2, word together with [SEP] tokens.""" + del label_list + tokens_a = tokenizer.tokenize(example.text_a) + tokens_b = tokenizer.tokenize(example.text_b) + tokens_word = tokenizer.tokenize(example.word) + + # Modifies `tokens_a` and `tokens_b` in place so that the total + # length is less than the specified length. + # Account for [CLS], [SEP], [SEP], [SEP] with "- 4" + # Here we only pop out the first two sentence tokens. + _truncate_seq_pair(tokens_a, tokens_b, + max_seq_length - 4 - len(tokens_word)) + + seg_id_a = 0 + seg_id_b = 1 + seg_id_c = 2 + seg_id_cls = 0 + seg_id_pad = 0 + + tokens = [] + segment_ids = [] + tokens.append("[CLS]") + segment_ids.append(seg_id_cls) + for token in tokens_a: + tokens.append(token) + segment_ids.append(seg_id_a) + tokens.append("[SEP]") + segment_ids.append(seg_id_a) + + for token in tokens_b: + tokens.append(token) + segment_ids.append(seg_id_b) + + tokens.append("[SEP]") + segment_ids.append(seg_id_b) + + for token in tokens_word: + tokens.append(token) + segment_ids.append(seg_id_c) + + tokens.append("[SEP]") + segment_ids.append(seg_id_c) + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < max_seq_length: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(seg_id_pad) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + label_id = example.label + if ex_index < 5: + logging.info("*** Example ***") + logging.info("guid: %s", (example.guid)) + logging.info("tokens: %s", + " ".join([tokenization.printable_text(x) for x in tokens])) + logging.info("input_ids: %s", " ".join([str(x) for x in input_ids])) + logging.info("input_mask: %s", " ".join([str(x) for x in input_mask])) + logging.info("segment_ids: %s", " ".join([str(x) for x in segment_ids])) + logging.info("label: %s (id = %s)", example.label, str(label_id)) + logging.info("weight: %s", example.weight) + logging.info("example_id: %s", example.example_id) + + feature = InputFeatures( + input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + label_id=label_id, + is_real_example=True, + weight=example.weight, + example_id=example.example_id) + + return feature + + +def file_based_convert_examples_to_features(examples, + label_list, + max_seq_length, + tokenizer, + output_file, + label_type=None, + featurize_fn=None): + """Convert a set of `InputExample`s to a TFRecord file.""" + + tf.io.gfile.makedirs(os.path.dirname(output_file)) + writer = tf.io.TFRecordWriter(output_file) + + for ex_index, example in enumerate(examples): + if ex_index % 10000 == 0: + logging.info("Writing example %d of %d", ex_index, len(examples)) + + if featurize_fn: + feature = featurize_fn(ex_index, example, label_list, max_seq_length, + tokenizer) + else: + feature = convert_single_example(ex_index, example, label_list, + max_seq_length, tokenizer) + + def create_int_feature(values): + f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + return f + + def create_float_feature(values): + f = tf.train.Feature(float_list=tf.train.FloatList(value=list(values))) + return f + + features = collections.OrderedDict() + features["input_ids"] = create_int_feature(feature.input_ids) + features["input_mask"] = create_int_feature(feature.input_mask) + features["segment_ids"] = create_int_feature(feature.segment_ids) + if label_type is not None and label_type == float: + features["label_ids"] = create_float_feature([feature.label_id]) + elif feature.label_id is not None: + features["label_ids"] = create_int_feature([feature.label_id]) + features["is_real_example"] = create_int_feature( + [int(feature.is_real_example)]) + if feature.weight is not None: + features["weight"] = create_float_feature([feature.weight]) + if feature.example_id is not None: + features["example_id"] = create_int_feature([feature.example_id]) + else: + features["example_id"] = create_int_feature([ex_index]) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + writer.write(tf_example.SerializeToString()) + writer.close() + + +def _truncate_seq_pair(tokens_a, tokens_b, max_length): + """Truncates a sequence pair in place to the maximum length.""" + + # This is a simple heuristic which will always truncate the longer sequence + # one token at a time. This makes more sense than truncating an equal percent + # of tokens from each, since if one sequence is very short then each token + # that's truncated likely contains more information than a longer sequence. + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + tokens_b.pop() + + +def generate_tf_record_from_data_file(processor, + data_dir, + tokenizer, + train_data_output_path=None, + eval_data_output_path=None, + test_data_output_path=None, + max_seq_length=128): + """Generates and saves training data into a tf record file. + + Args: + processor: Input processor object to be used for generating data. Subclass + of `DataProcessor`. + data_dir: Directory that contains train/eval/test data to process. + tokenizer: The tokenizer to be applied on the data. + train_data_output_path: Output to which processed tf record for training + will be saved. + eval_data_output_path: Output to which processed tf record for evaluation + will be saved. + test_data_output_path: Output to which processed tf record for testing + will be saved. Must be a pattern template with {} if processor has + language specific test data. + max_seq_length: Maximum sequence length of the to be generated + training/eval data. + + Returns: + A dictionary containing input meta data. + """ + assert train_data_output_path or eval_data_output_path + + label_list = processor.get_labels() + label_type = getattr(processor, "label_type", None) + is_regression = getattr(processor, "is_regression", False) + has_sample_weights = getattr(processor, "weight_key", False) + + num_training_data = 0 + if train_data_output_path: + train_input_data_examples = processor.get_train_examples(data_dir) + file_based_convert_examples_to_features(train_input_data_examples, + label_list, max_seq_length, + tokenizer, train_data_output_path, + label_type, + processor.featurize_example) + num_training_data = len(train_input_data_examples) + + if eval_data_output_path: + eval_input_data_examples = processor.get_dev_examples(data_dir) + file_based_convert_examples_to_features(eval_input_data_examples, + label_list, max_seq_length, + tokenizer, eval_data_output_path, + label_type, + processor.featurize_example) + + meta_data = { + "processor_type": processor.get_processor_name(), + "train_data_size": num_training_data, + "max_seq_length": max_seq_length, + } + + if test_data_output_path: + test_input_data_examples = processor.get_test_examples(data_dir) + if isinstance(test_input_data_examples, dict): + for language, examples in test_input_data_examples.items(): + file_based_convert_examples_to_features( + examples, label_list, max_seq_length, tokenizer, + test_data_output_path.format(language), label_type, + processor.featurize_example) + meta_data["test_{}_data_size".format(language)] = len(examples) + else: + file_based_convert_examples_to_features(test_input_data_examples, + label_list, max_seq_length, + tokenizer, test_data_output_path, + label_type, + processor.featurize_example) + meta_data["test_data_size"] = len(test_input_data_examples) + + if is_regression: + meta_data["task_type"] = "bert_regression" + meta_data["label_type"] = {int: "int", float: "float"}[label_type] + else: + meta_data["task_type"] = "bert_classification" + meta_data["num_labels"] = len(processor.get_labels()) + if has_sample_weights: + meta_data["has_sample_weights"] = True + + if eval_data_output_path: + meta_data["eval_data_size"] = len(eval_input_data_examples) + + return meta_data diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/classifier_data_lib_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/classifier_data_lib_test.py new file mode 100644 index 0000000000000000000000000000000000000000..c1db1a3d03f7b6daaa4816decb653814c675f3e2 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/classifier_data_lib_test.py @@ -0,0 +1,95 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for third_party.tensorflow_models.official.nlp.data.classifier_data_lib.""" + +import os +import tempfile + +from absl.testing import parameterized +import tensorflow as tf +import tensorflow_datasets as tfds + +from official.nlp.bert import tokenization +from official.nlp.data import classifier_data_lib + + +def decode_record(record, name_to_features): + """Decodes a record to a TensorFlow example.""" + return tf.io.parse_single_example(record, name_to_features) + + +class BertClassifierLibTest(tf.test.TestCase, parameterized.TestCase): + + def setUp(self): + super(BertClassifierLibTest, self).setUp() + self.model_dir = self.get_temp_dir() + self.processors = { + "CB": classifier_data_lib.CBProcessor, + "SUPERGLUE-RTE": classifier_data_lib.SuperGLUERTEProcessor, + "BOOLQ": classifier_data_lib.BoolQProcessor, + "WIC": classifier_data_lib.WiCProcessor, + } + + vocab_tokens = [ + "[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", + "##ing", "," + ] + with tempfile.NamedTemporaryFile(delete=False) as vocab_writer: + vocab_writer.write("".join([x + "\n" for x in vocab_tokens + ]).encode("utf-8")) + vocab_file = vocab_writer.name + self.tokenizer = tokenization.FullTokenizer(vocab_file) + + @parameterized.parameters( + {"task_type": "CB"}, + {"task_type": "BOOLQ"}, + {"task_type": "SUPERGLUE-RTE"}, + {"task_type": "WIC"}, + ) + def test_generate_dataset_from_tfds_processor(self, task_type): + with tfds.testing.mock_data(num_examples=5): + output_path = os.path.join(self.model_dir, task_type) + + processor = self.processors[task_type]() + + classifier_data_lib.generate_tf_record_from_data_file( + processor, + None, + self.tokenizer, + train_data_output_path=output_path, + eval_data_output_path=output_path, + test_data_output_path=output_path) + files = tf.io.gfile.glob(output_path) + self.assertNotEmpty(files) + + train_dataset = tf.data.TFRecordDataset(output_path) + seq_length = 128 + label_type = tf.int64 + name_to_features = { + "input_ids": tf.io.FixedLenFeature([seq_length], tf.int64), + "input_mask": tf.io.FixedLenFeature([seq_length], tf.int64), + "segment_ids": tf.io.FixedLenFeature([seq_length], tf.int64), + "label_ids": tf.io.FixedLenFeature([], label_type), + } + train_dataset = train_dataset.map( + lambda record: decode_record(record, name_to_features)) + + # If data is retrieved without error, then all requirements + # including data type/shapes are met. + _ = next(iter(train_dataset)) + + +if __name__ == "__main__": + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/create_finetuning_data.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/create_finetuning_data.py new file mode 100644 index 0000000000000000000000000000000000000000..01f2deaecde56e9927eb41452fd896539932d123 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/create_finetuning_data.py @@ -0,0 +1,442 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""BERT finetuning task dataset generator.""" + +import functools +import json +import os + +# Import libraries +from absl import app +from absl import flags +import tensorflow as tf +from official.nlp.bert import tokenization +from official.nlp.data import classifier_data_lib +from official.nlp.data import sentence_retrieval_lib +# word-piece tokenizer based squad_lib +from official.nlp.data import squad_lib as squad_lib_wp +# sentence-piece tokenizer based squad_lib +from official.nlp.data import squad_lib_sp +from official.nlp.data import tagging_data_lib + +FLAGS = flags.FLAGS + +# TODO(chendouble): consider moving each task to its own binary. +flags.DEFINE_enum( + "fine_tuning_task_type", "classification", + ["classification", "regression", "squad", "retrieval", "tagging"], + "The name of the BERT fine tuning task for which data " + "will be generated.") + +# BERT classification specific flags. +flags.DEFINE_string( + "input_data_dir", None, + "The input data dir. Should contain the .tsv files (or other data files) " + "for the task.") + +flags.DEFINE_enum( + "classification_task_name", "MNLI", [ + "AX", "COLA", "IMDB", "MNLI", "MRPC", "PAWS-X", "QNLI", "QQP", "RTE", + "SST-2", "STS-B", "WNLI", "XNLI", "XTREME-XNLI", "XTREME-PAWS-X", + "AX-g", "SUPERGLUE-RTE", "CB", "BoolQ", "WIC" + ], "The name of the task to train BERT classifier. The " + "difference between XTREME-XNLI and XNLI is: 1. the format " + "of input tsv files; 2. the dev set for XTREME is english " + "only and for XNLI is all languages combined. Same for " + "PAWS-X.") + +# MNLI task-specific flag. +flags.DEFINE_enum("mnli_type", "matched", ["matched", "mismatched"], + "The type of MNLI dataset.") + +# XNLI task-specific flag. +flags.DEFINE_string( + "xnli_language", "en", + "Language of training data for XNLI task. If the value is 'all', the data " + "of all languages will be used for training.") + +# PAWS-X task-specific flag. +flags.DEFINE_string( + "pawsx_language", "en", + "Language of training data for PAWS-X task. If the value is 'all', the data " + "of all languages will be used for training.") + +# XTREME classification specific flags. Only used in XtremePawsx and XtremeXnli. +flags.DEFINE_string( + "translated_input_data_dir", None, + "The translated input data dir. Should contain the .tsv files (or other " + "data files) for the task.") + +# Retrieval task-specific flags. +flags.DEFINE_enum("retrieval_task_name", "bucc", ["bucc", "tatoeba"], + "The name of sentence retrieval task for scoring") + +# Tagging task-specific flags. +flags.DEFINE_enum("tagging_task_name", "panx", ["panx", "udpos"], + "The name of BERT tagging (token classification) task.") + +flags.DEFINE_bool("tagging_only_use_en_train", True, + "Whether only use english training data in tagging.") + +# BERT Squad task-specific flags. +flags.DEFINE_string( + "squad_data_file", None, + "The input data file in for generating training data for BERT squad task.") + +flags.DEFINE_string( + "translated_squad_data_folder", None, + "The translated data folder for generating training data for BERT squad " + "task.") + +flags.DEFINE_integer( + "doc_stride", 128, + "When splitting up a long document into chunks, how much stride to " + "take between chunks.") + +flags.DEFINE_integer( + "max_query_length", 64, + "The maximum number of tokens for the question. Questions longer than " + "this will be truncated to this length.") + +flags.DEFINE_bool( + "version_2_with_negative", False, + "If true, the SQuAD examples contain some that do not have an answer.") + +flags.DEFINE_bool( + "xlnet_format", False, + "If true, then data will be preprocessed in a paragraph, query, class order" + " instead of the BERT-style class, paragraph, query order.") + +# XTREME specific flags. +flags.DEFINE_bool("only_use_en_dev", True, "Whether only use english dev data.") + +# Shared flags across BERT fine-tuning tasks. +flags.DEFINE_string("vocab_file", None, + "The vocabulary file that the BERT model was trained on.") + +flags.DEFINE_string( + "train_data_output_path", None, + "The path in which generated training input data will be written as tf" + " records.") + +flags.DEFINE_string( + "eval_data_output_path", None, + "The path in which generated evaluation input data will be written as tf" + " records.") + +flags.DEFINE_string( + "test_data_output_path", None, + "The path in which generated test input data will be written as tf" + " records. If None, do not generate test data. Must be a pattern template" + " as test_{}.tfrecords if processor has language specific test data.") + +flags.DEFINE_string("meta_data_file_path", None, + "The path in which input meta data will be written.") + +flags.DEFINE_bool( + "do_lower_case", True, + "Whether to lower case the input text. Should be True for uncased " + "models and False for cased models.") + +flags.DEFINE_integer( + "max_seq_length", 128, + "The maximum total input sequence length after WordPiece tokenization. " + "Sequences longer than this will be truncated, and sequences shorter " + "than this will be padded.") + +flags.DEFINE_string("sp_model_file", "", + "The path to the model used by sentence piece tokenizer.") + +flags.DEFINE_enum( + "tokenization", "WordPiece", ["WordPiece", "SentencePiece"], + "Specifies the tokenizer implementation, i.e., whether to use WordPiece " + "or SentencePiece tokenizer. Canonical BERT uses WordPiece tokenizer, " + "while ALBERT uses SentencePiece tokenizer.") + +flags.DEFINE_string( + "tfds_params", "", "Comma-separated list of TFDS parameter assigments for " + "generic classfication data import (for more details " + "see the TfdsProcessor class documentation).") + + +def generate_classifier_dataset(): + """Generates classifier dataset and returns input meta data.""" + if FLAGS.classification_task_name in [ + "COLA", + "WNLI", + "SST-2", + "MRPC", + "QQP", + "STS-B", + "MNLI", + "QNLI", + "RTE", + "AX", + "SUPERGLUE-RTE", + "CB", + "BoolQ", + "WIC", + ]: + assert not FLAGS.input_data_dir or FLAGS.tfds_params + else: + assert (FLAGS.input_data_dir and FLAGS.classification_task_name or + FLAGS.tfds_params) + + if FLAGS.tokenization == "WordPiece": + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + processor_text_fn = tokenization.convert_to_unicode + else: + assert FLAGS.tokenization == "SentencePiece" + tokenizer = tokenization.FullSentencePieceTokenizer(FLAGS.sp_model_file) + processor_text_fn = functools.partial( + tokenization.preprocess_text, lower=FLAGS.do_lower_case) + + if FLAGS.tfds_params: + processor = classifier_data_lib.TfdsProcessor( + tfds_params=FLAGS.tfds_params, process_text_fn=processor_text_fn) + return classifier_data_lib.generate_tf_record_from_data_file( + processor, + None, + tokenizer, + train_data_output_path=FLAGS.train_data_output_path, + eval_data_output_path=FLAGS.eval_data_output_path, + test_data_output_path=FLAGS.test_data_output_path, + max_seq_length=FLAGS.max_seq_length) + else: + processors = { + "ax": + classifier_data_lib.AxProcessor, + "cola": + classifier_data_lib.ColaProcessor, + "imdb": + classifier_data_lib.ImdbProcessor, + "mnli": + functools.partial( + classifier_data_lib.MnliProcessor, mnli_type=FLAGS.mnli_type), + "mrpc": + classifier_data_lib.MrpcProcessor, + "qnli": + classifier_data_lib.QnliProcessor, + "qqp": + classifier_data_lib.QqpProcessor, + "rte": + classifier_data_lib.RteProcessor, + "sst-2": + classifier_data_lib.SstProcessor, + "sts-b": + classifier_data_lib.StsBProcessor, + "xnli": + functools.partial( + classifier_data_lib.XnliProcessor, + language=FLAGS.xnli_language), + "paws-x": + functools.partial( + classifier_data_lib.PawsxProcessor, + language=FLAGS.pawsx_language), + "wnli": + classifier_data_lib.WnliProcessor, + "xtreme-xnli": + functools.partial( + classifier_data_lib.XtremeXnliProcessor, + translated_data_dir=FLAGS.translated_input_data_dir, + only_use_en_dev=FLAGS.only_use_en_dev), + "xtreme-paws-x": + functools.partial( + classifier_data_lib.XtremePawsxProcessor, + translated_data_dir=FLAGS.translated_input_data_dir, + only_use_en_dev=FLAGS.only_use_en_dev), + "ax-g": + classifier_data_lib.AXgProcessor, + "superglue-rte": + classifier_data_lib.SuperGLUERTEProcessor, + "cb": + classifier_data_lib.CBProcessor, + "boolq": + classifier_data_lib.BoolQProcessor, + "wic": + classifier_data_lib.WnliProcessor, + } + task_name = FLAGS.classification_task_name.lower() + if task_name not in processors: + raise ValueError("Task not found: %s" % (task_name)) + + processor = processors[task_name](process_text_fn=processor_text_fn) + return classifier_data_lib.generate_tf_record_from_data_file( + processor, + FLAGS.input_data_dir, + tokenizer, + train_data_output_path=FLAGS.train_data_output_path, + eval_data_output_path=FLAGS.eval_data_output_path, + test_data_output_path=FLAGS.test_data_output_path, + max_seq_length=FLAGS.max_seq_length) + + +def generate_regression_dataset(): + """Generates regression dataset and returns input meta data.""" + if FLAGS.tokenization == "WordPiece": + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + processor_text_fn = tokenization.convert_to_unicode + else: + assert FLAGS.tokenization == "SentencePiece" + tokenizer = tokenization.FullSentencePieceTokenizer(FLAGS.sp_model_file) + processor_text_fn = functools.partial( + tokenization.preprocess_text, lower=FLAGS.do_lower_case) + + if FLAGS.tfds_params: + processor = classifier_data_lib.TfdsProcessor( + tfds_params=FLAGS.tfds_params, process_text_fn=processor_text_fn) + return classifier_data_lib.generate_tf_record_from_data_file( + processor, + None, + tokenizer, + train_data_output_path=FLAGS.train_data_output_path, + eval_data_output_path=FLAGS.eval_data_output_path, + test_data_output_path=FLAGS.test_data_output_path, + max_seq_length=FLAGS.max_seq_length) + else: + raise ValueError("No data processor found for the given regression task.") + + +def generate_squad_dataset(): + """Generates squad training dataset and returns input meta data.""" + assert FLAGS.squad_data_file + if FLAGS.tokenization == "WordPiece": + return squad_lib_wp.generate_tf_record_from_json_file( + input_file_path=FLAGS.squad_data_file, + vocab_file_path=FLAGS.vocab_file, + output_path=FLAGS.train_data_output_path, + translated_input_folder=FLAGS.translated_squad_data_folder, + max_seq_length=FLAGS.max_seq_length, + do_lower_case=FLAGS.do_lower_case, + max_query_length=FLAGS.max_query_length, + doc_stride=FLAGS.doc_stride, + version_2_with_negative=FLAGS.version_2_with_negative, + xlnet_format=FLAGS.xlnet_format) + else: + assert FLAGS.tokenization == "SentencePiece" + return squad_lib_sp.generate_tf_record_from_json_file( + input_file_path=FLAGS.squad_data_file, + sp_model_file=FLAGS.sp_model_file, + output_path=FLAGS.train_data_output_path, + translated_input_folder=FLAGS.translated_squad_data_folder, + max_seq_length=FLAGS.max_seq_length, + do_lower_case=FLAGS.do_lower_case, + max_query_length=FLAGS.max_query_length, + doc_stride=FLAGS.doc_stride, + xlnet_format=FLAGS.xlnet_format, + version_2_with_negative=FLAGS.version_2_with_negative) + + +def generate_retrieval_dataset(): + """Generate retrieval test and dev dataset and returns input meta data.""" + assert (FLAGS.input_data_dir and FLAGS.retrieval_task_name) + if FLAGS.tokenization == "WordPiece": + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + processor_text_fn = tokenization.convert_to_unicode + else: + assert FLAGS.tokenization == "SentencePiece" + tokenizer = tokenization.FullSentencePieceTokenizer(FLAGS.sp_model_file) + processor_text_fn = functools.partial( + tokenization.preprocess_text, lower=FLAGS.do_lower_case) + + processors = { + "bucc": sentence_retrieval_lib.BuccProcessor, + "tatoeba": sentence_retrieval_lib.TatoebaProcessor, + } + + task_name = FLAGS.retrieval_task_name.lower() + if task_name not in processors: + raise ValueError("Task not found: %s" % task_name) + + processor = processors[task_name](process_text_fn=processor_text_fn) + + return sentence_retrieval_lib.generate_sentence_retrevial_tf_record( + processor, FLAGS.input_data_dir, tokenizer, FLAGS.eval_data_output_path, + FLAGS.test_data_output_path, FLAGS.max_seq_length) + + +def generate_tagging_dataset(): + """Generates tagging dataset.""" + processors = { + "panx": + functools.partial( + tagging_data_lib.PanxProcessor, + only_use_en_train=FLAGS.tagging_only_use_en_train, + only_use_en_dev=FLAGS.only_use_en_dev), + "udpos": + functools.partial( + tagging_data_lib.UdposProcessor, + only_use_en_train=FLAGS.tagging_only_use_en_train, + only_use_en_dev=FLAGS.only_use_en_dev), + } + task_name = FLAGS.tagging_task_name.lower() + if task_name not in processors: + raise ValueError("Task not found: %s" % task_name) + + if FLAGS.tokenization == "WordPiece": + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + processor_text_fn = tokenization.convert_to_unicode + elif FLAGS.tokenization == "SentencePiece": + tokenizer = tokenization.FullSentencePieceTokenizer(FLAGS.sp_model_file) + processor_text_fn = functools.partial( + tokenization.preprocess_text, lower=FLAGS.do_lower_case) + else: + raise ValueError("Unsupported tokenization: %s" % FLAGS.tokenization) + + processor = processors[task_name]() + return tagging_data_lib.generate_tf_record_from_data_file( + processor, FLAGS.input_data_dir, tokenizer, FLAGS.max_seq_length, + FLAGS.train_data_output_path, FLAGS.eval_data_output_path, + FLAGS.test_data_output_path, processor_text_fn) + + +def main(_): + if FLAGS.tokenization == "WordPiece": + if not FLAGS.vocab_file: + raise ValueError( + "FLAG vocab_file for word-piece tokenizer is not specified.") + else: + assert FLAGS.tokenization == "SentencePiece" + if not FLAGS.sp_model_file: + raise ValueError( + "FLAG sp_model_file for sentence-piece tokenizer is not specified.") + + if FLAGS.fine_tuning_task_type != "retrieval": + flags.mark_flag_as_required("train_data_output_path") + + if FLAGS.fine_tuning_task_type == "classification": + input_meta_data = generate_classifier_dataset() + elif FLAGS.fine_tuning_task_type == "regression": + input_meta_data = generate_regression_dataset() + elif FLAGS.fine_tuning_task_type == "retrieval": + input_meta_data = generate_retrieval_dataset() + elif FLAGS.fine_tuning_task_type == "squad": + input_meta_data = generate_squad_dataset() + else: + assert FLAGS.fine_tuning_task_type == "tagging" + input_meta_data = generate_tagging_dataset() + + tf.io.gfile.makedirs(os.path.dirname(FLAGS.meta_data_file_path)) + with tf.io.gfile.GFile(FLAGS.meta_data_file_path, "w") as writer: + writer.write(json.dumps(input_meta_data, indent=4) + "\n") + + +if __name__ == "__main__": + flags.mark_flag_as_required("meta_data_file_path") + app.run(main) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/create_pretraining_data.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/create_pretraining_data.py new file mode 100644 index 0000000000000000000000000000000000000000..93b7723d125a6e4916a8a595ef4c5a4b470bdcc9 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/create_pretraining_data.py @@ -0,0 +1,669 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create masked LM/next sentence masked_lm TF examples for BERT.""" + +import collections +import itertools +import random + +# Import libraries +from absl import app +from absl import flags +from absl import logging +import tensorflow as tf + +from official.nlp.bert import tokenization + +FLAGS = flags.FLAGS + +flags.DEFINE_string("input_file", None, + "Input raw text file (or comma-separated list of files).") + +flags.DEFINE_string( + "output_file", None, + "Output TF example file (or comma-separated list of files).") + +flags.DEFINE_string("vocab_file", None, + "The vocabulary file that the BERT model was trained on.") + +flags.DEFINE_bool( + "do_lower_case", True, + "Whether to lower case the input text. Should be True for uncased " + "models and False for cased models.") + +flags.DEFINE_bool( + "do_whole_word_mask", False, + "Whether to use whole word masking rather than per-WordPiece masking.") + +flags.DEFINE_integer( + "max_ngram_size", None, + "Mask contiguous whole words (n-grams) of up to `max_ngram_size` using a " + "weighting scheme to favor shorter n-grams. " + "Note: `--do_whole_word_mask=True` must also be set when n-gram masking.") + +flags.DEFINE_bool( + "gzip_compress", False, + "Whether to use `GZIP` compress option to get compressed TFRecord files.") + +flags.DEFINE_bool( + "use_v2_feature_names", False, + "Whether to use the feature names consistent with the models.") + +flags.DEFINE_integer("max_seq_length", 128, "Maximum sequence length.") + +flags.DEFINE_integer("max_predictions_per_seq", 20, + "Maximum number of masked LM predictions per sequence.") + +flags.DEFINE_integer("random_seed", 12345, "Random seed for data generation.") + +flags.DEFINE_integer( + "dupe_factor", 10, + "Number of times to duplicate the input data (with different masks).") + +flags.DEFINE_float("masked_lm_prob", 0.15, "Masked LM probability.") + +flags.DEFINE_float( + "short_seq_prob", 0.1, + "Probability of creating sequences which are shorter than the " + "maximum length.") + + +class TrainingInstance(object): + """A single training instance (sentence pair).""" + + def __init__(self, tokens, segment_ids, masked_lm_positions, masked_lm_labels, + is_random_next): + self.tokens = tokens + self.segment_ids = segment_ids + self.is_random_next = is_random_next + self.masked_lm_positions = masked_lm_positions + self.masked_lm_labels = masked_lm_labels + + def __str__(self): + s = "" + s += "tokens: %s\n" % (" ".join( + [tokenization.printable_text(x) for x in self.tokens])) + s += "segment_ids: %s\n" % (" ".join([str(x) for x in self.segment_ids])) + s += "is_random_next: %s\n" % self.is_random_next + s += "masked_lm_positions: %s\n" % (" ".join( + [str(x) for x in self.masked_lm_positions])) + s += "masked_lm_labels: %s\n" % (" ".join( + [tokenization.printable_text(x) for x in self.masked_lm_labels])) + s += "\n" + return s + + def __repr__(self): + return self.__str__() + + +def write_instance_to_example_files(instances, tokenizer, max_seq_length, + max_predictions_per_seq, output_files, + gzip_compress, use_v2_feature_names): + """Creates TF example files from `TrainingInstance`s.""" + writers = [] + for output_file in output_files: + writers.append( + tf.io.TFRecordWriter( + output_file, options="GZIP" if gzip_compress else "")) + + writer_index = 0 + + total_written = 0 + for (inst_index, instance) in enumerate(instances): + input_ids = tokenizer.convert_tokens_to_ids(instance.tokens) + input_mask = [1] * len(input_ids) + segment_ids = list(instance.segment_ids) + assert len(input_ids) <= max_seq_length + + while len(input_ids) < max_seq_length: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(0) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + masked_lm_positions = list(instance.masked_lm_positions) + masked_lm_ids = tokenizer.convert_tokens_to_ids(instance.masked_lm_labels) + masked_lm_weights = [1.0] * len(masked_lm_ids) + + while len(masked_lm_positions) < max_predictions_per_seq: + masked_lm_positions.append(0) + masked_lm_ids.append(0) + masked_lm_weights.append(0.0) + + next_sentence_label = 1 if instance.is_random_next else 0 + + features = collections.OrderedDict() + if use_v2_feature_names: + features["input_word_ids"] = create_int_feature(input_ids) + features["input_type_ids"] = create_int_feature(segment_ids) + else: + features["input_ids"] = create_int_feature(input_ids) + features["segment_ids"] = create_int_feature(segment_ids) + + features["input_mask"] = create_int_feature(input_mask) + features["masked_lm_positions"] = create_int_feature(masked_lm_positions) + features["masked_lm_ids"] = create_int_feature(masked_lm_ids) + features["masked_lm_weights"] = create_float_feature(masked_lm_weights) + features["next_sentence_labels"] = create_int_feature([next_sentence_label]) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + + writers[writer_index].write(tf_example.SerializeToString()) + writer_index = (writer_index + 1) % len(writers) + + total_written += 1 + + if inst_index < 20: + logging.info("*** Example ***") + logging.info("tokens: %s", " ".join( + [tokenization.printable_text(x) for x in instance.tokens])) + + for feature_name in features.keys(): + feature = features[feature_name] + values = [] + if feature.int64_list.value: + values = feature.int64_list.value + elif feature.float_list.value: + values = feature.float_list.value + logging.info("%s: %s", feature_name, " ".join([str(x) for x in values])) + + for writer in writers: + writer.close() + + logging.info("Wrote %d total instances", total_written) + + +def create_int_feature(values): + feature = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + return feature + + +def create_float_feature(values): + feature = tf.train.Feature(float_list=tf.train.FloatList(value=list(values))) + return feature + + +def create_training_instances(input_files, + tokenizer, + max_seq_length, + dupe_factor, + short_seq_prob, + masked_lm_prob, + max_predictions_per_seq, + rng, + do_whole_word_mask=False, + max_ngram_size=None): + """Create `TrainingInstance`s from raw text.""" + all_documents = [[]] + + # Input file format: + # (1) One sentence per line. These should ideally be actual sentences, not + # entire paragraphs or arbitrary spans of text. (Because we use the + # sentence boundaries for the "next sentence prediction" task). + # (2) Blank lines between documents. Document boundaries are needed so + # that the "next sentence prediction" task doesn't span between documents. + for input_file in input_files: + with tf.io.gfile.GFile(input_file, "rb") as reader: + while True: + line = tokenization.convert_to_unicode(reader.readline()) + if not line: + break + line = line.strip() + + # Empty lines are used as document delimiters + if not line: + all_documents.append([]) + tokens = tokenizer.tokenize(line) + if tokens: + all_documents[-1].append(tokens) + + # Remove empty documents + all_documents = [x for x in all_documents if x] + rng.shuffle(all_documents) + + vocab_words = list(tokenizer.vocab.keys()) + instances = [] + for _ in range(dupe_factor): + for document_index in range(len(all_documents)): + instances.extend( + create_instances_from_document( + all_documents, document_index, max_seq_length, short_seq_prob, + masked_lm_prob, max_predictions_per_seq, vocab_words, rng, + do_whole_word_mask, max_ngram_size)) + + rng.shuffle(instances) + return instances + + +def create_instances_from_document( + all_documents, document_index, max_seq_length, short_seq_prob, + masked_lm_prob, max_predictions_per_seq, vocab_words, rng, + do_whole_word_mask=False, + max_ngram_size=None): + """Creates `TrainingInstance`s for a single document.""" + document = all_documents[document_index] + + # Account for [CLS], [SEP], [SEP] + max_num_tokens = max_seq_length - 3 + + # We *usually* want to fill up the entire sequence since we are padding + # to `max_seq_length` anyways, so short sequences are generally wasted + # computation. However, we *sometimes* + # (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter + # sequences to minimize the mismatch between pre-training and fine-tuning. + # The `target_seq_length` is just a rough target however, whereas + # `max_seq_length` is a hard limit. + target_seq_length = max_num_tokens + if rng.random() < short_seq_prob: + target_seq_length = rng.randint(2, max_num_tokens) + + # We DON'T just concatenate all of the tokens from a document into a long + # sequence and choose an arbitrary split point because this would make the + # next sentence prediction task too easy. Instead, we split the input into + # segments "A" and "B" based on the actual "sentences" provided by the user + # input. + instances = [] + current_chunk = [] + current_length = 0 + i = 0 + while i < len(document): + segment = document[i] + current_chunk.append(segment) + current_length += len(segment) + if i == len(document) - 1 or current_length >= target_seq_length: + if current_chunk: + # `a_end` is how many segments from `current_chunk` go into the `A` + # (first) sentence. + a_end = 1 + if len(current_chunk) >= 2: + a_end = rng.randint(1, len(current_chunk) - 1) + + tokens_a = [] + for j in range(a_end): + tokens_a.extend(current_chunk[j]) + + tokens_b = [] + # Random next + is_random_next = False + if len(current_chunk) == 1 or rng.random() < 0.5: + is_random_next = True + target_b_length = target_seq_length - len(tokens_a) + + # This should rarely go for more than one iteration for large + # corpora. However, just to be careful, we try to make sure that + # the random document is not the same as the document + # we're processing. + for _ in range(10): + random_document_index = rng.randint(0, len(all_documents) - 1) + if random_document_index != document_index: + break + + random_document = all_documents[random_document_index] + random_start = rng.randint(0, len(random_document) - 1) + for j in range(random_start, len(random_document)): + tokens_b.extend(random_document[j]) + if len(tokens_b) >= target_b_length: + break + # We didn't actually use these segments so we "put them back" so + # they don't go to waste. + num_unused_segments = len(current_chunk) - a_end + i -= num_unused_segments + # Actual next + else: + is_random_next = False + for j in range(a_end, len(current_chunk)): + tokens_b.extend(current_chunk[j]) + truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng) + + assert len(tokens_a) >= 1 + assert len(tokens_b) >= 1 + + tokens = [] + segment_ids = [] + tokens.append("[CLS]") + segment_ids.append(0) + for token in tokens_a: + tokens.append(token) + segment_ids.append(0) + + tokens.append("[SEP]") + segment_ids.append(0) + + for token in tokens_b: + tokens.append(token) + segment_ids.append(1) + tokens.append("[SEP]") + segment_ids.append(1) + + (tokens, masked_lm_positions, + masked_lm_labels) = create_masked_lm_predictions( + tokens, masked_lm_prob, max_predictions_per_seq, vocab_words, rng, + do_whole_word_mask, max_ngram_size) + instance = TrainingInstance( + tokens=tokens, + segment_ids=segment_ids, + is_random_next=is_random_next, + masked_lm_positions=masked_lm_positions, + masked_lm_labels=masked_lm_labels) + instances.append(instance) + current_chunk = [] + current_length = 0 + i += 1 + + return instances + + +MaskedLmInstance = collections.namedtuple("MaskedLmInstance", + ["index", "label"]) + +# A _Gram is a [half-open) interval of token indices which form a word. +# E.g., +# words: ["The", "doghouse"] +# tokens: ["The", "dog", "##house"] +# grams: [(0,1), (1,3)] +_Gram = collections.namedtuple("_Gram", ["begin", "end"]) + + +def _window(iterable, size): + """Helper to create a sliding window iterator with a given size. + + E.g., + input = [1, 2, 3, 4] + _window(input, 1) => [1], [2], [3], [4] + _window(input, 2) => [1, 2], [2, 3], [3, 4] + _window(input, 3) => [1, 2, 3], [2, 3, 4] + _window(input, 4) => [1, 2, 3, 4] + _window(input, 5) => None + + Args: + iterable: elements to iterate over. + size: size of the window. + + Yields: + Elements of `iterable` batched into a sliding window of length `size`. + """ + i = iter(iterable) + window = [] + try: + for e in range(0, size): + window.append(next(i)) + yield window + except StopIteration: + # handle the case where iterable's length is less than the window size. + return + for e in i: + window = window[1:] + [e] + yield window + + +def _contiguous(sorted_grams): + """Test whether a sequence of grams is contiguous. + + Args: + sorted_grams: _Grams which are sorted in increasing order. + Returns: + True if `sorted_grams` are touching each other. + + E.g., + _contiguous([(1, 4), (4, 5), (5, 10)]) == True + _contiguous([(1, 2), (4, 5)]) == False + """ + for a, b in _window(sorted_grams, 2): + if a.end != b.begin: + return False + return True + + +def _masking_ngrams(grams, max_ngram_size, max_masked_tokens, rng): + """Create a list of masking {1, ..., n}-grams from a list of one-grams. + + This is an extention of 'whole word masking' to mask multiple, contiguous + words such as (e.g., "the red boat"). + + Each input gram represents the token indices of a single word, + words: ["the", "red", "boat"] + tokens: ["the", "red", "boa", "##t"] + grams: [(0,1), (1,2), (2,4)] + + For a `max_ngram_size` of three, possible outputs masks include: + 1-grams: (0,1), (1,2), (2,4) + 2-grams: (0,2), (1,4) + 3-grams; (0,4) + + Output masks will not overlap and contain less than `max_masked_tokens` total + tokens. E.g., for the example above with `max_masked_tokens` as three, + valid outputs are, + [(0,1), (1,2)] # "the", "red" covering two tokens + [(1,2), (2,4)] # "red", "boa", "##t" covering three tokens + + The length of the selected n-gram follows a zipf weighting to + favor shorter n-gram sizes (weight(1)=1, weight(2)=1/2, weight(3)=1/3, ...). + + Args: + grams: List of one-grams. + max_ngram_size: Maximum number of contiguous one-grams combined to create + an n-gram. + max_masked_tokens: Maximum total number of tokens to be masked. + rng: `random.Random` generator. + + Returns: + A list of n-grams to be used as masks. + """ + if not grams: + return None + + grams = sorted(grams) + num_tokens = grams[-1].end + + # Ensure our grams are valid (i.e., they don't overlap). + for a, b in _window(grams, 2): + if a.end > b.begin: + raise ValueError("overlapping grams: {}".format(grams)) + + # Build map from n-gram length to list of n-grams. + ngrams = {i: [] for i in range(1, max_ngram_size+1)} + for gram_size in range(1, max_ngram_size+1): + for g in _window(grams, gram_size): + if _contiguous(g): + # Add an n-gram which spans these one-grams. + ngrams[gram_size].append(_Gram(g[0].begin, g[-1].end)) + + # Shuffle each list of n-grams. + for v in ngrams.values(): + rng.shuffle(v) + + # Create the weighting for n-gram length selection. + # Stored cummulatively for `random.choices` below. + cummulative_weights = list( + itertools.accumulate([1./n for n in range(1, max_ngram_size+1)])) + + output_ngrams = [] + # Keep a bitmask of which tokens have been masked. + masked_tokens = [False] * num_tokens + # Loop until we have enough masked tokens or there are no more candidate + # n-grams of any length. + # Each code path should ensure one or more elements from `ngrams` are removed + # to guarentee this loop terminates. + while (sum(masked_tokens) < max_masked_tokens and + sum(len(s) for s in ngrams.values())): + # Pick an n-gram size based on our weights. + sz = random.choices(range(1, max_ngram_size+1), + cum_weights=cummulative_weights)[0] + + # Ensure this size doesn't result in too many masked tokens. + # E.g., a two-gram contains _at least_ two tokens. + if sum(masked_tokens) + sz > max_masked_tokens: + # All n-grams of this length are too long and can be removed from + # consideration. + ngrams[sz].clear() + continue + + # All of the n-grams of this size have been used. + if not ngrams[sz]: + continue + + # Choose a random n-gram of the given size. + gram = ngrams[sz].pop() + num_gram_tokens = gram.end-gram.begin + + # Check if this would add too many tokens. + if num_gram_tokens + sum(masked_tokens) > max_masked_tokens: + continue + + # Check if any of the tokens in this gram have already been masked. + if sum(masked_tokens[gram.begin:gram.end]): + continue + + # Found a usable n-gram! Mark its tokens as masked and add it to return. + masked_tokens[gram.begin:gram.end] = [True] * (gram.end-gram.begin) + output_ngrams.append(gram) + return output_ngrams + + +def _wordpieces_to_grams(tokens): + """Reconstitue grams (words) from `tokens`. + + E.g., + tokens: ['[CLS]', 'That', 'lit', '##tle', 'blue', 'tru', '##ck', '[SEP]'] + grams: [ [1,2), [2, 4), [4,5) , [5, 6)] + + Args: + tokens: list of wordpieces + Returns: + List of _Grams representing spans of whole words + (without "[CLS]" and "[SEP]"). + """ + grams = [] + gram_start_pos = None + for i, token in enumerate(tokens): + if gram_start_pos is not None and token.startswith("##"): + continue + if gram_start_pos is not None: + grams.append(_Gram(gram_start_pos, i)) + if token not in ["[CLS]", "[SEP]"]: + gram_start_pos = i + else: + gram_start_pos = None + if gram_start_pos is not None: + grams.append(_Gram(gram_start_pos, len(tokens))) + return grams + + +def create_masked_lm_predictions(tokens, masked_lm_prob, + max_predictions_per_seq, vocab_words, rng, + do_whole_word_mask, + max_ngram_size=None): + """Creates the predictions for the masked LM objective.""" + if do_whole_word_mask: + grams = _wordpieces_to_grams(tokens) + else: + # Here we consider each token to be a word to allow for sub-word masking. + if max_ngram_size: + raise ValueError("cannot use ngram masking without whole word masking") + grams = [_Gram(i, i+1) for i in range(0, len(tokens)) + if tokens[i] not in ["[CLS]", "[SEP]"]] + + num_to_predict = min(max_predictions_per_seq, + max(1, int(round(len(tokens) * masked_lm_prob)))) + # Generate masks. If `max_ngram_size` in [0, None] it means we're doing + # whole word masking or token level masking. Both of these can be treated + # as the `max_ngram_size=1` case. + masked_grams = _masking_ngrams(grams, max_ngram_size or 1, + num_to_predict, rng) + masked_lms = [] + output_tokens = list(tokens) + for gram in masked_grams: + # 80% of the time, replace all n-gram tokens with [MASK] + if rng.random() < 0.8: + replacement_action = lambda idx: "[MASK]" + else: + # 10% of the time, keep all the original n-gram tokens. + if rng.random() < 0.5: + replacement_action = lambda idx: tokens[idx] + # 10% of the time, replace each n-gram token with a random word. + else: + replacement_action = lambda idx: rng.choice(vocab_words) + + for idx in range(gram.begin, gram.end): + output_tokens[idx] = replacement_action(idx) + masked_lms.append(MaskedLmInstance(index=idx, label=tokens[idx])) + + assert len(masked_lms) <= num_to_predict + masked_lms = sorted(masked_lms, key=lambda x: x.index) + + masked_lm_positions = [] + masked_lm_labels = [] + for p in masked_lms: + masked_lm_positions.append(p.index) + masked_lm_labels.append(p.label) + + return (output_tokens, masked_lm_positions, masked_lm_labels) + + +def truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng): + """Truncates a pair of sequences to a maximum sequence length.""" + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_num_tokens: + break + + trunc_tokens = tokens_a if len(tokens_a) > len(tokens_b) else tokens_b + assert len(trunc_tokens) >= 1 + + # We want to sometimes truncate from the front and sometimes from the + # back to add more randomness and avoid biases. + if rng.random() < 0.5: + del trunc_tokens[0] + else: + trunc_tokens.pop() + + +def main(_): + tokenizer = tokenization.FullTokenizer( + vocab_file=FLAGS.vocab_file, do_lower_case=FLAGS.do_lower_case) + + input_files = [] + for input_pattern in FLAGS.input_file.split(","): + input_files.extend(tf.io.gfile.glob(input_pattern)) + + logging.info("*** Reading from input files ***") + for input_file in input_files: + logging.info(" %s", input_file) + + rng = random.Random(FLAGS.random_seed) + instances = create_training_instances( + input_files, tokenizer, FLAGS.max_seq_length, FLAGS.dupe_factor, + FLAGS.short_seq_prob, FLAGS.masked_lm_prob, FLAGS.max_predictions_per_seq, + rng, FLAGS.do_whole_word_mask, FLAGS.max_ngram_size) + + output_files = FLAGS.output_file.split(",") + logging.info("*** Writing to output files ***") + for output_file in output_files: + logging.info(" %s", output_file) + + write_instance_to_example_files(instances, tokenizer, FLAGS.max_seq_length, + FLAGS.max_predictions_per_seq, output_files, + FLAGS.gzip_compress, + FLAGS.use_v2_feature_names) + + +if __name__ == "__main__": + flags.mark_flag_as_required("input_file") + flags.mark_flag_as_required("output_file") + flags.mark_flag_as_required("vocab_file") + app.run(main) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/create_pretraining_data_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/create_pretraining_data_test.py new file mode 100644 index 0000000000000000000000000000000000000000..79a38ba8506ac428d48188f0eb4fbf2ce26b4422 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/create_pretraining_data_test.py @@ -0,0 +1,128 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for official.nlp.data.create_pretraining_data.""" +import random + +import tensorflow as tf + +from official.nlp.data import create_pretraining_data as cpd + +_VOCAB_WORDS = ["vocab_1", "vocab_2"] + + +class CreatePretrainingDataTest(tf.test.TestCase): + + def assertTokens(self, input_tokens, output_tokens, masked_positions, + masked_labels): + # Ensure the masked positions are unique. + self.assertCountEqual(masked_positions, set(masked_positions)) + + # Ensure we can reconstruct the input from the output. + reconstructed_tokens = output_tokens + for pos, label in zip(masked_positions, masked_labels): + reconstructed_tokens[pos] = label + self.assertEqual(input_tokens, reconstructed_tokens) + + # Ensure each label is valid. + for pos, label in zip(masked_positions, masked_labels): + output_token = output_tokens[pos] + if (output_token == "[MASK]" or output_token in _VOCAB_WORDS or + output_token == input_tokens[pos]): + continue + self.fail("invalid mask value: {}".format(output_token)) + + def test_wordpieces_to_grams(self): + tests = [ + (["That", "cone"], [(0, 1), (1, 2)]), + (["That", "cone", "##s"], [(0, 1), (1, 3)]), + (["Swit", "##zer", "##land"], [(0, 3)]), + (["[CLS]", "Up", "##dog"], [(1, 3)]), + (["[CLS]", "Up", "##dog", "[SEP]", "Down"], [(1, 3), (4, 5)]), + ] + for inp, expected in tests: + output = cpd._wordpieces_to_grams(inp) + self.assertEqual(expected, output) + + def test_window(self): + input_list = [1, 2, 3, 4] + window_outputs = [ + (1, [[1], [2], [3], [4]]), + (2, [[1, 2], [2, 3], [3, 4]]), + (3, [[1, 2, 3], [2, 3, 4]]), + (4, [[1, 2, 3, 4]]), + (5, []), + ] + for window, expected in window_outputs: + output = cpd._window(input_list, window) + self.assertEqual(expected, list(output)) + + def test_create_masked_lm_predictions(self): + tokens = ["[CLS]", "a", "##a", "b", "##b", "c", "##c", "[SEP]"] + rng = random.Random(123) + for _ in range(0, 5): + output_tokens, masked_positions, masked_labels = ( + cpd.create_masked_lm_predictions( + tokens=tokens, + masked_lm_prob=1.0, + max_predictions_per_seq=3, + vocab_words=_VOCAB_WORDS, + rng=rng, + do_whole_word_mask=False, + max_ngram_size=None)) + self.assertEqual(len(masked_positions), 3) + self.assertEqual(len(masked_labels), 3) + self.assertTokens(tokens, output_tokens, masked_positions, masked_labels) + + def test_create_masked_lm_predictions_whole_word(self): + tokens = ["[CLS]", "a", "##a", "b", "##b", "c", "##c", "[SEP]"] + rng = random.Random(345) + for _ in range(0, 5): + output_tokens, masked_positions, masked_labels = ( + cpd.create_masked_lm_predictions( + tokens=tokens, + masked_lm_prob=1.0, + max_predictions_per_seq=3, + vocab_words=_VOCAB_WORDS, + rng=rng, + do_whole_word_mask=True, + max_ngram_size=None)) + # since we can't get exactly three tokens without breaking a word we + # only take two. + self.assertEqual(len(masked_positions), 2) + self.assertEqual(len(masked_labels), 2) + self.assertTokens(tokens, output_tokens, masked_positions, masked_labels) + # ensure that we took an entire word. + self.assertIn(masked_labels, [["a", "##a"], ["b", "##b"], ["c", "##c"]]) + + def test_create_masked_lm_predictions_ngram(self): + tokens = ["[CLS]"] + ["tok{}".format(i) for i in range(0, 512)] + ["[SEP]"] + rng = random.Random(345) + for _ in range(0, 5): + output_tokens, masked_positions, masked_labels = ( + cpd.create_masked_lm_predictions( + tokens=tokens, + masked_lm_prob=1.0, + max_predictions_per_seq=76, + vocab_words=_VOCAB_WORDS, + rng=rng, + do_whole_word_mask=True, + max_ngram_size=3)) + self.assertEqual(len(masked_positions), 76) + self.assertEqual(len(masked_labels), 76) + self.assertTokens(tokens, output_tokens, masked_positions, masked_labels) + + +if __name__ == "__main__": + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/create_xlnet_pretraining_data.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/create_xlnet_pretraining_data.py new file mode 100644 index 0000000000000000000000000000000000000000..363164fcae001a61da53b0bb6e0afb9f4e92fd42 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/create_xlnet_pretraining_data.py @@ -0,0 +1,721 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create LM TF examples for XLNet.""" + +import json +import math +import os + +import random +from typing import Iterable, Mapping, List, Optional, Tuple +import unicodedata + +# Import libraries + +from absl import app +from absl import flags +from absl import logging + +import dataclasses +import numpy as np +import tensorflow as tf + +from official.nlp.bert import tokenization + +special_symbols = { + "": 0, + "": 1, + "": 2, + "": 3, + "": 4, + "": 5, + "": 6, + "": 7, + "": 8, +} + +FLAGS = flags.FLAGS + +flags.DEFINE_integer("seq_length", 512, + help="Sequence length.") +flags.DEFINE_integer("reuse_length", 256, + help="Number of token that can be reused as memory. " + "Could be half of `seq_len`.") +flags.DEFINE_string("input_file", None, + "Input raw text file (or comma-separated list of files).") +flags.DEFINE_string( + "save_dir", None, + "Directory for saving processed data.") +flags.DEFINE_string("sp_model_file", "", + "The path to the model used by sentence piece tokenizer.") +flags.DEFINE_bool("use_eod_token", True, + "Whether or not to include EOD tokens.") +flags.DEFINE_bool("bi_data", True, "Whether or not to use bi-directional data.") +flags.DEFINE_bool( + "do_lower_case", True, + "Whether to lower case the input text. Should be True for uncased " + "models and False for cased models.") +flags.DEFINE_integer("per_host_batch_size", 32, "Batch size per host.") +flags.DEFINE_integer("num_cores_per_host", 16, + "The number of (TPU) cores per host.") +flags.DEFINE_string("prefix", "", "Filename prefix.") +flags.DEFINE_string("suffix", "", "Filename suffix.") + +flags.DEFINE_integer("task_id", None, + "The id of the current task.") +flags.DEFINE_integer("num_tasks", None, + "The total number of tasks.") +flags.DEFINE_integer("num_passes", 1, "The number of times to run the script.") + + +@dataclasses.dataclass +class TrainingInstance: + """Representation of a single XLNet Pretraining instance.""" + data: Iterable[int] + segment_ids: Iterable[int] + boundary_indices: Iterable[int] + label: int + + def to_feature(self) -> Mapping[str, tf.train.Feature]: + feat = lambda x: tf.train.Feature(int64_list=tf.train.Int64List(value=x)) + return dict( + input_word_ids=feat(self.data), + input_type_ids=feat(self.segment_ids), + boundary_indices=feat(self.boundary_indices), + label=feat([self.label])) + + def to_example(self) -> tf.train.Example: + return tf.train.Example( + features=tf.train.Features(feature=self.to_feature())) + + def __str__(self): + def seq_to_str(seq): + return " ".join([str(x) for x in seq]) + + s = "" + s += "tokens: %s\n" % seq_to_str(self.data) + s += "segment_ids: %s\n" % seq_to_str(self.segment_ids) + s += "boundary_indices: %s\n" % seq_to_str(self.boundary_indices) + s += "label: %s\n" % self.label + s += "\n" + return s + + def __repr__(self): + return self.__str__() + + +def _preprocess_line(line: str, do_lower_case: bool = False) -> str: + """Preprocesses an individual raw text line. + + This function will: + - Remove extraneous spaces. + - Replace `` with ", and '' with ". + - Replaces accents. + - Applies lower casing. + + Args: + line: The input line to preprocess. + do_lower_case: Whether or not to lower case the text. + + Returns: + The preprocessed line. + + """ + line = " ".join(line.split()) + line = line.replace("``", "\"").replace("''", "\"") + + # Replace accents. + line = unicodedata.normalize("NFKD", line) + line = "".join([c for c in line if not unicodedata.combining(c)]) + + if do_lower_case: + line = line.lower() + return line + + +def preprocess_and_tokenize_input_files( + input_files: Iterable[str], + tokenizer: tokenization.FullSentencePieceTokenizer, + use_eod: bool = True, + do_lower_case: bool = False, + log_example_freq: int = 100000) -> List[Tuple[np.array, np.array]]: + """Preprocesses and encodes raw text from input files. + + This function preprocesses raw text and encodes them into tokens using a + `SentencePieceModel` tokenization method. This also provides the sentence + indicator for each token. + + Args: + input_files: The list of input file names. + tokenizer: The SentencePiece tokenizer that has the attribute `sp_model`. + use_eod: Whether or not to use an EOD indicator. If `False`, then EOD is + not included. + do_lower_case: Whether or not to apply lower casing during raw text + preprocessing. + log_example_freq: The optional field for how many lines to process before + emitting an info log. + + Returns: + The preprocessed list. Each entry in the list is a tuple consisting of + the token IDs and the sentence IDs. + + """ + all_data = [] + eod_symbol = special_symbols[""] + + total_number_of_lines = 0 + + # Input file format: + # (1) One sentence per line. These should ideally be actual sentences, not + # entire paragraphs or arbitrary spans of text. (Because we use the + # sentence boundaries for the "next sentence prediction" task). + # (2) Blank lines between documents. Document boundaries are needed so + # that the "next sentence prediction" task doesn't span between documents. + for input_file in input_files: + line_count = 0 + logging.info("Preprocessing %s", input_file) + + all_tokens = [] + all_sentence_ids = [] + + sentence_id = True + + with tf.io.gfile.GFile(input_file, "rb") as reader: + while True: + line = tokenization.convert_to_unicode(reader.readline()) + if not line: + break + + line_count += 1 + if line_count % log_example_freq == 0: + logging.info("Loading line %d", line_count) + + line = line.strip() + + if not line: + if use_eod: + token_ids = [eod_symbol] + sentence_id = not sentence_id + else: + continue + else: + preprocessed_line = _preprocess_line( + line=line, do_lower_case=do_lower_case) + token_ids = tokenization.encode_ids( + sp_model=tokenizer.sp_model, text=preprocessed_line) + + all_tokens.extend(token_ids) + all_sentence_ids.extend([sentence_id] * len(token_ids)) + sentence_id = not sentence_id + logging.info("Finished processing %s. Number of lines: %d", + input_file, line_count) + if line_count == 0: + continue + total_number_of_lines += line_count + all_tokens = np.array(all_tokens, dtype=np.int64) + all_sentence_ids = np.array(all_sentence_ids, dtype=np.bool) + all_data.append((all_tokens, all_sentence_ids)) + + logging.info("Completed text preprocessing. Total number of lines: %d", + total_number_of_lines) + return all_data + + +def _reshape_to_batch_dimensions( + tokens: np.array, + sentence_ids: np.array, + per_host_batch_size: int) -> Tuple[np.array, np.array]: + """Truncates and reshapes input data with a batch major dimension. + + Args: + tokens: The input token ids. This should have the same shape as + `sentence_ids`. + sentence_ids: The input sentence ids. This should have the same shape as + `token_ids`. + per_host_batch_size: The target per-host batch size. + + Returns: + The tuple of reshaped tokens and sentence_ids. + """ + num_steps = len(tokens) // per_host_batch_size + truncated_data_length = num_steps * per_host_batch_size + + logging.info("per_host_batch_size: %d", per_host_batch_size) + logging.info("num_steps: %d", num_steps) + def truncate_and_reshape(a): + return a[:truncated_data_length].reshape((per_host_batch_size, num_steps)) + + return (truncate_and_reshape(tokens), truncate_and_reshape(sentence_ids)) + + +def _create_a_and_b_segments( + tokens: np.array, + sentence_ids: np.array, + begin_index: int, + total_length: int, + no_cut_probability: float = 0.5): + """Splits segments A and B from a single instance of tokens and sentence ids. + + Args: + tokens: The 1D input token ids. This represents an individual entry within a + batch. + sentence_ids: The 1D input sentence ids. This represents an indivdual entry + within a batch. This should be the same length as `tokens`. + begin_index: The reference beginning index to split data. + total_length: The target combined length of segments A and B. + no_cut_probability: The probability of not cutting a segment despite + a cut possibly existing. + + Returns: + A tuple consisting of A data, B data, and label. + + """ + data_length = tokens.shape[0] + if begin_index + total_length >= data_length: + logging.info("[_create_segments]: begin_index %d + total_length %d >= " + "data_length %d", begin_index, total_length, data_length) + return None + + end_index = begin_index + 1 + cut_indices = [] + + # Identify all indices where sentence IDs change from one to the next. + while end_index < data_length: + if sentence_ids[end_index] != sentence_ids[end_index - 1]: + if end_index - begin_index >= total_length: + break + cut_indices.append(end_index) + end_index += 1 + + a_begin = begin_index + + if not cut_indices or random.random() < no_cut_probability: + # Segments A and B are contained within the same sentence. + label = 0 + if not cut_indices: + a_end = end_index + else: + a_end = random.choice(cut_indices) + b_length = max(1, total_length - (a_end - a_begin)) + b_begin = random.randint(0, data_length - 1 - b_length) + b_end = b_begin + b_length + + while b_begin > 0 and sentence_ids[b_begin - 1] == sentence_ids[b_begin]: + b_begin -= 1 + while (b_end < data_length - 1 and + sentence_ids[b_end - 1] == sentence_ids[b_end]): + b_end += 1 + else: + # Segments A and B are different sentences. + label = 1 + a_end = random.choice(cut_indices) + b_begin = a_end + b_end = end_index + + while a_end - a_begin + b_end - b_begin > total_length: + if a_end - a_begin > b_end - b_begin: + # Delete only the right side for the LM objective. + a_end -= 1 + else: + b_end -= 1 + if a_end >= data_length or b_end >= data_length: + logging.info("[_create_segments]: a_end %d or b_end %d >= data_length %d", + a_end, b_end, data_length) + return None + + a_data = tokens[a_begin: a_end] + b_data = tokens[b_begin: b_end] + return a_data, b_data, label + + +def _is_functional_piece(piece: str) -> bool: + return piece != "" and piece.startswith("<") and piece.endswith(">") + + +def _is_start_piece(piece: str) -> bool: + special_pieces = set(list('!"#$%&\"()*+,-./:;?@[\\]^_`{|}~')) + if (piece.startswith("▁") or piece in special_pieces): + return True + else: + return False + + +def _get_boundary_indices( + data: np.array, + tokenizer: tokenization.FullSentencePieceTokenizer) -> np.array: + """Gets the boundary indices of whole words.""" + seq_length = len(data) + boundary_indices = [] + for index, piece in enumerate(tokenizer.convert_ids_to_tokens(data.tolist())): + if _is_start_piece(piece) and not _is_functional_piece(piece): + boundary_indices.append(index) + boundary_indices.append(seq_length) + return boundary_indices + + +def _convert_tokens_to_instances( + tokens: np.array, + sentence_ids: np.array, + per_host_batch_size: int, + seq_length: int, + reuse_length: int, + bi_data: bool, + tokenizer: tokenization.FullSentencePieceTokenizer, + num_cores_per_host: int = 0, + logging_frequency: int = 500) -> List[TrainingInstance]: + """Converts tokens and sentence IDs into individual training instances. + + The format of data in the XLNet pretraining task is very similar to the + BERT pretraining task. Two segments A and B are randomly sampled, and the + contatenation of A and B into a single sequence is used to perform + language modeling. + + To create an XLNet Pretraining instance from a single long sequence, S: + - Create a segment of length `reuse_length`. This first segment represents + past tokens. During modeling, this segment is used to cache obtained + content representations for the segment recurrence mechanism. + - Similar to BERT, create a segment of length `seq_length` - `reuse_length` + composed of A and B segments. + For XLNet, the order is "A", "SEP", "B", "SEP", "CLS". + + Args: + tokens: All tokens concatenated into a single list. + sentence_ids: All sentence IDs concatenated into a single list. + per_host_batch_size: The target batch size per host. + seq_length: The max sequence length. + reuse_length: The number of tokens to use from the previous segment. + bi_data: Whether or not to use bidirectional data. + tokenizer: The SentencePiece tokenizer that has the attribute `sp_model`. + num_cores_per_host: The number of cores per host. This is required if + `bi_data` = `True`. + logging_frequency: The frequency at which to log status updates. + + Returns: + A list of `TrainingInstance` objects. + """ + instances = [] + + per_core_batch_size = (per_host_batch_size // num_cores_per_host + if bi_data else None) + + if bi_data: + logging.info("Bi-directional data enabled.") + assert per_host_batch_size % (2 * num_cores_per_host) == 0 + forward_tokens, forward_sentence_ids = _reshape_to_batch_dimensions( + tokens=tokens, + sentence_ids=sentence_ids, + per_host_batch_size=per_host_batch_size // 2) + forward_data_shape = (num_cores_per_host, 1, per_core_batch_size // 2, -1) + + forward_tokens = forward_tokens.reshape(forward_data_shape) + forward_sentence_ids = forward_sentence_ids.reshape(forward_data_shape) + + backwards_tokens = forward_tokens[:, :, :, ::-1] + backwards_sentence_ids = forward_sentence_ids[:, :, :, ::-1] + + tokens = np.concatenate([forward_tokens, backwards_tokens], 1).reshape( + per_host_batch_size, -1) + sentence_ids = np.concatenate( + [forward_sentence_ids, backwards_sentence_ids]).reshape( + per_host_batch_size, -1) + else: + logging.info("Bi-directional data disabled.") + tokens, sentence_ids = _reshape_to_batch_dimensions( + tokens=tokens, + sentence_ids=sentence_ids, + per_host_batch_size=per_host_batch_size) + + logging.info("Tokens shape: %s", tokens.shape) + + data_length = tokens.shape[1] + sep = np.array([special_symbols[""]], dtype=np.int64) + cls = np.array([special_symbols[""]], dtype=np.int64) + # 2 sep, 1 cls + num_special_tokens = 3 + + data_index = 0 + batch_number = 0 + step_size = reuse_length if reuse_length else seq_length + num_batches = math.ceil(data_length / step_size) + + while data_index + seq_length <= data_length: + if batch_number % logging_frequency == 0: + logging.info("Processing batch %d of %d", batch_number, num_batches) + + for batch_index in range(per_host_batch_size): + previous_segment_tokens = tokens[ + batch_index, data_index: data_index + reuse_length] + + results = _create_a_and_b_segments( + tokens=tokens[batch_index], + sentence_ids=sentence_ids[batch_index], + begin_index=data_index + reuse_length, + total_length=seq_length - reuse_length - num_special_tokens) + + if results is None: + logging.info("Stopping at data index: %d", data_index) + break + a_data, b_data, label = results + + data = np.concatenate( + [previous_segment_tokens, a_data, sep, b_data, sep, cls]) + a_length = a_data.shape[0] + b_length = b_data.shape[0] + segment_ids = ([0] * (reuse_length + a_length) + [0] + + [1] * b_length + [1] + [2]) + boundary_indices = _get_boundary_indices(tokenizer=tokenizer, + data=data) + assert len(data) == seq_length + assert len(segment_ids) == seq_length + assert len(boundary_indices) > 0 # pylint: disable=g-explicit-length-test + + instances.append(TrainingInstance( + data=data, + segment_ids=segment_ids, + boundary_indices=boundary_indices, + label=label)) + batch_number += 1 + data_index += step_size + return instances + + +def write_instances_to_tfrecord( + instances: Iterable[TrainingInstance], + save_path: str): + """Writes instances to TFRecord.""" + record_writer = tf.io.TFRecordWriter(save_path) + logging.info("Start writing to %s.", save_path) + + for i, instance in enumerate(instances): + if i < 5: + logging.info("Instance %d: %s", i, str(instance)) + record_writer.write(instance.to_example().SerializeToString()) + + record_writer.close() + logging.info("Done writing %s.", save_path) + + +def shuffle_and_combine_preprocessed_data( + all_data: List[Tuple[np.array, np.array]]) -> Tuple[np.array, np.array]: + """Shuffles and combines preprocessed token/sentence IDs from documents.""" + document_permutation = np.random.permutation(len(all_data)) + + previous_sentence_id = None + + all_tokens, all_sentence_ids = [], [] + for document_index in document_permutation: + tokens, sentence_ids = all_data[document_index] + # pylint: disable=g-explicit-length-test + if len(tokens) == 0: + continue + if (previous_sentence_id is not None and + sentence_ids[0] == previous_sentence_id): + sentence_ids = np.logical_not(sentence_ids) + + all_tokens.append(tokens) + all_sentence_ids.append(sentence_ids) + + previous_sentence_id = sentence_ids[-1] + + return np.concatenate(all_tokens), np.concatenate(all_sentence_ids) + + +def get_tfrecord_name( + per_host_batch_size: int, + num_cores_per_host: int, + seq_length: int, + bi_data: bool, + reuse_length: int, + do_lower_case: bool, + use_eod_token: bool, + prefix: str = "", + suffix: str = "", + pass_id: int = 0, + num_passes: int = 1, + task_id: int = None, + num_tasks: int = None) -> str: + """Formats the resulting TFRecord name based on provided inputs.""" + components = [] + if prefix: + components.append(prefix) + components.append("seqlen-{}".format(seq_length)) + if reuse_length == 0: + components.append("memless") + else: + components.append("reuse-{}".format(reuse_length)) + components.append("bs-{}".format(per_host_batch_size)) + components.append("cores-{}".format(num_cores_per_host)) + + if do_lower_case: + components.append("uncased") + else: + components.append("cased") + if use_eod_token: + components.append("eod") + if bi_data: + components.append("bi") + else: + components.append("uni") + + if suffix: + components.append(suffix) + + s = "_".join(components) + ".tfrecord" + if num_passes == 1 and task_id is None: + return s + + if task_id is None: + num_tasks = 1 + task_id = 0 + + current_shard = task_id * num_passes + pass_id + total_shards = num_tasks * num_passes + return s + "-{}-of-{}".format(current_shard, total_shards) + + +def create_tfrecords( + tokenizer: tokenization.FullSentencePieceTokenizer, + input_file_or_files: str, + use_eod_token: bool, + do_lower_case: bool, + per_host_batch_size: int, + seq_length: int, + reuse_length: int, + bi_data: bool, + num_cores_per_host: int, + save_dir: str, + prefix: str = "", + suffix: str = "", + num_tasks: Optional[int] = None, + task_id: Optional[int] = None, + num_passes: int = 1): + """Runs the end-to-end preprocessing pipeline.""" + + logging.info("Input configuration:") + logging.info("input file(s): %s", input_file_or_files) + logging.info("use_eod_token: %s", use_eod_token) + logging.info("do_lower_case: %s", do_lower_case) + logging.info("per_host_batch_size: %d", per_host_batch_size) + logging.info("seq_length: %d", seq_length) + logging.info("reuse_length: %d", reuse_length) + logging.info("bi_data: %s", bi_data) + logging.info("num_cores_per_host: %d", num_cores_per_host) + logging.info("save_dir: %s", save_dir) + if task_id is not None and num_tasks is not None: + logging.info("task_id: %d", task_id) + logging.info("num_tasks: %d", num_tasks) + + input_files = [] + for input_pattern in input_file_or_files.split(","): + input_files.extend(tf.io.gfile.glob(input_pattern)) + + logging.info("*** Reading from input files ***") + for input_file in input_files: + logging.info(" %s", input_file) + + logging.info("Shuffling the files with a fixed random seed.") + np.random.shuffle(input_files) + if num_tasks is not None: + assert task_id is not None + logging.info("Total number of input files: %d", len(input_files)) + logging.info("Splitting into %d shards of %d files each.", + num_tasks, len(input_files) // num_tasks) + input_files = input_files[task_id::num_tasks] + + all_data = preprocess_and_tokenize_input_files( + input_files=input_files, + tokenizer=tokenizer, + use_eod=use_eod_token, + do_lower_case=do_lower_case) + for pass_id in range(num_passes): + logging.info("Beginning pass %d of %d", pass_id, num_passes) + tokens, sentence_ids = shuffle_and_combine_preprocessed_data(all_data) + + assert len(tokens) == len(sentence_ids) + + filename = get_tfrecord_name( + per_host_batch_size=per_host_batch_size, + num_cores_per_host=num_cores_per_host, + seq_length=seq_length, + bi_data=bi_data, + use_eod_token=use_eod_token, + reuse_length=reuse_length, + do_lower_case=do_lower_case, + prefix=prefix, + suffix=suffix, + pass_id=pass_id, + num_passes=num_passes, + num_tasks=num_tasks, + task_id=task_id) + save_path = os.path.join(save_dir, filename) + if os.path.exists(save_path): + # If the path already exists, then we were probably preempted but + # previously wrote this file. + logging.info("%s already exists, skipping this batch.", save_path) + else: + instances = _convert_tokens_to_instances( + tokenizer=tokenizer, + tokens=tokens, + sentence_ids=sentence_ids, + per_host_batch_size=per_host_batch_size, + seq_length=seq_length, + reuse_length=reuse_length, + bi_data=bi_data, + num_cores_per_host=num_cores_per_host) + write_instances_to_tfrecord(instances=instances, save_path=save_path) + + if task_id is None or task_id == 0: + corpus_info = { + "vocab_size": 32000, + "per_host_batch_size": per_host_batch_size, + "num_cores_per_host": num_cores_per_host, + "seq_length": seq_length, + "reuse_length": reuse_length, + "do_lower_case": do_lower_case, + "bi_data": bi_data, + "use_eod_token": use_eod_token, + } + corpus_fname = os.path.basename(filename) + ".json" + corpus_destination = os.path.join(save_dir, corpus_fname) + logging.info("Saving corpus info to %s", corpus_destination) + + with tf.io.gfile.GFile(corpus_destination, "w") as fp: + json.dump(corpus_info, fp) + + +def main(_): + tokenizer = tokenization.FullSentencePieceTokenizer(FLAGS.sp_model_file) + create_tfrecords( + tokenizer=tokenizer, + input_file_or_files=FLAGS.input_file, + use_eod_token=FLAGS.use_eod_token, + do_lower_case=FLAGS.do_lower_case, + per_host_batch_size=FLAGS.per_host_batch_size, + seq_length=FLAGS.seq_length, + reuse_length=FLAGS.reuse_length, + bi_data=FLAGS.bi_data, + num_cores_per_host=FLAGS.num_cores_per_host, + save_dir=FLAGS.save_dir, + prefix=FLAGS.prefix, + suffix=FLAGS.suffix, + num_tasks=FLAGS.num_tasks, + task_id=FLAGS.task_id, + num_passes=FLAGS.num_passes) + + +if __name__ == "__main__": + np.random.seed(0) + logging.set_verbosity(logging.INFO) + app.run(main) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/create_xlnet_pretraining_data_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/create_xlnet_pretraining_data_test.py new file mode 100644 index 0000000000000000000000000000000000000000..5630411a7eb0e92b2baf6e203547d1c9063ebd79 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/create_xlnet_pretraining_data_test.py @@ -0,0 +1,355 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for official.nlp.data.create_xlnet_pretraining_data.""" +import os +import tempfile +from typing import List + +from absl import logging +from absl.testing import parameterized + +import numpy as np +import tensorflow as tf + +from official.nlp.data import create_xlnet_pretraining_data as cpd + +_VOCAB_WORDS = ["vocab_1", "vocab_2"] + + +# pylint: disable=invalid-name +def _create_files( + temp_dir: str, file_contents: List[List[str]]) -> List[str]: + """Writes arbitrary documents into files.""" + root_dir = tempfile.mkdtemp(dir=temp_dir) + files = [] + + for i, file_content in enumerate(file_contents): + destination = os.path.join(root_dir, "%d.txt" % i) + with open(destination, "wb") as f: + for line in file_content: + f.write(line.encode("utf-8")) + files.append(destination) + return files + + +def _get_mock_tokenizer(): + """Creates a mock tokenizer.""" + + class MockSpieceModel: + """Mock Spiece model for testing.""" + + def __init__(self): + self._special_piece_to_id = { + "": 0, + } + for piece in set(list('!"#$%&\"()*+,-./:;?@[\\]^_`{|}~')): + self._special_piece_to_id[piece] = 1 + + def EncodeAsPieces(self, inputs: str) -> List[str]: + return inputs + + def SampleEncodeAsPieces(self, + inputs: str, + nbest_size: int, + theta: float) -> List[str]: + del nbest_size, theta + return inputs + + def PieceToId(self, piece: str) -> int: + return ord(piece[0]) + + def IdToPiece(self, id_: int) -> str: + return chr(id_) * 3 + + class Tokenizer: + """Mock Tokenizer for testing.""" + + def __init__(self): + self.sp_model = MockSpieceModel() + + def convert_ids_to_tokens(self, ids: List[int]) -> List[str]: + return [self.sp_model.IdToPiece(id_) for id_ in ids] + + return Tokenizer() + + +class PreprocessDataTest(tf.test.TestCase): + + def test_remove_extraneous_space(self): + line = " abc " + output = cpd._preprocess_line(line) + self.assertEqual(output, "abc") + + def test_symbol_replacements(self): + self.assertEqual(cpd._preprocess_line("``abc``"), "\"abc\"") + self.assertEqual(cpd._preprocess_line("''abc''"), "\"abc\"") + + def test_accent_replacements(self): + self.assertEqual(cpd._preprocess_line("åbc"), "abc") + + def test_lower_case(self): + self.assertEqual(cpd._preprocess_line("ABC", do_lower_case=True), "abc") + + def test_end_to_end(self): + self.assertEqual( + cpd._preprocess_line("HelLo ``wórLd``", do_lower_case=True), + "hello \"world\"") + + +class PreprocessAndTokenizeFilesTest(tf.test.TestCase): + + def test_basic_end_to_end(self): + documents = [ + [ + "This is sentence 1.\n", + "This is sentence 2.\n", + "Sentence 3 is what this is.\n", + ], + [ + "This is the second document.\n", + "This is the second line of the second document.\n" + ], + ] + input_files = _create_files(temp_dir=self.get_temp_dir(), + file_contents=documents) + all_data = cpd.preprocess_and_tokenize_input_files( + input_files=input_files, + tokenizer=_get_mock_tokenizer(), + log_example_freq=1) + + self.assertEqual(len(all_data), len(documents)) + for token_ids, sentence_ids in all_data: + self.assertEqual(len(token_ids), len(sentence_ids)) + + def test_basic_correctness(self): + documents = [["a\n", "b\n", "c\n"]] + input_files = _create_files(temp_dir=self.get_temp_dir(), + file_contents=documents) + all_data = cpd.preprocess_and_tokenize_input_files( + input_files=input_files, + tokenizer=_get_mock_tokenizer(), + log_example_freq=1) + + token_ids, sentence_ids = all_data[0] + + self.assertAllClose(token_ids, [97, 98, 99]) + self.assertAllClose(sentence_ids, [True, False, True]) + + def test_correctness_with_spaces_and_accents(self): + documents = [[ + " å \n", + "b \n", + " c \n", + ]] + input_files = _create_files(temp_dir=self.get_temp_dir(), + file_contents=documents) + all_data = cpd.preprocess_and_tokenize_input_files( + input_files=input_files, + tokenizer=_get_mock_tokenizer(), + log_example_freq=1) + + token_ids, sentence_ids = all_data[0] + + self.assertAllClose(token_ids, [97, 98, 99]) + self.assertAllClose(sentence_ids, [True, False, True]) + + +class BatchReshapeTests(tf.test.TestCase): + + def test_basic_functionality(self): + per_host_batch_size = 3 + mock_shape = (20,) + + # Should truncate and reshape. + expected_result_shape = (3, 6) + + tokens = np.zeros(mock_shape) + sentence_ids = np.zeros(mock_shape) + + reshaped_data = cpd._reshape_to_batch_dimensions( + tokens=tokens, + sentence_ids=sentence_ids, + per_host_batch_size=per_host_batch_size) + for values in reshaped_data: + self.assertEqual(len(values.flatten()) % per_host_batch_size, 0) + self.assertAllClose(values.shape, expected_result_shape) + + +class CreateSegmentsTest(tf.test.TestCase): + + def test_basic_functionality(self): + data_length = 10 + tokens = np.arange(data_length) + sentence_ids = np.concatenate([np.zeros(data_length // 2), + np.ones(data_length // 2)]) + begin_index = 0 + total_length = 8 + a_data, b_data, label = cpd._create_a_and_b_segments( + tokens=tokens, + sentence_ids=sentence_ids, + begin_index=begin_index, + total_length=total_length, + no_cut_probability=0.) + self.assertAllClose(a_data, [0, 1, 2, 3]) + self.assertAllClose(b_data, [5, 6, 7, 8]) + self.assertEqual(label, 1) + + def test_no_cut(self): + data_length = 10 + tokens = np.arange(data_length) + sentence_ids = np.zeros(data_length) + + begin_index = 0 + total_length = 8 + a_data, b_data, label = cpd._create_a_and_b_segments( + tokens=tokens, + sentence_ids=sentence_ids, + begin_index=begin_index, + total_length=total_length, + no_cut_probability=0.) + self.assertGreater(len(a_data), 0) + self.assertGreater(len(b_data), 0) + self.assertEqual(label, 0) + + def test_no_cut_with_probability(self): + data_length = 10 + tokens = np.arange(data_length) + sentence_ids = np.concatenate([np.zeros(data_length // 2), + np.ones(data_length // 2)]) + begin_index = 0 + total_length = 8 + a_data, b_data, label = cpd._create_a_and_b_segments( + tokens=tokens, + sentence_ids=sentence_ids, + begin_index=begin_index, + total_length=total_length, + no_cut_probability=1.) + self.assertGreater(len(a_data), 0) + self.assertGreater(len(b_data), 0) + self.assertEqual(label, 0) + + +class CreateInstancesTest(tf.test.TestCase): + """Tests conversions of Token/Sentence IDs to training instances.""" + + def test_basic(self): + data_length = 12 + tokens = np.arange(data_length) + sentence_ids = np.zeros(data_length) + seq_length = 8 + instances = cpd._convert_tokens_to_instances( + tokens=tokens, + sentence_ids=sentence_ids, + per_host_batch_size=2, + seq_length=seq_length, + reuse_length=4, + tokenizer=_get_mock_tokenizer(), + bi_data=False, + num_cores_per_host=1, + logging_frequency=1) + for instance in instances: + self.assertEqual(len(instance.data), seq_length) + self.assertEqual(len(instance.segment_ids), seq_length) + self.assertIsInstance(instance.label, int) + self.assertIsInstance(instance.boundary_indices, list) + + +class TFRecordPathTests(tf.test.TestCase): + + def test_basic(self): + base_kwargs = dict( + per_host_batch_size=1, + num_cores_per_host=1, + seq_length=2, + reuse_length=1) + + config1 = dict( + prefix="test", + suffix="", + bi_data=True, + use_eod_token=False, + do_lower_case=True) + config1.update(base_kwargs) + expectation1 = "test_seqlen-2_reuse-1_bs-1_cores-1_uncased_bi.tfrecord" + self.assertEqual(cpd.get_tfrecord_name(**config1), expectation1) + + config2 = dict( + prefix="", + suffix="test", + bi_data=False, + use_eod_token=False, + do_lower_case=False) + config2.update(base_kwargs) + expectation2 = "seqlen-2_reuse-1_bs-1_cores-1_cased_uni_test.tfrecord" + self.assertEqual(cpd.get_tfrecord_name(**config2), expectation2) + + config3 = dict( + prefix="", + suffix="", + use_eod_token=True, + bi_data=False, + do_lower_case=True) + config3.update(base_kwargs) + expectation3 = "seqlen-2_reuse-1_bs-1_cores-1_uncased_eod_uni.tfrecord" + self.assertEqual(cpd.get_tfrecord_name(**config3), expectation3) + + +class TestCreateTFRecords(parameterized.TestCase, tf.test.TestCase): + + @parameterized.named_parameters( + ("bi_data_only", True, False, False), + ("eod_token_only", False, True, True), + ("lower_case_only", False, False, True), + ("all_enabled", True, True, True), + ) + def test_end_to_end(self, + bi_data: bool, + use_eod_token: bool, + do_lower_case: bool): + tokenizer = _get_mock_tokenizer() + + num_documents = 5 + sentences_per_document = 10 + document_length = 50 + + documents = [ + ["a " * document_length for _ in range(sentences_per_document)] + for _ in range(num_documents)] + + save_dir = tempfile.mkdtemp(dir=self.get_temp_dir()) + files = _create_files(temp_dir=self.get_temp_dir(), file_contents=documents) + + cpd.create_tfrecords( + tokenizer=tokenizer, + input_file_or_files=",".join(files), + use_eod_token=use_eod_token, + do_lower_case=do_lower_case, + per_host_batch_size=8, + seq_length=8, + reuse_length=4, + bi_data=bi_data, + num_cores_per_host=2, + save_dir=save_dir) + + self.assertTrue(any(filter(lambda x: x.endswith(".json"), + os.listdir(save_dir)))) + self.assertTrue(any(filter(lambda x: x.endswith(".tfrecord"), + os.listdir(save_dir)))) + + +if __name__ == "__main__": + np.random.seed(0) + logging.set_verbosity(logging.INFO) + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/data_loader.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/data_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..2b181270658f42f0819f01fbed5af7989b1d3e5d --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/data_loader.py @@ -0,0 +1,48 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""An abstraction that NLP models define input pipelines.""" + +import abc +from typing import Optional + +import tensorflow as tf + + +class DataLoader(metaclass=abc.ABCMeta): + """An abstract class defining the APIs for tf.data input pipeline.""" + + @abc.abstractmethod + def load( + self, + input_context: Optional[tf.distribute.InputContext] = None + ) -> tf.data.Dataset: + """Implements DataLoader load method. + + Builds the entire input pipeline inside the load method. Users can define + states inside the DataLoader class and returns a tf.data dataset + object. + + Args: + input_context: This is a context class that is passed to the user's input + function and contains information about the compute replicas and input + pipelines. This object is used for multi-host inputs and passed by the + distribution strategy. + + Returns: + A per-host tf.data dataset. Note that, we usually create the distributed + dataset through the load method, so we should not directly return a + distributed dataset here. + """ + pass diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/data_loader_factory.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/data_loader_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..9602ea295283e5490d1bcb5cc67df9f99ebdb0ca --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/data_loader_factory.py @@ -0,0 +1,58 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A global factory to access NLP registered data loaders.""" + +from official.core import registry + +_REGISTERED_DATA_LOADER_CLS = {} + + +def register_data_loader_cls(data_config_cls): + """Decorates a factory of DataLoader for lookup by a subclass of DataConfig. + + This decorator supports registration of data loaders as follows: + + ``` + @dataclasses.dataclass + class MyDataConfig(DataConfig): + # Add fields here. + pass + + @register_data_loader_cls(MyDataConfig) + class MyDataLoader: + # Inherits def __init__(self, data_config). + pass + + my_data_config = MyDataConfig() + + # Returns MyDataLoader(my_data_config). + my_loader = get_data_loader(my_data_config) + ``` + + Args: + data_config_cls: a subclass of DataConfig (*not* an instance + of DataConfig). + + Returns: + A callable for use as class decorator that registers the decorated class + for creation from an instance of data_config_cls. + """ + return registry.register(_REGISTERED_DATA_LOADER_CLS, data_config_cls) + + +def get_data_loader(data_config): + """Creates a data_loader from data_config.""" + return registry.lookup(_REGISTERED_DATA_LOADER_CLS, data_config.__class__)( + data_config) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/data_loader_factory_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/data_loader_factory_test.py new file mode 100644 index 0000000000000000000000000000000000000000..8aa86757df64a445692ce4bf8ff64e6649b6dfa6 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/data_loader_factory_test.py @@ -0,0 +1,45 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for official.nlp.data.data_loader_factory.""" + +import dataclasses +import tensorflow as tf + +from official.core import config_definitions as cfg +from official.nlp.data import data_loader_factory + + +@dataclasses.dataclass +class MyDataConfig(cfg.DataConfig): + is_training: bool = True + + +@data_loader_factory.register_data_loader_cls(MyDataConfig) +class MyDataLoader: + + def __init__(self, params): + self.params = params + + +class DataLoaderFactoryTest(tf.test.TestCase): + + def test_register_and_load(self): + train_config = MyDataConfig() + train_loader = data_loader_factory.get_data_loader(train_config) + self.assertTrue(train_loader.params.is_training) + + +if __name__ == "__main__": + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/dual_encoder_dataloader.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/dual_encoder_dataloader.py new file mode 100644 index 0000000000000000000000000000000000000000..af9f1090fceda600373ef785d91fd215e8621fd8 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/dual_encoder_dataloader.py @@ -0,0 +1,145 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Loads dataset for the dual encoder (retrieval) task.""" +import functools +import itertools +from typing import Iterable, Mapping, Optional, Tuple + +import dataclasses +import tensorflow as tf +import tensorflow_hub as hub + +from official.core import config_definitions as cfg +from official.core import input_reader +from official.nlp.data import data_loader +from official.nlp.data import data_loader_factory +from official.nlp.modeling import layers + + +@dataclasses.dataclass +class DualEncoderDataConfig(cfg.DataConfig): + """Data config for dual encoder task (tasks/dual_encoder).""" + # Either set `input_path`... + input_path: str = '' + # ...or `tfds_name` and `tfds_split` to specify input. + tfds_name: str = '' + tfds_split: str = '' + global_batch_size: int = 32 + # Either build preprocessing with Python code by specifying these values... + vocab_file: str = '' + lower_case: bool = True + # ...or load preprocessing from a SavedModel at this location. + preprocessing_hub_module_url: str = '' + + left_text_fields: Tuple[str] = ('left_input',) + right_text_fields: Tuple[str] = ('right_input',) + is_training: bool = True + seq_length: int = 128 + + +@data_loader_factory.register_data_loader_cls(DualEncoderDataConfig) +class DualEncoderDataLoader(data_loader.DataLoader): + """A class to load dataset for dual encoder task (tasks/dual_encoder).""" + + def __init__(self, params): + if bool(params.tfds_name) == bool(params.input_path): + raise ValueError('Must specify either `tfds_name` and `tfds_split` ' + 'or `input_path`.') + if bool(params.vocab_file) == bool(params.preprocessing_hub_module_url): + raise ValueError('Must specify exactly one of vocab_file (with matching ' + 'lower_case flag) or preprocessing_hub_module_url.') + self._params = params + self._seq_length = params.seq_length + self._left_text_fields = params.left_text_fields + self._right_text_fields = params.right_text_fields + + if params.preprocessing_hub_module_url: + preprocessing_hub_module = hub.load(params.preprocessing_hub_module_url) + self._tokenizer = preprocessing_hub_module.tokenize + self._pack_inputs = functools.partial( + preprocessing_hub_module.bert_pack_inputs, + seq_length=params.seq_length) + else: + self._tokenizer = layers.BertTokenizer( + vocab_file=params.vocab_file, lower_case=params.lower_case) + self._pack_inputs = layers.BertPackInputs( + seq_length=params.seq_length, + special_tokens_dict=self._tokenizer.get_special_tokens_dict()) + + def _decode(self, record: tf.Tensor): + """Decodes a serialized tf.Example.""" + name_to_features = { + x: tf.io.FixedLenFeature([], tf.string) + for x in itertools.chain( + *[self._left_text_fields, self._right_text_fields]) + } + example = tf.io.parse_single_example(record, name_to_features) + + # tf.Example only supports tf.int64, but the TPU only supports tf.int32. + # So cast all int64 to int32. + for name in example: + t = example[name] + if t.dtype == tf.int64: + t = tf.cast(t, tf.int32) + example[name] = t + + return example + + def _bert_tokenize( + self, record: Mapping[str, tf.Tensor], + text_fields: Iterable[str]) -> Tuple[tf.Tensor, tf.Tensor, tf.Tensor]: + """Tokenize the input in text_fields using BERT tokenizer. + + Args: + record: A tfexample record contains the features. + text_fields: A list of fields to be tokenzied. + + Returns: + The tokenized features in a tuple of (input_word_ids, input_mask, + input_type_ids). + """ + segments_text = [record[x] for x in text_fields] + segments_tokens = [self._tokenizer(s) for s in segments_text] + segments = [tf.cast(x.merge_dims(1, 2), tf.int32) for x in segments_tokens] + return self._pack_inputs(segments) + + def _bert_preprocess( + self, record: Mapping[str, tf.Tensor]) -> Mapping[str, tf.Tensor]: + """Perform the bert word piece tokenization for left and right inputs.""" + + def _switch_prefix(string, old, new): + if string.startswith(old): return new + string[len(old):] + raise ValueError('Expected {} to start with {}'.format(string, old)) + + def _switch_key_prefix(d, old, new): + return {_switch_prefix(key, old, new): value for key, value in d.items()} + + model_inputs = _switch_key_prefix( + self._bert_tokenize(record, self._left_text_fields), + 'input_', 'left_') + model_inputs.update(_switch_key_prefix( + self._bert_tokenize(record, self._right_text_fields), + 'input_', 'right_')) + return model_inputs + + def load(self, input_context: Optional[tf.distribute.InputContext] = None): + """Returns a tf.dataset.Dataset.""" + reader = input_reader.InputReader( + params=self._params, + # Skip `decoder_fn` for tfds input. + decoder_fn=self._decode if self._params.input_path else None, + dataset_fn=tf.data.TFRecordDataset, + postprocess_fn=self._bert_preprocess) + return reader.read(input_context) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/dual_encoder_dataloader_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/dual_encoder_dataloader_test.py new file mode 100644 index 0000000000000000000000000000000000000000..358b0d9635c747a72e0e8f40951f11eb2c5755a0 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/dual_encoder_dataloader_test.py @@ -0,0 +1,131 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for official.nlp.data.dual_encoder_dataloader.""" +import os + +from absl.testing import parameterized +import tensorflow as tf + +from official.nlp.data import dual_encoder_dataloader + + +_LEFT_FEATURE_NAME = 'left_input' +_RIGHT_FEATURE_NAME = 'right_input' + + +def _create_fake_dataset(output_path): + """Creates a fake dataset contains examples for training a dual encoder model. + + The created dataset contains examples with two byteslist features keyed by + _LEFT_FEATURE_NAME and _RIGHT_FEATURE_NAME. + + Args: + output_path: The output path of the fake dataset. + """ + def create_str_feature(values): + return tf.train.Feature(bytes_list=tf.train.BytesList(value=values)) + + with tf.io.TFRecordWriter(output_path) as writer: + for _ in range(100): + features = {} + features[_LEFT_FEATURE_NAME] = create_str_feature([b'hello world.']) + features[_RIGHT_FEATURE_NAME] = create_str_feature([b'world hello.']) + + tf_example = tf.train.Example( + features=tf.train.Features(feature=features)) + writer.write(tf_example.SerializeToString()) + + +def _make_vocab_file(vocab, output_path): + with tf.io.gfile.GFile(output_path, 'w') as f: + f.write('\n'.join(vocab + [''])) + + +class DualEncoderDataTest(tf.test.TestCase, parameterized.TestCase): + + def test_load_dataset(self): + seq_length = 16 + batch_size = 10 + train_data_path = os.path.join(self.get_temp_dir(), 'train.tf_record') + vocab_path = os.path.join(self.get_temp_dir(), 'vocab.txt') + + _create_fake_dataset(train_data_path) + _make_vocab_file( + ['[PAD]', '[UNK]', '[CLS]', '[SEP]', 'he', '#llo', 'world'], vocab_path) + + data_config = dual_encoder_dataloader.DualEncoderDataConfig( + input_path=train_data_path, + seq_length=seq_length, + vocab_file=vocab_path, + lower_case=True, + left_text_fields=(_LEFT_FEATURE_NAME,), + right_text_fields=(_RIGHT_FEATURE_NAME,), + global_batch_size=batch_size) + dataset = dual_encoder_dataloader.DualEncoderDataLoader( + data_config).load() + features = next(iter(dataset)) + self.assertCountEqual( + ['left_word_ids', 'left_mask', 'left_type_ids', 'right_word_ids', + 'right_mask', 'right_type_ids'], + features.keys()) + self.assertEqual(features['left_word_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['left_mask'].shape, (batch_size, seq_length)) + self.assertEqual(features['left_type_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['right_word_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['right_mask'].shape, (batch_size, seq_length)) + self.assertEqual(features['right_type_ids'].shape, (batch_size, seq_length)) + + @parameterized.parameters(False, True) + def test_load_tfds(self, use_preprocessing_hub): + seq_length = 16 + batch_size = 10 + if use_preprocessing_hub: + vocab_path = '' + preprocessing_hub = ( + 'https://tfhub.dev/tensorflow/bert_multi_cased_preprocess/3') + else: + vocab_path = os.path.join(self.get_temp_dir(), 'vocab.txt') + _make_vocab_file( + ['[PAD]', '[UNK]', '[CLS]', '[SEP]', 'he', '#llo', 'world'], + vocab_path) + preprocessing_hub = '' + + data_config = dual_encoder_dataloader.DualEncoderDataConfig( + tfds_name='para_crawl/enmt', + tfds_split='train', + seq_length=seq_length, + vocab_file=vocab_path, + lower_case=True, + left_text_fields=('en',), + right_text_fields=('mt',), + preprocessing_hub_module_url=preprocessing_hub, + global_batch_size=batch_size) + dataset = dual_encoder_dataloader.DualEncoderDataLoader( + data_config).load() + features = next(iter(dataset)) + self.assertCountEqual( + ['left_word_ids', 'left_mask', 'left_type_ids', 'right_word_ids', + 'right_mask', 'right_type_ids'], + features.keys()) + self.assertEqual(features['left_word_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['left_mask'].shape, (batch_size, seq_length)) + self.assertEqual(features['left_type_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['right_word_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['right_mask'].shape, (batch_size, seq_length)) + self.assertEqual(features['right_type_ids'].shape, (batch_size, seq_length)) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/pretrain_dataloader.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/pretrain_dataloader.py new file mode 100644 index 0000000000000000000000000000000000000000..dbb7953c3fd7f1562d7b3ec07c58b09eefef8e25 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/pretrain_dataloader.py @@ -0,0 +1,604 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Loads dataset for the BERT pretraining task.""" +from typing import Mapping, Optional + +from absl import logging + +import dataclasses +import numpy as np +import tensorflow as tf +from official.core import config_definitions as cfg +from official.core import input_reader +from official.nlp.data import data_loader +from official.nlp.data import data_loader_factory + + +@dataclasses.dataclass +class BertPretrainDataConfig(cfg.DataConfig): + """Data config for BERT pretraining task (tasks/masked_lm).""" + input_path: str = '' + global_batch_size: int = 512 + is_training: bool = True + seq_length: int = 512 + max_predictions_per_seq: int = 76 + use_next_sentence_label: bool = True + use_position_id: bool = False + # Historically, BERT implementations take `input_ids` and `segment_ids` as + # feature names. Inside the TF Model Garden implementation, the Keras model + # inputs are set as `input_word_ids` and `input_type_ids`. When + # v2_feature_names is True, the data loader assumes the tf.Examples use + # `input_word_ids` and `input_type_ids` as keys. + use_v2_feature_names: bool = False + + +@data_loader_factory.register_data_loader_cls(BertPretrainDataConfig) +class BertPretrainDataLoader(data_loader.DataLoader): + """A class to load dataset for bert pretraining task.""" + + def __init__(self, params): + """Inits `BertPretrainDataLoader` class. + + Args: + params: A `BertPretrainDataConfig` object. + """ + self._params = params + self._seq_length = params.seq_length + self._max_predictions_per_seq = params.max_predictions_per_seq + self._use_next_sentence_label = params.use_next_sentence_label + self._use_position_id = params.use_position_id + + def _name_to_features(self): + name_to_features = { + 'input_mask': + tf.io.FixedLenFeature([self._seq_length], tf.int64), + 'masked_lm_positions': + tf.io.FixedLenFeature([self._max_predictions_per_seq], tf.int64), + 'masked_lm_ids': + tf.io.FixedLenFeature([self._max_predictions_per_seq], tf.int64), + 'masked_lm_weights': + tf.io.FixedLenFeature([self._max_predictions_per_seq], tf.float32), + } + if self._params.use_v2_feature_names: + name_to_features.update({ + 'input_word_ids': tf.io.FixedLenFeature([self._seq_length], tf.int64), + 'input_type_ids': tf.io.FixedLenFeature([self._seq_length], tf.int64), + }) + else: + name_to_features.update({ + 'input_ids': tf.io.FixedLenFeature([self._seq_length], tf.int64), + 'segment_ids': tf.io.FixedLenFeature([self._seq_length], tf.int64), + }) + if self._use_next_sentence_label: + name_to_features['next_sentence_labels'] = tf.io.FixedLenFeature([1], + tf.int64) + if self._use_position_id: + name_to_features['position_ids'] = tf.io.FixedLenFeature( + [self._seq_length], tf.int64) + return name_to_features + + def _decode(self, record: tf.Tensor): + """Decodes a serialized tf.Example.""" + name_to_features = self._name_to_features() + example = tf.io.parse_single_example(record, name_to_features) + + # tf.Example only supports tf.int64, but the TPU only supports tf.int32. + # So cast all int64 to int32. + for name in list(example.keys()): + t = example[name] + if t.dtype == tf.int64: + t = tf.cast(t, tf.int32) + example[name] = t + + return example + + def _parse(self, record: Mapping[str, tf.Tensor]): + """Parses raw tensors into a dict of tensors to be consumed by the model.""" + x = { + 'input_mask': record['input_mask'], + 'masked_lm_positions': record['masked_lm_positions'], + 'masked_lm_ids': record['masked_lm_ids'], + 'masked_lm_weights': record['masked_lm_weights'], + } + if self._params.use_v2_feature_names: + x['input_word_ids'] = record['input_word_ids'] + x['input_type_ids'] = record['input_type_ids'] + else: + x['input_word_ids'] = record['input_ids'] + x['input_type_ids'] = record['segment_ids'] + if self._use_next_sentence_label: + x['next_sentence_labels'] = record['next_sentence_labels'] + if self._use_position_id: + x['position_ids'] = record['position_ids'] + + return x + + def load(self, input_context: Optional[tf.distribute.InputContext] = None): + """Returns a tf.dataset.Dataset.""" + reader = input_reader.InputReader( + params=self._params, decoder_fn=self._decode, parser_fn=self._parse) + return reader.read(input_context) + + +@dataclasses.dataclass +class XLNetPretrainDataConfig(cfg.DataConfig): + """Data config for XLNet pretraining task. + + Attributes: + input_path: See base class. + global_batch_size: See base calss. + is_training: See base class. + seq_length: The length of each sequence. + max_predictions_per_seq: The number of predictions per sequence. + reuse_length: The number of tokens in a previous segment to reuse. This + should be the same value used during pretrain data creation. + sample_strategy: The strategy used to sample factorization permutations. + Possible values: 'single_token', 'whole_word', 'token_span', 'word_span'. + min_num_tokens: The minimum number of tokens to sample in a span. + This is used when `sample_strategy` is 'token_span'. + max_num_tokens: The maximum number of tokens to sample in a span. + This is used when `sample_strategy` is 'token_span'. + min_num_words: The minimum number of words to sample in a span. + This is used when `sample_strategy` is 'word_span'. + max_num_words: The maximum number of words to sample in a span. + This is used when `sample_strategy` is 'word_span'. + permutation_size: The length of the longest permutation. This can be set + to `reuse_length`. This should NOT be greater than `reuse_length`, + otherwise this may introduce data leaks. + leak_ratio: The percentage of masked tokens that are leaked. + segment_sep_id: The ID of the SEP token used when preprocessing + the dataset. + segment_cls_id: The ID of the CLS token used when preprocessing + the dataset. + + """ + input_path: str = '' + global_batch_size: int = 512 + is_training: bool = True + seq_length: int = 512 + max_predictions_per_seq: int = 76 + reuse_length: int = 256 + sample_strategy: str = 'word_span' + min_num_tokens: int = 1 + max_num_tokens: int = 5 + min_num_words: int = 1 + max_num_words: int = 5 + permutation_size: int = 256 + leak_ratio: float = 0.1 + segment_sep_id: int = 4 + segment_cls_id: int = 3 + + +@data_loader_factory.register_data_loader_cls(XLNetPretrainDataConfig) +class XLNetPretrainDataLoader(data_loader.DataLoader): + """A class to load dataset for xlnet pretraining task.""" + + def __init__(self, params: XLNetPretrainDataConfig): + """Inits `XLNetPretrainDataLoader` class. + + Args: + params: A `XLNetPretrainDataConfig` object. + """ + self._params = params + self._seq_length = params.seq_length + self._max_predictions_per_seq = params.max_predictions_per_seq + self._reuse_length = params.reuse_length + self._num_replicas_in_sync = None + self._permutation_size = params.permutation_size + self._sep_id = params.segment_sep_id + self._cls_id = params.segment_cls_id + self._sample_strategy = params.sample_strategy + self._leak_ratio = params.leak_ratio + + def _decode(self, record: tf.Tensor): + """Decodes a serialized tf.Example.""" + name_to_features = { + 'input_word_ids': + tf.io.FixedLenFeature([self._seq_length], tf.int64), + 'input_type_ids': + tf.io.FixedLenFeature([self._seq_length], tf.int64), + 'boundary_indices': + tf.io.VarLenFeature(tf.int64), + } + example = tf.io.parse_single_example(record, name_to_features) + + # tf.Example only supports tf.int64, but the TPU only supports tf.int32. + # So cast all int64 to int32. + for name in list(example.keys()): + t = example[name] + if t.dtype == tf.int64: + t = tf.cast(t, tf.int32) + example[name] = t + + return example + + def _parse(self, record: Mapping[str, tf.Tensor]): + """Parses raw tensors into a dict of tensors to be consumed by the model.""" + x = {} + + inputs = record['input_word_ids'] + x['input_type_ids'] = record['input_type_ids'] + + if self._sample_strategy in ['whole_word', 'word_span']: + boundary = tf.sparse.to_dense(record['boundary_indices']) + else: + boundary = None + + input_mask = self._online_sample_mask(inputs=inputs, boundary=boundary) + + if self._reuse_length > 0: + if self._permutation_size > self._reuse_length: + logging.warning( + '`permutation_size` is greater than `reuse_length` (%d > %d).' + 'This may introduce data leakage.', + self._permutation_size, self._reuse_length) + + # Enable the memory mechanism. + # Permute the reuse and non-reuse segments separately. + non_reuse_len = self._seq_length - self._reuse_length + if not (self._reuse_length % self._permutation_size == 0 + and non_reuse_len % self._permutation_size == 0): + raise ValueError('`reuse_length` and `seq_length` should both be ' + 'a multiple of `permutation_size`.') + + # Creates permutation mask and target mask for the first reuse_len tokens. + # The tokens in this part are reused from the last sequence. + perm_mask_0, target_mask_0, tokens_0, masked_0 = self._get_factorization( + inputs=inputs[:self._reuse_length], + input_mask=input_mask[:self._reuse_length]) + + # Creates permutation mask and target mask for the rest of tokens in + # current example, which are concatentation of two new segments. + perm_mask_1, target_mask_1, tokens_1, masked_1 = self._get_factorization( + inputs[self._reuse_length:], input_mask[self._reuse_length:]) + + perm_mask_0 = tf.concat( + [perm_mask_0, + tf.zeros([self._reuse_length, non_reuse_len], dtype=tf.int32)], + axis=1) + perm_mask_1 = tf.concat( + [tf.ones([non_reuse_len, self._reuse_length], dtype=tf.int32), + perm_mask_1], axis=1) + perm_mask = tf.concat([perm_mask_0, perm_mask_1], axis=0) + target_mask = tf.concat([target_mask_0, target_mask_1], axis=0) + tokens = tf.concat([tokens_0, tokens_1], axis=0) + masked_tokens = tf.concat([masked_0, masked_1], axis=0) + else: + # Disable the memory mechanism. + if self._seq_length % self._permutation_size != 0: + raise ValueError('`seq_length` should be a multiple of ' + '`permutation_size`.') + # Permute the entire sequence together + perm_mask, target_mask, tokens, masked_tokens = self._get_factorization( + inputs=inputs, input_mask=input_mask) + x['permutation_mask'] = tf.reshape( + perm_mask, [self._seq_length, self._seq_length]) + x['input_word_ids'] = tokens + x['masked_tokens'] = masked_tokens + + target = tokens + if self._max_predictions_per_seq is not None: + indices = tf.range(self._seq_length, dtype=tf.int32) + bool_target_mask = tf.cast(target_mask, tf.bool) + indices = tf.boolean_mask(indices, bool_target_mask) + + # account for extra padding due to CLS/SEP. + actual_num_predict = tf.shape(indices)[0] + pad_len = self._max_predictions_per_seq - actual_num_predict + + target_mapping = tf.one_hot(indices, self._seq_length, dtype=tf.int32) + paddings = tf.zeros([pad_len, self._seq_length], + dtype=target_mapping.dtype) + target_mapping = tf.concat([target_mapping, paddings], axis=0) + x['target_mapping'] = tf.reshape( + target_mapping, [self._max_predictions_per_seq, self._seq_length]) + + target = tf.boolean_mask(target, bool_target_mask) + paddings = tf.zeros([pad_len], dtype=target.dtype) + target = tf.concat([target, paddings], axis=0) + x['target'] = tf.reshape(target, [self._max_predictions_per_seq]) + + target_mask = tf.concat([ + tf.ones([actual_num_predict], dtype=tf.int32), + tf.zeros([pad_len], dtype=tf.int32) + ], axis=0) + x['target_mask'] = tf.reshape(target_mask, + [self._max_predictions_per_seq]) + else: + x['target'] = tf.reshape(target, [self._seq_length]) + x['target_mask'] = tf.reshape(target_mask, [self._seq_length]) + return x + + def _index_pair_to_mask(self, + begin_indices: tf.Tensor, + end_indices: tf.Tensor, + inputs: tf.Tensor) -> tf.Tensor: + """Converts beginning and end indices into an actual mask.""" + non_func_mask = tf.logical_and( + tf.not_equal(inputs, self._sep_id), tf.not_equal(inputs, self._cls_id)) + all_indices = tf.where( + non_func_mask, + tf.range(self._seq_length, dtype=tf.int32), + tf.constant(-1, shape=[self._seq_length], dtype=tf.int32)) + candidate_matrix = tf.cast( + tf.logical_and(all_indices[None, :] >= begin_indices[:, None], + all_indices[None, :] < end_indices[:, None]), tf.float32) + cumsum_matrix = tf.reshape( + tf.cumsum(tf.reshape(candidate_matrix, [-1])), [-1, self._seq_length]) + masked_matrix = tf.cast(cumsum_matrix <= self._max_predictions_per_seq, + tf.float32) + target_mask = tf.reduce_sum(candidate_matrix * masked_matrix, axis=0) + return tf.cast(target_mask, tf.bool) + + def _single_token_mask(self, inputs: tf.Tensor) -> tf.Tensor: + """Samples individual tokens as prediction targets.""" + all_indices = tf.range(self._seq_length, dtype=tf.int32) + non_func_mask = tf.logical_and( + tf.not_equal(inputs, self._sep_id), tf.not_equal(inputs, self._cls_id)) + non_func_indices = tf.boolean_mask(all_indices, non_func_mask) + + masked_pos = tf.random.shuffle(non_func_indices) + masked_pos = tf.sort(masked_pos[:self._max_predictions_per_seq]) + + sparse_indices = tf.stack( + [tf.zeros_like(masked_pos), masked_pos], axis=-1) + sparse_indices = tf.cast(sparse_indices, tf.int64) + + sparse_indices = tf.sparse.SparseTensor( + sparse_indices, + values=tf.ones_like(masked_pos), + dense_shape=(1, self._seq_length)) + + target_mask = tf.sparse.to_dense( + sp_input=sparse_indices, + default_value=0) + + return tf.squeeze(tf.cast(target_mask, tf.bool)) + + def _whole_word_mask(self, + inputs: tf.Tensor, + boundary: tf.Tensor) -> tf.Tensor: + """Samples whole words as prediction targets.""" + pair_indices = tf.concat([boundary[:-1, None], boundary[1:, None]], axis=1) + cand_pair_indices = tf.random.shuffle( + pair_indices)[:self._max_predictions_per_seq] + begin_indices = cand_pair_indices[:, 0] + end_indices = cand_pair_indices[:, 1] + + return self._index_pair_to_mask( + begin_indices=begin_indices, + end_indices=end_indices, + inputs=inputs) + + def _token_span_mask(self, inputs: tf.Tensor) -> tf.Tensor: + """Samples token spans as prediction targets.""" + min_num_tokens = self._params.min_num_tokens + max_num_tokens = self._params.max_num_tokens + + mask_alpha = self._seq_length / self._max_predictions_per_seq + round_to_int = lambda x: tf.cast(tf.round(x), tf.int32) + + # Sample span lengths from a zipf distribution + span_len_seq = np.arange(min_num_tokens, max_num_tokens + 1) + probs = np.array([1.0 / (i + 1) for i in span_len_seq]) + + probs /= np.sum(probs) + logits = tf.constant(np.log(probs), dtype=tf.float32) + span_lens = tf.random.categorical( + logits=logits[None], + num_samples=self._max_predictions_per_seq, + dtype=tf.int32, + )[0] + min_num_tokens + + # Sample the ratio [0.0, 1.0) of left context lengths + span_lens_float = tf.cast(span_lens, tf.float32) + left_ratio = tf.random.uniform( + shape=[self._max_predictions_per_seq], minval=0.0, maxval=1.0) + left_ctx_len = left_ratio * span_lens_float * (mask_alpha - 1) + left_ctx_len = round_to_int(left_ctx_len) + + # Compute the offset from left start to the right end + right_offset = round_to_int(span_lens_float * mask_alpha) - left_ctx_len + + # Get the actual begin and end indices + begin_indices = ( + tf.cumsum(left_ctx_len) + tf.cumsum(right_offset, exclusive=True)) + end_indices = begin_indices + span_lens + + # Remove out of range indices + valid_idx_mask = end_indices < self._seq_length + begin_indices = tf.boolean_mask(begin_indices, valid_idx_mask) + end_indices = tf.boolean_mask(end_indices, valid_idx_mask) + + # Shuffle valid indices + num_valid = tf.cast(tf.shape(begin_indices)[0], tf.int32) + order = tf.random.shuffle(tf.range(num_valid, dtype=tf.int32)) + begin_indices = tf.gather(begin_indices, order) + end_indices = tf.gather(end_indices, order) + + return self._index_pair_to_mask( + begin_indices=begin_indices, + end_indices=end_indices, + inputs=inputs) + + def _word_span_mask(self, + inputs: tf.Tensor, + boundary: tf.Tensor): + """Sample whole word spans as prediction targets.""" + min_num_words = self._params.min_num_words + max_num_words = self._params.max_num_words + + # Note: 1.2 is the token-to-word ratio + mask_alpha = self._seq_length / self._max_predictions_per_seq / 1.2 + round_to_int = lambda x: tf.cast(tf.round(x), tf.int32) + + # Sample span lengths from a zipf distribution + span_len_seq = np.arange(min_num_words, max_num_words + 1) + probs = np.array([1.0 / (i + 1) for i in span_len_seq]) + probs /= np.sum(probs) + logits = tf.constant(np.log(probs), dtype=tf.float32) + + # Sample `num_predict` words here: note that this is over sampling + span_lens = tf.random.categorical( + logits=logits[None], + num_samples=self._max_predictions_per_seq, + dtype=tf.int32, + )[0] + min_num_words + + # Sample the ratio [0.0, 1.0) of left context lengths + span_lens_float = tf.cast(span_lens, tf.float32) + left_ratio = tf.random.uniform( + shape=[self._max_predictions_per_seq], minval=0.0, maxval=1.0) + left_ctx_len = left_ratio * span_lens_float * (mask_alpha - 1) + + left_ctx_len = round_to_int(left_ctx_len) + right_offset = round_to_int(span_lens_float * mask_alpha) - left_ctx_len + + begin_indices = ( + tf.cumsum(left_ctx_len) + tf.cumsum(right_offset, exclusive=True)) + end_indices = begin_indices + span_lens + + # Remove out of range indices + max_boundary_index = tf.cast(tf.shape(boundary)[0] - 1, tf.int32) + valid_idx_mask = end_indices < max_boundary_index + begin_indices = tf.boolean_mask(begin_indices, valid_idx_mask) + end_indices = tf.boolean_mask(end_indices, valid_idx_mask) + + begin_indices = tf.gather(boundary, begin_indices) + end_indices = tf.gather(boundary, end_indices) + + # Shuffle valid indices + num_valid = tf.cast(tf.shape(begin_indices)[0], tf.int32) + order = tf.random.shuffle(tf.range(num_valid, dtype=tf.int32)) + begin_indices = tf.gather(begin_indices, order) + end_indices = tf.gather(end_indices, order) + + return self._index_pair_to_mask( + begin_indices=begin_indices, + end_indices=end_indices, + inputs=inputs) + + def _online_sample_mask(self, + inputs: tf.Tensor, + boundary: tf.Tensor) -> tf.Tensor: + """Samples target positions for predictions. + + Descriptions of each strategy: + - 'single_token': Samples individual tokens as prediction targets. + - 'token_span': Samples spans of tokens as prediction targets. + - 'whole_word': Samples individual words as prediction targets. + - 'word_span': Samples spans of words as prediction targets. + + Args: + inputs: The input tokens. + boundary: The `int` Tensor of indices indicating whole word boundaries. + This is used in 'whole_word' and 'word_span' + + Returns: + The sampled `bool` input mask. + + Raises: + `ValueError`: if `max_predictions_per_seq` is not set or if boundary is + not provided for 'whole_word' and 'word_span' sample strategies. + """ + if self._max_predictions_per_seq is None: + raise ValueError('`max_predictions_per_seq` must be set.') + + if boundary is None and 'word' in self._sample_strategy: + raise ValueError('`boundary` must be provided for {} strategy'.format( + self._sample_strategy)) + + if self._sample_strategy == 'single_token': + return self._single_token_mask(inputs) + elif self._sample_strategy == 'token_span': + return self._token_span_mask(inputs) + elif self._sample_strategy == 'whole_word': + return self._whole_word_mask(inputs, boundary) + elif self._sample_strategy == 'word_span': + return self._word_span_mask(inputs, boundary) + else: + raise NotImplementedError('Invalid sample strategy.') + + def _get_factorization(self, + inputs: tf.Tensor, + input_mask: tf.Tensor): + """Samples a permutation of the factorization order. + + Args: + inputs: the input tokens. + input_mask: the `bool` Tensor of the same shape as `inputs`. + If `True`, then this means select for partial prediction. + + Returns: + perm_mask: An `int32` Tensor of shape [seq_length, seq_length] consisting + of 0s and 1s. If perm_mask[i][j] == 0, then this means that the i-th + token (in original order) cannot attend to the jth attention token. + target_mask: An `int32` Tensor of shape [seq_len] consisting of 0s and 1s. + If target_mask[i] == 1, then the i-th token needs to be predicted and + the mask will be used as input. This token will be included in the loss. + If target_mask[i] == 0, then the token (or [SEP], [CLS]) will be used as + input. This token will not be included in the loss. + tokens: int32 Tensor of shape [seq_length]. + masked_tokens: int32 Tensor of shape [seq_length]. + + """ + factorization_length = tf.shape(inputs)[0] + # Generate permutation indices + index = tf.range(factorization_length, dtype=tf.int32) + index = tf.transpose(tf.reshape(index, [-1, self._permutation_size])) + index = tf.random.shuffle(index) + index = tf.reshape(tf.transpose(index), [-1]) + + input_mask = tf.cast(input_mask, tf.bool) + + # non-functional tokens + non_func_tokens = tf.logical_not( + tf.logical_or( + tf.equal(inputs, self._sep_id), tf.equal(inputs, self._cls_id))) + masked_tokens = tf.logical_and(input_mask, non_func_tokens) + non_masked_or_func_tokens = tf.logical_not(masked_tokens) + + smallest_index = -2 * tf.ones([factorization_length], dtype=tf.int32) + + # Similar to BERT, randomly leak some masked tokens + if self._leak_ratio > 0: + leak_tokens = tf.logical_and( + masked_tokens, + tf.random.uniform([factorization_length], + maxval=1.0) < self._leak_ratio) + can_attend_self = tf.logical_or(non_masked_or_func_tokens, leak_tokens) + else: + can_attend_self = non_masked_or_func_tokens + to_index = tf.where(can_attend_self, smallest_index, index) + from_index = tf.where(can_attend_self, to_index + 1, to_index) + + # For masked tokens, can attend if i > j + # For context tokens, always can attend each other + can_attend = from_index[:, None] > to_index[None, :] + + perm_mask = tf.cast(can_attend, tf.int32) + + # Only masked tokens are included in the loss + target_mask = tf.cast(masked_tokens, tf.int32) + + return perm_mask, target_mask, inputs, masked_tokens + + def load(self, input_context: Optional[tf.distribute.InputContext] = None): + """Returns a tf.dataset.Dataset.""" + if input_context: + self._num_replicas_in_sync = input_context.num_replicas_in_sync + reader = input_reader.InputReader( + params=self._params, decoder_fn=self._decode, parser_fn=self._parse) + return reader.read(input_context) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/pretrain_dataloader_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/pretrain_dataloader_test.py new file mode 100644 index 0000000000000000000000000000000000000000..5f3807c907ad9cbb2007425ac13bba620491dce6 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/pretrain_dataloader_test.py @@ -0,0 +1,242 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for official.nlp.data.pretrain_dataloader.""" +import itertools +import os + +from absl.testing import parameterized +import numpy as np +import tensorflow as tf + +from official.nlp.data import pretrain_dataloader + + +def create_int_feature(values): + f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + return f + + +def _create_fake_bert_dataset( + output_path, + seq_length, + max_predictions_per_seq, + use_position_id, + use_next_sentence_label, + use_v2_feature_names=False): + """Creates a fake dataset.""" + writer = tf.io.TFRecordWriter(output_path) + + def create_float_feature(values): + f = tf.train.Feature(float_list=tf.train.FloatList(value=list(values))) + return f + + for _ in range(100): + features = {} + input_ids = np.random.randint(100, size=(seq_length)) + features["input_mask"] = create_int_feature(np.ones_like(input_ids)) + if use_v2_feature_names: + features["input_word_ids"] = create_int_feature(input_ids) + features["input_type_ids"] = create_int_feature(np.ones_like(input_ids)) + else: + features["input_ids"] = create_int_feature(input_ids) + features["segment_ids"] = create_int_feature(np.ones_like(input_ids)) + + features["masked_lm_positions"] = create_int_feature( + np.random.randint(100, size=(max_predictions_per_seq))) + features["masked_lm_ids"] = create_int_feature( + np.random.randint(100, size=(max_predictions_per_seq))) + features["masked_lm_weights"] = create_float_feature( + [1.0] * max_predictions_per_seq) + + if use_next_sentence_label: + features["next_sentence_labels"] = create_int_feature([1]) + + if use_position_id: + features["position_ids"] = create_int_feature(range(0, seq_length)) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + writer.write(tf_example.SerializeToString()) + writer.close() + + +def _create_fake_xlnet_dataset( + output_path, seq_length, max_predictions_per_seq): + """Creates a fake dataset.""" + writer = tf.io.TFRecordWriter(output_path) + for _ in range(100): + features = {} + input_ids = np.random.randint(100, size=(seq_length)) + num_boundary_indices = np.random.randint(1, seq_length) + + if max_predictions_per_seq is not None: + input_mask = np.zeros_like(input_ids) + input_mask[:max_predictions_per_seq] = 1 + np.random.shuffle(input_mask) + else: + input_mask = np.ones_like(input_ids) + + features["input_mask"] = create_int_feature(input_mask) + features["input_word_ids"] = create_int_feature(input_ids) + features["input_type_ids"] = create_int_feature(np.ones_like(input_ids)) + features["boundary_indices"] = create_int_feature( + sorted(np.random.randint(seq_length, size=(num_boundary_indices)))) + features["target"] = create_int_feature(input_ids + 1) + features["label"] = create_int_feature([1]) + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + writer.write(tf_example.SerializeToString()) + writer.close() + + +class BertPretrainDataTest(tf.test.TestCase, parameterized.TestCase): + + @parameterized.parameters(itertools.product( + (False, True), + (False, True), + )) + def test_load_data(self, use_next_sentence_label, use_position_id): + train_data_path = os.path.join(self.get_temp_dir(), "train.tf_record") + seq_length = 128 + max_predictions_per_seq = 20 + _create_fake_bert_dataset( + train_data_path, + seq_length, + max_predictions_per_seq, + use_next_sentence_label=use_next_sentence_label, + use_position_id=use_position_id) + data_config = pretrain_dataloader.BertPretrainDataConfig( + input_path=train_data_path, + max_predictions_per_seq=max_predictions_per_seq, + seq_length=seq_length, + global_batch_size=10, + is_training=True, + use_next_sentence_label=use_next_sentence_label, + use_position_id=use_position_id) + + dataset = pretrain_dataloader.BertPretrainDataLoader(data_config).load() + features = next(iter(dataset)) + self.assertLen(features, + 6 + int(use_next_sentence_label) + int(use_position_id)) + self.assertIn("input_word_ids", features) + self.assertIn("input_mask", features) + self.assertIn("input_type_ids", features) + self.assertIn("masked_lm_positions", features) + self.assertIn("masked_lm_ids", features) + self.assertIn("masked_lm_weights", features) + + self.assertEqual("next_sentence_labels" in features, + use_next_sentence_label) + self.assertEqual("position_ids" in features, use_position_id) + + def test_v2_feature_names(self): + train_data_path = os.path.join(self.get_temp_dir(), "train.tf_record") + seq_length = 128 + max_predictions_per_seq = 20 + _create_fake_bert_dataset( + train_data_path, + seq_length, + max_predictions_per_seq, + use_next_sentence_label=True, + use_position_id=False, + use_v2_feature_names=True) + data_config = pretrain_dataloader.BertPretrainDataConfig( + input_path=train_data_path, + max_predictions_per_seq=max_predictions_per_seq, + seq_length=seq_length, + global_batch_size=10, + is_training=True, + use_next_sentence_label=True, + use_position_id=False, + use_v2_feature_names=True) + + dataset = pretrain_dataloader.BertPretrainDataLoader(data_config).load() + features = next(iter(dataset)) + self.assertIn("input_word_ids", features) + self.assertIn("input_mask", features) + self.assertIn("input_type_ids", features) + self.assertIn("masked_lm_positions", features) + self.assertIn("masked_lm_ids", features) + self.assertIn("masked_lm_weights", features) + + +class XLNetPretrainDataTest(parameterized.TestCase, tf.test.TestCase): + + @parameterized.parameters(itertools.product( + ("single_token", "whole_word", "token_span"), + (0, 64), + (20, None), + )) + def test_load_data( + self, sample_strategy, reuse_length, max_predictions_per_seq): + train_data_path = os.path.join(self.get_temp_dir(), "train.tf_record") + seq_length = 128 + batch_size = 5 + + _create_fake_xlnet_dataset( + train_data_path, seq_length, max_predictions_per_seq) + + data_config = pretrain_dataloader.XLNetPretrainDataConfig( + input_path=train_data_path, + max_predictions_per_seq=max_predictions_per_seq, + seq_length=seq_length, + global_batch_size=batch_size, + is_training=True, + reuse_length=reuse_length, + sample_strategy=sample_strategy, + min_num_tokens=1, + max_num_tokens=2, + permutation_size=seq_length // 2, + leak_ratio=0.1) + + if max_predictions_per_seq is None: + with self.assertRaises(ValueError): + dataset = pretrain_dataloader.XLNetPretrainDataLoader( + data_config).load() + features = next(iter(dataset)) + else: + dataset = pretrain_dataloader.XLNetPretrainDataLoader(data_config).load() + features = next(iter(dataset)) + + self.assertIn("input_word_ids", features) + self.assertIn("input_type_ids", features) + self.assertIn("permutation_mask", features) + self.assertIn("masked_tokens", features) + self.assertIn("target", features) + self.assertIn("target_mask", features) + + self.assertAllClose(features["input_word_ids"].shape, + (batch_size, seq_length)) + self.assertAllClose(features["input_type_ids"].shape, + (batch_size, seq_length)) + self.assertAllClose(features["permutation_mask"].shape, + (batch_size, seq_length, seq_length)) + self.assertAllClose(features["masked_tokens"].shape, + (batch_size, seq_length,)) + if max_predictions_per_seq is not None: + self.assertIn("target_mapping", features) + self.assertAllClose(features["target_mapping"].shape, + (batch_size, max_predictions_per_seq, seq_length)) + self.assertAllClose(features["target_mask"].shape, + (batch_size, max_predictions_per_seq)) + self.assertAllClose(features["target"].shape, + (batch_size, max_predictions_per_seq)) + else: + self.assertAllClose(features["target_mask"].shape, + (batch_size, seq_length)) + self.assertAllClose(features["target"].shape, + (batch_size, seq_length)) + + +if __name__ == "__main__": + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/pretrain_dynamic_dataloader.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/pretrain_dynamic_dataloader.py new file mode 100644 index 0000000000000000000000000000000000000000..c1de4ba54b86a3386708e3f56b76e8e3726c397d --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/pretrain_dynamic_dataloader.py @@ -0,0 +1,211 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dataset loader for the pre-training with dynamic sequence length.""" +from typing import Optional, Tuple + +import dataclasses +import tensorflow as tf + +from official.core import config_definitions as cfg +from official.core import input_reader +from official.nlp.data import data_loader_factory +from official.nlp.data import pretrain_dataloader + + +@dataclasses.dataclass +class BertPretrainDataConfig(cfg.DataConfig): + """Data config for BERT pretraining task (tasks/masked_lm).""" + input_path: str = '' + global_batch_size: int = 512 + is_training: bool = True + seq_bucket_lengths: Tuple[int, ...] = (128, 256, 384, 512,) + # TODO(rxsang): `seq_bucket_window_scale` is only useful when round robin + # tf.data service is disabled. Deprecate this flag once we always enable round + # robin tf.data service. + seq_bucket_window_scale: int = 8 + use_next_sentence_label: bool = True + use_position_id: bool = False + deterministic: bool = False + enable_tf_data_service: bool = False + enable_round_robin_tf_data_service: bool = False + tf_data_service_job_name: str = 'bert_pretrain' + use_v2_feature_names: bool = False + + +@data_loader_factory.register_data_loader_cls(BertPretrainDataConfig) +class PretrainingDynamicDataLoader(pretrain_dataloader.BertPretrainDataLoader): + """Dataset loader for bert-style pretraining with dynamic sequenece length. + + Bucketizes the input id features by the seq_bucket_lengths and features are + padded to the bucket boundaries. The mask features are usually short than + input id features and can also be dynamic. We require the mask feature lengths + within a bucket must be the same. For example, with [128, 256] buckets, + the mask features for bucket 128 should always have the length as X and + features for bucket 256 should always have the length as Y. + + The dataloader does not filter out empty masks. Make sure to handle this + in the model. + """ + + def __init__(self, params): + self._params = params + if len(params.seq_bucket_lengths) < 1: + raise ValueError('The seq_bucket_lengths cannot be empty.') + self._seq_bucket_lengths = params.seq_bucket_lengths + self._seq_bucket_window_scale = params.seq_bucket_window_scale + self._global_batch_size = params.global_batch_size + self._use_next_sentence_label = params.use_next_sentence_label + self._use_position_id = params.use_position_id + self._drop_remainder = params.drop_remainder + self._enable_tf_data_service = params.enable_tf_data_service + self._enable_round_robin_tf_data_service = ( + params.enable_round_robin_tf_data_service) + self._mask_keys = [ + 'masked_lm_positions', 'masked_lm_ids', 'masked_lm_weights' + ] + + def _decode(self, record: tf.Tensor): + """Decodes a serialized tf.Example.""" + name_to_features = { + 'input_ids': tf.io.VarLenFeature(tf.int64), + 'input_mask': tf.io.VarLenFeature(tf.int64), + 'segment_ids': tf.io.VarLenFeature(tf.int64), + 'masked_lm_positions': tf.io.VarLenFeature(tf.int64), + 'masked_lm_ids': tf.io.VarLenFeature(tf.int64), + 'masked_lm_weights': tf.io.VarLenFeature(tf.float32), + } + if self._use_next_sentence_label: + name_to_features['next_sentence_labels'] = tf.io.FixedLenFeature([1], + tf.int64) + dynamic_keys = ['input_ids', 'input_mask', 'segment_ids'] + if self._use_position_id: + name_to_features['position_ids'] = tf.io.VarLenFeature(tf.int64) + dynamic_keys.append('position_ids') + + example = tf.io.parse_single_example(record, name_to_features) + for key in dynamic_keys + self._mask_keys: + example[key] = tf.sparse.to_dense(example[key]) + + # Truncate padded data after the first non pad in the + # sequence length dimension. + # Pad before the first non pad from the back should not be removed. + mask = tf.math.greater( + tf.math.cumsum(example['input_ids'], reverse=True), 0) + for key in dynamic_keys: + example[key] = tf.boolean_mask(example[key], mask) + + # masked_lm_ids should be 0 padded. + # Change mask features to -1 padding so that we can differentiate + # padding from data or from bucketizing. + mask = tf.math.not_equal(example['masked_lm_ids'], 0) + example['masked_lm_ids'] = tf.where( + mask, example['masked_lm_ids'], + -tf.ones(tf.shape(example['masked_lm_ids']), dtype=example[key].dtype)) + + # tf.Example only supports tf.int64, but the TPU only supports tf.int32. + # So cast all int64 to int32. + # tf.data service uses dataset graph fingerprint to distinguish input + # pipeline jobs, thus we sort the keys here to make sure they are generated + # in a deterministic order each time the dataset function is traced. + for name in sorted(list(example.keys())): + t = example[name] + if t.dtype == tf.int64: + t = tf.cast(t, tf.int32) + example[name] = t + + return example + + def _bucketize_and_batch( + self, + dataset, + input_context: Optional[tf.distribute.InputContext] = None): + """Bucketize by sequence length and batch the datasets.""" + per_replica_batch_size = input_context.get_per_replica_batch_size( + self._global_batch_size) if input_context else self._global_batch_size + + def element_length_func(example, seq_len_dim): + return tf.shape(example['input_word_ids'])[seq_len_dim] + + bucket_boundaries = [length + 1 for length in self._seq_bucket_lengths] + bucket_batch_sizes = [per_replica_batch_size] * (len(bucket_boundaries) + 1) + + # Bucketize and batch the dataset with per replica batch size first. + dataset = dataset.apply( + tf.data.experimental.bucket_by_sequence_length( + lambda example: tf.cast(element_length_func(example, 0), tf.int32), + bucket_boundaries, + bucket_batch_sizes, + pad_to_bucket_boundary=True, + drop_remainder=self._drop_remainder)) + if input_context: + window_size = input_context.num_replicas_in_sync + if self._enable_tf_data_service and ( + not self._enable_round_robin_tf_data_service): + # If tf.data service is enabled but round-robin behavior is not enabled, + # different TPU workers may fetch data from one tf.data service worker + # in different speed. We set the window size to be + # `seq_bucket_window_scale` larger to leave buffer if some workers are + # fetching data faster than others, so all the data within the same + # global batch can still have more chances to be in the same bucket. + window_size *= self._seq_bucket_window_scale + + # Group `num_replicas_in_sync` batches from same bucket together, so all + # replicas can get the same sequence length for one global step. + dataset = dataset.apply( + tf.data.experimental.group_by_window( + key_func=lambda example: tf.cast( # pylint: disable=g-long-lambda + element_length_func(example, 1), tf.int64), + reduce_func=lambda _, x: tf.data.Dataset.from_tensors(x), + window_size=window_size)) + dataset = dataset.flat_map(lambda x: x) + + def _remove_pads_from_bucketize(features): + # All mask features must have the same effective length. + # The real masked ids padding token is -1 and 0 comes from + # bucket_by_sequence_length. + mask = tf.math.not_equal(features['masked_lm_ids'], 0) + + mask_per_example = tf.math.reduce_sum(tf.cast(mask, tf.int32), axis=1) + normalized = tf.cast( + mask_per_example / tf.math.reduce_max(mask_per_example), tf.int32) + assert_op = tf.debugging.assert_equal( + tf.math.reduce_sum(normalized), per_replica_batch_size, + 'Number of non padded mask tokens is not the same for each example ' + 'in the same sequence length.') + with tf.control_dependencies([assert_op]): + for key in self._mask_keys: + features[key] = tf.reshape( + tf.boolean_mask( + features[key], mask), [per_replica_batch_size, -1]) + # Revert masked_lm_ids to be 0-padded. + mask = tf.math.not_equal(features['masked_lm_ids'], -1) + features['masked_lm_ids'] = tf.where( + mask, features['masked_lm_ids'], + tf.zeros( + tf.shape(features['masked_lm_ids']), + dtype=features['masked_lm_ids'].dtype)) + return features + + dataset = dataset.map(_remove_pads_from_bucketize) + return dataset + + def load(self, input_context: Optional[tf.distribute.InputContext] = None): + """Returns a tf.dataset.Dataset.""" + reader = input_reader.InputReader( + params=self._params, + decoder_fn=self._decode, + parser_fn=self._parse, + transform_and_batch_fn=self._bucketize_and_batch) + return reader.read(input_context) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/pretrain_dynamic_dataloader_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/pretrain_dynamic_dataloader_test.py new file mode 100644 index 0000000000000000000000000000000000000000..188e6d495b71acc2699b82f71a86acb5efbf99f5 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/pretrain_dynamic_dataloader_test.py @@ -0,0 +1,245 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for nlp.data.pretrain_dynamic_dataloader.""" +import os + +from absl import logging +from absl.testing import parameterized +import numpy as np +import orbit +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import strategy_combinations +from official.nlp.configs import bert +from official.nlp.configs import encoders +from official.nlp.data import pretrain_dataloader +from official.nlp.data import pretrain_dynamic_dataloader +from official.nlp.tasks import masked_lm + + +def _create_fake_dataset(output_path, seq_length, num_masked_tokens, + max_seq_length, num_examples): + """Creates a fake dataset.""" + writer = tf.io.TFRecordWriter(output_path) + + def create_int_feature(values): + f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + return f + + def create_float_feature(values): + f = tf.train.Feature(float_list=tf.train.FloatList(value=list(values))) + return f + + rng = np.random.default_rng(37) + for _ in range(num_examples): + features = {} + padding = np.zeros(shape=(max_seq_length - seq_length), dtype=np.int32) + input_ids = rng.integers(low=1, high=100, size=(seq_length)) + features['input_ids'] = create_int_feature( + np.concatenate((input_ids, padding))) + features['input_mask'] = create_int_feature( + np.concatenate((np.ones_like(input_ids), padding))) + features['segment_ids'] = create_int_feature( + np.concatenate((np.ones_like(input_ids), padding))) + features['position_ids'] = create_int_feature( + np.concatenate((np.ones_like(input_ids), padding))) + features['masked_lm_positions'] = create_int_feature( + rng.integers(60, size=(num_masked_tokens), dtype=np.int64)) + features['masked_lm_ids'] = create_int_feature( + rng.integers(100, size=(num_masked_tokens), dtype=np.int64)) + features['masked_lm_weights'] = create_float_feature( + np.ones((num_masked_tokens,), dtype=np.float32)) + features['next_sentence_labels'] = create_int_feature(np.array([0])) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + writer.write(tf_example.SerializeToString()) + writer.close() + + +class PretrainDynamicDataLoaderTest(tf.test.TestCase, parameterized.TestCase): + + @combinations.generate( + combinations.combine( + distribution_strategy=[ + strategy_combinations.cloud_tpu_strategy, + ], + mode='eager')) + def test_distribution_strategy(self, distribution_strategy): + max_seq_length = 128 + batch_size = 8 + input_path = os.path.join(self.get_temp_dir(), 'train.tf_record') + _create_fake_dataset( + input_path, + seq_length=60, + num_masked_tokens=20, + max_seq_length=max_seq_length, + num_examples=batch_size) + data_config = pretrain_dynamic_dataloader.BertPretrainDataConfig( + is_training=False, + input_path=input_path, + seq_bucket_lengths=[64, 128], + global_batch_size=batch_size) + dataloader = pretrain_dynamic_dataloader.PretrainingDynamicDataLoader( + data_config) + distributed_ds = orbit.utils.make_distributed_dataset( + distribution_strategy, dataloader.load) + train_iter = iter(distributed_ds) + with distribution_strategy.scope(): + config = masked_lm.MaskedLMConfig( + init_checkpoint=self.get_temp_dir(), + model=bert.PretrainerConfig( + encoders.EncoderConfig( + bert=encoders.BertEncoderConfig( + vocab_size=30522, num_layers=1)), + cls_heads=[ + bert.ClsHeadConfig( + inner_dim=10, num_classes=2, name='next_sentence') + ]), + train_data=data_config) + task = masked_lm.MaskedLMTask(config) + model = task.build_model() + metrics = task.build_metrics() + + @tf.function + def step_fn(features): + return task.validation_step(features, model, metrics=metrics) + + distributed_outputs = distribution_strategy.run( + step_fn, args=(next(train_iter),)) + local_results = tf.nest.map_structure( + distribution_strategy.experimental_local_results, distributed_outputs) + logging.info('Dynamic padding: local_results= %s', str(local_results)) + dynamic_metrics = {} + for metric in metrics: + dynamic_metrics[metric.name] = metric.result() + + data_config = pretrain_dataloader.BertPretrainDataConfig( + is_training=False, + input_path=input_path, + seq_length=max_seq_length, + max_predictions_per_seq=20, + global_batch_size=batch_size) + dataloader = pretrain_dataloader.BertPretrainDataLoader(data_config) + distributed_ds = orbit.utils.make_distributed_dataset( + distribution_strategy, dataloader.load) + train_iter = iter(distributed_ds) + with distribution_strategy.scope(): + metrics = task.build_metrics() + + @tf.function + def step_fn_b(features): + return task.validation_step(features, model, metrics=metrics) + + distributed_outputs = distribution_strategy.run( + step_fn_b, args=(next(train_iter),)) + local_results = tf.nest.map_structure( + distribution_strategy.experimental_local_results, distributed_outputs) + logging.info('Static padding: local_results= %s', str(local_results)) + static_metrics = {} + for metric in metrics: + static_metrics[metric.name] = metric.result() + for key in static_metrics: + # We need to investigate the differences on losses. + if key != 'next_sentence_loss': + self.assertEqual(dynamic_metrics[key], static_metrics[key]) + + def test_load_dataset(self): + tf.random.set_seed(0) + max_seq_length = 128 + batch_size = 2 + input_path_1 = os.path.join(self.get_temp_dir(), 'train_1.tf_record') + _create_fake_dataset( + input_path_1, + seq_length=60, + num_masked_tokens=20, + max_seq_length=max_seq_length, + num_examples=batch_size) + input_path_2 = os.path.join(self.get_temp_dir(), 'train_2.tf_record') + _create_fake_dataset( + input_path_2, + seq_length=100, + num_masked_tokens=70, + max_seq_length=max_seq_length, + num_examples=batch_size) + input_paths = ','.join([input_path_1, input_path_2]) + data_config = pretrain_dynamic_dataloader.BertPretrainDataConfig( + is_training=False, + input_path=input_paths, + seq_bucket_lengths=[64, 128], + use_position_id=True, + global_batch_size=batch_size, + deterministic=True) + dataset = pretrain_dynamic_dataloader.PretrainingDynamicDataLoader( + data_config).load() + dataset_it = iter(dataset) + features = next(dataset_it) + self.assertCountEqual([ + 'input_word_ids', + 'input_mask', + 'input_type_ids', + 'next_sentence_labels', + 'masked_lm_positions', + 'masked_lm_ids', + 'masked_lm_weights', + 'position_ids', + ], features.keys()) + # Sequence length dimension should be bucketized and pad to 64. + self.assertEqual(features['input_word_ids'].shape, (batch_size, 64)) + self.assertEqual(features['input_mask'].shape, (batch_size, 64)) + self.assertEqual(features['input_type_ids'].shape, (batch_size, 64)) + self.assertEqual(features['position_ids'].shape, (batch_size, 64)) + self.assertEqual(features['masked_lm_positions'].shape, (batch_size, 20)) + features = next(dataset_it) + self.assertEqual(features['input_word_ids'].shape, (batch_size, 128)) + self.assertEqual(features['input_mask'].shape, (batch_size, 128)) + self.assertEqual(features['input_type_ids'].shape, (batch_size, 128)) + self.assertEqual(features['position_ids'].shape, (batch_size, 128)) + self.assertEqual(features['masked_lm_positions'].shape, (batch_size, 70)) + + def test_load_dataset_not_same_masks(self): + max_seq_length = 128 + batch_size = 2 + input_path_1 = os.path.join(self.get_temp_dir(), 'train_3.tf_record') + _create_fake_dataset( + input_path_1, + seq_length=60, + num_masked_tokens=20, + max_seq_length=max_seq_length, + num_examples=batch_size) + input_path_2 = os.path.join(self.get_temp_dir(), 'train_4.tf_record') + _create_fake_dataset( + input_path_2, + seq_length=60, + num_masked_tokens=15, + max_seq_length=max_seq_length, + num_examples=batch_size) + input_paths = ','.join([input_path_1, input_path_2]) + data_config = pretrain_dynamic_dataloader.BertPretrainDataConfig( + is_training=False, + input_path=input_paths, + seq_bucket_lengths=[64, 128], + use_position_id=True, + global_batch_size=batch_size * 2) + dataset = pretrain_dynamic_dataloader.PretrainingDynamicDataLoader( + data_config).load() + dataset_it = iter(dataset) + with self.assertRaisesRegex( + tf.errors.InvalidArgumentError, '.*Number of non padded mask tokens.*'): + next(dataset_it) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/question_answering_dataloader.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/question_answering_dataloader.py new file mode 100644 index 0000000000000000000000000000000000000000..0f721ed773a927e8caa8c3cfbaa5cf2ef6c896e5 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/question_answering_dataloader.py @@ -0,0 +1,110 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Loads dataset for the question answering (e.g, SQuAD) task.""" +from typing import Mapping, Optional + +import dataclasses +import tensorflow as tf +from official.core import config_definitions as cfg +from official.core import input_reader +from official.nlp.data import data_loader +from official.nlp.data import data_loader_factory + + +@dataclasses.dataclass +class QADataConfig(cfg.DataConfig): + """Data config for question answering task (tasks/question_answering).""" + # For training, `input_path` is expected to be a pre-processed TFRecord file, + # while for evaluation, it is expected to be a raw JSON file (b/173814590). + input_path: str = '' + global_batch_size: int = 48 + is_training: bool = True + seq_length: int = 384 + # Settings below are question answering specific. + version_2_with_negative: bool = False + # Settings below are only used for eval mode. + input_preprocessed_data_path: str = '' + doc_stride: int = 128 + query_length: int = 64 + # The path to the vocab file of word piece tokenizer or the + # model of the sentence piece tokenizer. + vocab_file: str = '' + tokenization: str = 'WordPiece' # WordPiece or SentencePiece + do_lower_case: bool = True + xlnet_format: bool = False + + +@data_loader_factory.register_data_loader_cls(QADataConfig) +class QuestionAnsweringDataLoader(data_loader.DataLoader): + """A class to load dataset for sentence prediction (classification) task.""" + + def __init__(self, params): + self._params = params + self._seq_length = params.seq_length + self._is_training = params.is_training + self._xlnet_format = params.xlnet_format + + def _decode(self, record: tf.Tensor): + """Decodes a serialized tf.Example.""" + name_to_features = { + 'input_ids': tf.io.FixedLenFeature([self._seq_length], tf.int64), + 'input_mask': tf.io.FixedLenFeature([self._seq_length], tf.int64), + 'segment_ids': tf.io.FixedLenFeature([self._seq_length], tf.int64), + } + if self._xlnet_format: + name_to_features['class_index'] = tf.io.FixedLenFeature([], tf.int64) + name_to_features['paragraph_mask'] = tf.io.FixedLenFeature( + [self._seq_length], tf.int64) + if self._is_training: + name_to_features['is_impossible'] = tf.io.FixedLenFeature([], tf.int64) + + if self._is_training: + name_to_features['start_positions'] = tf.io.FixedLenFeature([], tf.int64) + name_to_features['end_positions'] = tf.io.FixedLenFeature([], tf.int64) + else: + name_to_features['unique_ids'] = tf.io.FixedLenFeature([], tf.int64) + example = tf.io.parse_single_example(record, name_to_features) + + # tf.Example only supports tf.int64, but the TPU only supports tf.int32. + # So cast all int64 to int32. + for name in example: + t = example[name] + if t.dtype == tf.int64: + t = tf.cast(t, tf.int32) + example[name] = t + + return example + + def _parse(self, record: Mapping[str, tf.Tensor]): + """Parses raw tensors into a dict of tensors to be consumed by the model.""" + x, y = {}, {} + for name, tensor in record.items(): + if name in ('start_positions', 'end_positions', 'is_impossible'): + y[name] = tensor + elif name == 'input_ids': + x['input_word_ids'] = tensor + elif name == 'segment_ids': + x['input_type_ids'] = tensor + else: + x[name] = tensor + if name == 'start_positions' and self._xlnet_format: + x[name] = tensor + return (x, y) + + def load(self, input_context: Optional[tf.distribute.InputContext] = None): + """Returns a tf.dataset.Dataset.""" + reader = input_reader.InputReader( + params=self._params, decoder_fn=self._decode, parser_fn=self._parse) + return reader.read(input_context) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/question_answering_dataloader_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/question_answering_dataloader_test.py new file mode 100644 index 0000000000000000000000000000000000000000..c853bc080cddf9fc5c26a0f7f21cff19088bad9f --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/question_answering_dataloader_test.py @@ -0,0 +1,74 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for official.nlp.data.question_answering_dataloader.""" +import os + +import numpy as np +import tensorflow as tf + +from official.nlp.data import question_answering_dataloader + + +def _create_fake_dataset(output_path, seq_length): + """Creates a fake dataset.""" + writer = tf.io.TFRecordWriter(output_path) + + def create_int_feature(values): + f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + return f + + for _ in range(100): + features = {} + input_ids = np.random.randint(100, size=(seq_length)) + features['input_ids'] = create_int_feature(input_ids) + features['input_mask'] = create_int_feature(np.ones_like(input_ids)) + features['segment_ids'] = create_int_feature(np.ones_like(input_ids)) + features['start_positions'] = create_int_feature(np.array([0])) + features['end_positions'] = create_int_feature(np.array([10])) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + writer.write(tf_example.SerializeToString()) + writer.close() + + +class QuestionAnsweringDataTest(tf.test.TestCase): + + def test_load_dataset(self): + seq_length = 128 + batch_size = 10 + input_path = os.path.join(self.get_temp_dir(), 'train.tf_record') + _create_fake_dataset(input_path, seq_length) + data_config = question_answering_dataloader.QADataConfig( + is_training=True, + input_path=input_path, + seq_length=seq_length, + global_batch_size=batch_size) + dataset = question_answering_dataloader.QuestionAnsweringDataLoader( + data_config).load() + features, labels = next(iter(dataset)) + + self.assertCountEqual(['input_word_ids', 'input_mask', 'input_type_ids'], + features.keys()) + self.assertEqual(features['input_word_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['input_mask'].shape, (batch_size, seq_length)) + self.assertEqual(features['input_type_ids'].shape, (batch_size, seq_length)) + + self.assertCountEqual(['start_positions', 'end_positions'], labels.keys()) + self.assertEqual(labels['start_positions'].shape, (batch_size,)) + self.assertEqual(labels['end_positions'].shape, (batch_size,)) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/sentence_prediction_dataloader.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/sentence_prediction_dataloader.py new file mode 100644 index 0000000000000000000000000000000000000000..3517edfb9757f26869522d5c40b1c01f256de8e0 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/sentence_prediction_dataloader.py @@ -0,0 +1,267 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Loads dataset for the sentence prediction (classification) task.""" +import functools +from typing import List, Mapping, Optional, Tuple + +import dataclasses +import tensorflow as tf +import tensorflow_hub as hub + +from official.common import dataset_fn +from official.core import config_definitions as cfg +from official.core import input_reader +from official.nlp import modeling +from official.nlp.data import data_loader +from official.nlp.data import data_loader_factory + +LABEL_TYPES_MAP = {'int': tf.int64, 'float': tf.float32} + + +@dataclasses.dataclass +class SentencePredictionDataConfig(cfg.DataConfig): + """Data config for sentence prediction task (tasks/sentence_prediction).""" + input_path: str = '' + global_batch_size: int = 32 + is_training: bool = True + seq_length: int = 128 + label_type: str = 'int' + # Whether to include the example id number. + include_example_id: bool = False + label_field: str = 'label_ids' + # Maps the key in TfExample to feature name. + # E.g 'label_ids' to 'next_sentence_labels' + label_name: Optional[Tuple[str, str]] = None + # Either tfrecord, sstable, or recordio. + file_type: str = 'tfrecord' + + +@data_loader_factory.register_data_loader_cls(SentencePredictionDataConfig) +class SentencePredictionDataLoader(data_loader.DataLoader): + """A class to load dataset for sentence prediction (classification) task.""" + + def __init__(self, params): + self._params = params + self._seq_length = params.seq_length + self._include_example_id = params.include_example_id + self._label_field = params.label_field + if params.label_name: + self._label_name_mapping = dict([params.label_name]) + else: + self._label_name_mapping = dict() + + def name_to_features_spec(self): + """Defines features to decode. Subclass may override to append features.""" + label_type = LABEL_TYPES_MAP[self._params.label_type] + name_to_features = { + 'input_ids': tf.io.FixedLenFeature([self._seq_length], tf.int64), + 'input_mask': tf.io.FixedLenFeature([self._seq_length], tf.int64), + 'segment_ids': tf.io.FixedLenFeature([self._seq_length], tf.int64), + self._label_field: tf.io.FixedLenFeature([], label_type), + } + if self._include_example_id: + name_to_features['example_id'] = tf.io.FixedLenFeature([], tf.int64) + + return name_to_features + + def _decode(self, record: tf.Tensor): + """Decodes a serialized tf.Example.""" + example = tf.io.parse_single_example(record, self.name_to_features_spec()) + + # tf.Example only supports tf.int64, but the TPU only supports tf.int32. + # So cast all int64 to int32. + for name in example: + t = example[name] + if t.dtype == tf.int64: + t = tf.cast(t, tf.int32) + example[name] = t + + return example + + def _parse(self, record: Mapping[str, tf.Tensor]): + """Parses raw tensors into a dict of tensors to be consumed by the model.""" + key_mapping = { + 'input_ids': 'input_word_ids', + 'input_mask': 'input_mask', + 'segment_ids': 'input_type_ids' + } + ret = {} + for record_key in record: + if record_key in key_mapping: + ret[key_mapping[record_key]] = record[record_key] + else: + ret[record_key] = record[record_key] + + if self._label_field in self._label_name_mapping: + ret[self._label_name_mapping[self._label_field]] = record[ + self._label_field] + + return ret + + def load(self, input_context: Optional[tf.distribute.InputContext] = None): + """Returns a tf.dataset.Dataset.""" + reader = input_reader.InputReader( + dataset_fn=dataset_fn.pick_dataset_fn(self._params.file_type), + params=self._params, + decoder_fn=self._decode, + parser_fn=self._parse) + return reader.read(input_context) + + +@dataclasses.dataclass +class SentencePredictionTextDataConfig(cfg.DataConfig): + """Data config for sentence prediction task with raw text.""" + # Either set `input_path`... + input_path: str = '' + # Either `int` or `float`. + label_type: str = 'int' + # ...or `tfds_name` and `tfds_split` to specify input. + tfds_name: str = '' + tfds_split: str = '' + # The name of the text feature fields. The text features will be + # concatenated in order. + text_fields: Optional[List[str]] = None + label_field: str = 'label' + global_batch_size: int = 32 + seq_length: int = 128 + is_training: bool = True + # Either build preprocessing with Python code by specifying these values + # for modeling.layers.BertTokenizer()/SentencepieceTokenizer().... + tokenization: str = 'WordPiece' # WordPiece or SentencePiece + # Text vocab file if tokenization is WordPiece, or sentencepiece.ModelProto + # file if tokenization is SentencePiece. + vocab_file: str = '' + lower_case: bool = True + # ...or load preprocessing from a SavedModel at this location. + preprocessing_hub_module_url: str = '' + # Either tfrecord or sstsable or recordio. + file_type: str = 'tfrecord' + include_example_id: bool = False + + +class TextProcessor(tf.Module): + """Text features processing for sentence prediction task.""" + + def __init__(self, + seq_length: int, + vocab_file: Optional[str] = None, + tokenization: Optional[str] = None, + lower_case: Optional[bool] = True, + preprocessing_hub_module_url: Optional[str] = None): + if preprocessing_hub_module_url: + self._preprocessing_hub_module = hub.load(preprocessing_hub_module_url) + self._tokenizer = self._preprocessing_hub_module.tokenize + self._pack_inputs = functools.partial( + self._preprocessing_hub_module.bert_pack_inputs, + seq_length=seq_length) + return + + if tokenization == 'WordPiece': + self._tokenizer = modeling.layers.BertTokenizer( + vocab_file=vocab_file, lower_case=lower_case) + elif tokenization == 'SentencePiece': + self._tokenizer = modeling.layers.SentencepieceTokenizer( + model_file_path=vocab_file, + lower_case=lower_case, + strip_diacritics=True) # Strip diacritics to follow ALBERT model + else: + raise ValueError('Unsupported tokenization: %s' % tokenization) + + self._pack_inputs = modeling.layers.BertPackInputs( + seq_length=seq_length, + special_tokens_dict=self._tokenizer.get_special_tokens_dict()) + + def __call__(self, segments): + segments = [self._tokenizer(s) for s in segments] + # BertTokenizer returns a RaggedTensor with shape [batch, word, subword], + # and SentencepieceTokenizer returns a RaggedTensor with shape + # [batch, sentencepiece], + segments = [ + tf.cast(x.merge_dims(1, -1) if x.shape.rank > 2 else x, tf.int32) + for x in segments + ] + return self._pack_inputs(segments) + + +@data_loader_factory.register_data_loader_cls(SentencePredictionTextDataConfig) +class SentencePredictionTextDataLoader(data_loader.DataLoader): + """Loads dataset with raw text for sentence prediction task.""" + + def __init__(self, params): + if bool(params.tfds_name) != bool(params.tfds_split): + raise ValueError('`tfds_name` and `tfds_split` should be specified or ' + 'unspecified at the same time.') + if bool(params.tfds_name) == bool(params.input_path): + raise ValueError('Must specify either `tfds_name` and `tfds_split` ' + 'or `input_path`.') + if not params.text_fields: + raise ValueError('Unexpected empty text fields.') + if bool(params.vocab_file) == bool(params.preprocessing_hub_module_url): + raise ValueError('Must specify exactly one of vocab_file (with matching ' + 'lower_case flag) or preprocessing_hub_module_url.') + + self._params = params + self._text_fields = params.text_fields + self._label_field = params.label_field + self._label_type = params.label_type + self._include_example_id = params.include_example_id + self._text_processor = TextProcessor( + seq_length=params.seq_length, + vocab_file=params.vocab_file, + tokenization=params.tokenization, + lower_case=params.lower_case, + preprocessing_hub_module_url=params.preprocessing_hub_module_url) + + def _bert_preprocess(self, record: Mapping[str, tf.Tensor]): + """Berts preprocess.""" + segments = [record[x] for x in self._text_fields] + model_inputs = self._text_processor(segments) + for key in record: + if key not in self._text_fields: + model_inputs[key] = record[key] + return model_inputs + + def name_to_features_spec(self): + name_to_features = {} + for text_field in self._text_fields: + name_to_features[text_field] = tf.io.FixedLenFeature([], tf.string) + + label_type = LABEL_TYPES_MAP[self._label_type] + name_to_features[self._label_field] = tf.io.FixedLenFeature([], label_type) + if self._include_example_id: + name_to_features['example_id'] = tf.io.FixedLenFeature([], tf.int64) + return name_to_features + + def _decode(self, record: tf.Tensor): + """Decodes a serialized tf.Example.""" + example = tf.io.parse_single_example(record, self.name_to_features_spec()) + # tf.Example only supports tf.int64, but the TPU only supports tf.int32. + # So cast all int64 to int32. + for name in example: + t = example[name] + if t.dtype == tf.int64: + t = tf.cast(t, tf.int32) + example[name] = t + + return example + + def load(self, input_context: Optional[tf.distribute.InputContext] = None): + """Returns a tf.dataset.Dataset.""" + reader = input_reader.InputReader( + dataset_fn=dataset_fn.pick_dataset_fn(self._params.file_type), + decoder_fn=self._decode if self._params.input_path else None, + params=self._params, + postprocess_fn=self._bert_preprocess) + return reader.read(input_context) diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/sentence_prediction_dataloader_test.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/sentence_prediction_dataloader_test.py new file mode 100644 index 0000000000000000000000000000000000000000..876b9d421d26a06d8442570a65a60e63022c2fd1 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/sentence_prediction_dataloader_test.py @@ -0,0 +1,290 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for official.nlp.data.sentence_prediction_dataloader.""" +import os + +from absl.testing import parameterized +import numpy as np +import tensorflow as tf + +from sentencepiece import SentencePieceTrainer +from official.nlp.data import sentence_prediction_dataloader as loader + + +def _create_fake_preprocessed_dataset(output_path, seq_length, label_type): + """Creates a fake dataset.""" + writer = tf.io.TFRecordWriter(output_path) + + def create_int_feature(values): + f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + return f + + def create_float_feature(values): + f = tf.train.Feature(float_list=tf.train.FloatList(value=list(values))) + return f + + for _ in range(100): + features = {} + input_ids = np.random.randint(100, size=(seq_length)) + features['input_ids'] = create_int_feature(input_ids) + features['input_mask'] = create_int_feature(np.ones_like(input_ids)) + features['segment_ids'] = create_int_feature(np.ones_like(input_ids)) + + if label_type == 'int': + features['label_ids'] = create_int_feature([1]) + elif label_type == 'float': + features['label_ids'] = create_float_feature([0.5]) + else: + raise ValueError('Unsupported label_type: %s' % label_type) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + writer.write(tf_example.SerializeToString()) + writer.close() + + +def _create_fake_raw_dataset(output_path, text_fields, label_type): + """Creates a fake tf record file.""" + writer = tf.io.TFRecordWriter(output_path) + + def create_str_feature(value): + f = tf.train.Feature(bytes_list=tf.train.BytesList(value=value)) + return f + + def create_int_feature(values): + f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + return f + + def create_float_feature(values): + f = tf.train.Feature(float_list=tf.train.FloatList(value=list(values))) + return f + + for _ in range(100): + features = {} + for text_field in text_fields: + features[text_field] = create_str_feature([b'hello world']) + + if label_type == 'int': + features['label'] = create_int_feature([0]) + elif label_type == 'float': + features['label'] = create_float_feature([0.5]) + else: + raise ValueError('Unexpected label_type: %s' % label_type) + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + writer.write(tf_example.SerializeToString()) + writer.close() + + +def _create_fake_sentencepiece_model(output_dir): + vocab = ['a', 'b', 'c', 'd', 'e', 'abc', 'def', 'ABC', 'DEF'] + model_prefix = os.path.join(output_dir, 'spm_model') + input_text_file_path = os.path.join(output_dir, 'train_input.txt') + with tf.io.gfile.GFile(input_text_file_path, 'w') as f: + f.write(' '.join(vocab + ['\n'])) + # Add 7 more tokens: , , [CLS], [SEP], [MASK], , . + full_vocab_size = len(vocab) + 7 + flags = dict( + model_prefix=model_prefix, + model_type='word', + input=input_text_file_path, + pad_id=0, + unk_id=1, + control_symbols='[CLS],[SEP],[MASK]', + vocab_size=full_vocab_size, + bos_id=full_vocab_size - 2, + eos_id=full_vocab_size - 1) + SentencePieceTrainer.Train(' '.join( + ['--{}={}'.format(k, v) for k, v in flags.items()])) + return model_prefix + '.model' + + +def _create_fake_vocab_file(vocab_file_path): + tokens = ['[PAD]'] + for i in range(1, 100): + tokens.append('[unused%d]' % i) + tokens.extend(['[UNK]', '[CLS]', '[SEP]', '[MASK]', 'hello', 'world']) + with tf.io.gfile.GFile(vocab_file_path, 'w') as outfile: + outfile.write('\n'.join(tokens)) + + +class SentencePredictionDataTest(tf.test.TestCase, parameterized.TestCase): + + @parameterized.parameters(('int', tf.int32), ('float', tf.float32)) + def test_load_dataset(self, label_type, expected_label_type): + input_path = os.path.join(self.get_temp_dir(), 'train.tf_record') + batch_size = 10 + seq_length = 128 + _create_fake_preprocessed_dataset(input_path, seq_length, label_type) + data_config = loader.SentencePredictionDataConfig( + input_path=input_path, + seq_length=seq_length, + global_batch_size=batch_size, + label_type=label_type) + dataset = loader.SentencePredictionDataLoader(data_config).load() + features = next(iter(dataset)) + self.assertCountEqual( + ['input_word_ids', 'input_type_ids', 'input_mask', 'label_ids'], + features.keys()) + self.assertEqual(features['input_word_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['input_mask'].shape, (batch_size, seq_length)) + self.assertEqual(features['input_type_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['label_ids'].shape, (batch_size,)) + self.assertEqual(features['label_ids'].dtype, expected_label_type) + + def test_load_dataset_with_label_mapping(self): + input_path = os.path.join(self.get_temp_dir(), 'train.tf_record') + batch_size = 10 + seq_length = 128 + _create_fake_preprocessed_dataset(input_path, seq_length, 'int') + data_config = loader.SentencePredictionDataConfig( + input_path=input_path, + seq_length=seq_length, + global_batch_size=batch_size, + label_type='int', + label_name=('label_ids', 'next_sentence_labels')) + dataset = loader.SentencePredictionDataLoader(data_config).load() + features = next(iter(dataset)) + self.assertCountEqual([ + 'input_word_ids', 'input_mask', 'input_type_ids', + 'next_sentence_labels', 'label_ids' + ], features.keys()) + self.assertEqual(features['input_word_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['input_mask'].shape, (batch_size, seq_length)) + self.assertEqual(features['input_type_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['label_ids'].shape, (batch_size,)) + self.assertEqual(features['label_ids'].dtype, tf.int32) + self.assertEqual(features['next_sentence_labels'].shape, (batch_size,)) + self.assertEqual(features['next_sentence_labels'].dtype, tf.int32) + + +class SentencePredictionTfdsDataLoaderTest(tf.test.TestCase, + parameterized.TestCase): + + @parameterized.parameters(True, False) + def test_python_wordpiece_preprocessing(self, use_tfds): + batch_size = 10 + seq_length = 256 # Non-default value. + lower_case = True + + tf_record_path = os.path.join(self.get_temp_dir(), 'train.tf_record') + text_fields = ['sentence1', 'sentence2'] + if not use_tfds: + _create_fake_raw_dataset(tf_record_path, text_fields, label_type='int') + + vocab_file_path = os.path.join(self.get_temp_dir(), 'vocab.txt') + _create_fake_vocab_file(vocab_file_path) + + data_config = loader.SentencePredictionTextDataConfig( + input_path='' if use_tfds else tf_record_path, + tfds_name='glue/mrpc' if use_tfds else '', + tfds_split='train' if use_tfds else '', + text_fields=text_fields, + global_batch_size=batch_size, + seq_length=seq_length, + is_training=True, + lower_case=lower_case, + vocab_file=vocab_file_path) + dataset = loader.SentencePredictionTextDataLoader(data_config).load() + features = next(iter(dataset)) + label_field = data_config.label_field + expected_keys = [ + 'input_word_ids', 'input_type_ids', 'input_mask', label_field + ] + if use_tfds: + expected_keys += ['idx'] + self.assertCountEqual(expected_keys, features.keys()) + self.assertEqual(features['input_word_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['input_mask'].shape, (batch_size, seq_length)) + self.assertEqual(features['input_type_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features[label_field].shape, (batch_size,)) + + @parameterized.parameters(True, False) + def test_python_sentencepiece_preprocessing(self, use_tfds): + batch_size = 10 + seq_length = 256 # Non-default value. + lower_case = True + + tf_record_path = os.path.join(self.get_temp_dir(), 'train.tf_record') + text_fields = ['sentence1', 'sentence2'] + if not use_tfds: + _create_fake_raw_dataset(tf_record_path, text_fields, label_type='int') + + sp_model_file_path = _create_fake_sentencepiece_model(self.get_temp_dir()) + data_config = loader.SentencePredictionTextDataConfig( + input_path='' if use_tfds else tf_record_path, + tfds_name='glue/mrpc' if use_tfds else '', + tfds_split='train' if use_tfds else '', + text_fields=text_fields, + global_batch_size=batch_size, + seq_length=seq_length, + is_training=True, + lower_case=lower_case, + tokenization='SentencePiece', + vocab_file=sp_model_file_path, + ) + dataset = loader.SentencePredictionTextDataLoader(data_config).load() + features = next(iter(dataset)) + label_field = data_config.label_field + expected_keys = [ + 'input_word_ids', 'input_type_ids', 'input_mask', label_field + ] + if use_tfds: + expected_keys += ['idx'] + self.assertCountEqual(expected_keys, features.keys()) + self.assertEqual(features['input_word_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['input_mask'].shape, (batch_size, seq_length)) + self.assertEqual(features['input_type_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features[label_field].shape, (batch_size,)) + + @parameterized.parameters(True, False) + def test_saved_model_preprocessing(self, use_tfds): + batch_size = 10 + seq_length = 256 # Non-default value. + + tf_record_path = os.path.join(self.get_temp_dir(), 'train.tf_record') + text_fields = ['sentence1', 'sentence2'] + if not use_tfds: + _create_fake_raw_dataset(tf_record_path, text_fields, label_type='float') + + vocab_file_path = os.path.join(self.get_temp_dir(), 'vocab.txt') + _create_fake_vocab_file(vocab_file_path) + data_config = loader.SentencePredictionTextDataConfig( + input_path='' if use_tfds else tf_record_path, + tfds_name='glue/mrpc' if use_tfds else '', + tfds_split='train' if use_tfds else '', + text_fields=text_fields, + global_batch_size=batch_size, + seq_length=seq_length, + is_training=True, + preprocessing_hub_module_url=( + 'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3'), + label_type='int' if use_tfds else 'float', + ) + dataset = loader.SentencePredictionTextDataLoader(data_config).load() + features = next(iter(dataset)) + label_field = data_config.label_field + expected_keys = [ + 'input_word_ids', 'input_type_ids', 'input_mask', label_field + ] + if use_tfds: + expected_keys += ['idx'] + self.assertCountEqual(expected_keys, features.keys()) + self.assertEqual(features['input_word_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features['input_mask'].shape, (batch_size, seq_length)) + self.assertEqual(features['input_type_ids'].shape, (batch_size, seq_length)) + self.assertEqual(features[label_field].shape, (batch_size,)) + + +if __name__ == '__main__': + tf.test.main() diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/sentence_retrieval_lib.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/sentence_retrieval_lib.py new file mode 100644 index 0000000000000000000000000000000000000000..0bfd8e4dec5afba3eb00ff23e3f75a0cc5818958 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/sentence_retrieval_lib.py @@ -0,0 +1,166 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""BERT library to process data for cross lingual sentence retrieval task.""" + +import os + +from absl import logging +from official.nlp.bert import tokenization +from official.nlp.data import classifier_data_lib + + +class BuccProcessor(classifier_data_lib.DataProcessor): + """Procssor for Xtreme BUCC data set.""" + supported_languages = ["de", "fr", "ru", "zh"] + + def __init__(self, process_text_fn=tokenization.convert_to_unicode): + super(BuccProcessor, self).__init__(process_text_fn) + self.languages = BuccProcessor.supported_languages + + def get_dev_examples(self, data_dir, file_pattern): + return self._create_examples( + self._read_tsv(os.path.join(data_dir, file_pattern.format("dev"))), + "sample") + + def get_test_examples(self, data_dir, file_pattern): + return self._create_examples( + self._read_tsv(os.path.join(data_dir, file_pattern.format("test"))), + "test") + + @staticmethod + def get_processor_name(): + """See base class.""" + return "BUCC" + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + guid = "%s-%s" % (set_type, i) + example_id = int(line[0].split("-")[1]) + text_a = self.process_text_fn(line[1]) + examples.append( + classifier_data_lib.InputExample( + guid=guid, text_a=text_a, example_id=example_id)) + return examples + + +class TatoebaProcessor(classifier_data_lib.DataProcessor): + """Procssor for Xtreme Tatoeba data set.""" + supported_languages = [ + "af", "ar", "bg", "bn", "de", "el", "es", "et", "eu", "fa", "fi", "fr", + "he", "hi", "hu", "id", "it", "ja", "jv", "ka", "kk", "ko", "ml", "mr", + "nl", "pt", "ru", "sw", "ta", "te", "th", "tl", "tr", "ur", "vi", "zh" + ] + + def __init__(self, process_text_fn=tokenization.convert_to_unicode): + super(TatoebaProcessor, self).__init__(process_text_fn) + self.languages = TatoebaProcessor.supported_languages + + def get_test_examples(self, data_dir, file_path): + return self._create_examples( + self._read_tsv(os.path.join(data_dir, file_path)), "test") + + @staticmethod + def get_processor_name(): + """See base class.""" + return "TATOEBA" + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + guid = "%s-%s" % (set_type, i) + text_a = self.process_text_fn(line[0]) + examples.append( + classifier_data_lib.InputExample( + guid=guid, text_a=text_a, example_id=i)) + return examples + + +def generate_sentence_retrevial_tf_record(processor, + data_dir, + tokenizer, + eval_data_output_path=None, + test_data_output_path=None, + max_seq_length=128): + """Generates the tf records for retrieval tasks. + + Args: + processor: Input processor object to be used for generating data. Subclass + of `DataProcessor`. + data_dir: Directory that contains train/eval data to process. Data files + should be in from. + tokenizer: The tokenizer to be applied on the data. + eval_data_output_path: Output to which processed tf record for evaluation + will be saved. + test_data_output_path: Output to which processed tf record for testing + will be saved. Must be a pattern template with {} if processor has + language specific test data. + max_seq_length: Maximum sequence length of the to be generated + training/eval data. + + Returns: + A dictionary containing input meta data. + """ + assert eval_data_output_path or test_data_output_path + + if processor.get_processor_name() == "BUCC": + path_pattern = "{}-en.{{}}.{}" + + if processor.get_processor_name() == "TATOEBA": + path_pattern = "{}-en.{}" + + meta_data = { + "processor_type": processor.get_processor_name(), + "max_seq_length": max_seq_length, + "number_eval_data": {}, + "number_test_data": {}, + } + logging.info("Start to process %s task data", processor.get_processor_name()) + + for lang_a in processor.languages: + for lang_b in [lang_a, "en"]: + if eval_data_output_path: + eval_input_data_examples = processor.get_dev_examples( + data_dir, os.path.join(path_pattern.format(lang_a, lang_b))) + + num_eval_data = len(eval_input_data_examples) + logging.info("Processing %d dev examples of %s-en.%s", num_eval_data, + lang_a, lang_b) + output_file = os.path.join( + eval_data_output_path, + "{}-en-{}.{}.tfrecords".format(lang_a, lang_b, "dev")) + classifier_data_lib.file_based_convert_examples_to_features( + eval_input_data_examples, None, max_seq_length, tokenizer, + output_file, None) + meta_data["number_eval_data"][f"{lang_a}-en.{lang_b}"] = num_eval_data + + if test_data_output_path: + test_input_data_examples = processor.get_test_examples( + data_dir, os.path.join(path_pattern.format(lang_a, lang_b))) + + num_test_data = len(test_input_data_examples) + logging.info("Processing %d test examples of %s-en.%s", num_test_data, + lang_a, lang_b) + output_file = os.path.join( + test_data_output_path, + "{}-en-{}.{}.tfrecords".format(lang_a, lang_b, "test")) + classifier_data_lib.file_based_convert_examples_to_features( + test_input_data_examples, None, max_seq_length, tokenizer, + output_file, None) + meta_data["number_test_data"][f"{lang_a}-en.{lang_b}"] = num_test_data + + return meta_data diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/squad_lib.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/squad_lib.py new file mode 100644 index 0000000000000000000000000000000000000000..e96838664c38db4f6cdc2d39f10ad68baeac25e5 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/squad_lib.py @@ -0,0 +1,975 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Library to process data for SQuAD 1.1 and SQuAD 2.0.""" +# pylint: disable=g-bad-import-order +import collections +import copy +import json +import math +import os + +import six + +from absl import logging +import tensorflow as tf + +from official.nlp.bert import tokenization + + +class SquadExample(object): + """A single training/test example for simple sequence classification. + + For examples without an answer, the start and end position are -1. + + Attributes: + qas_id: ID of the question-answer pair. + question_text: Original text for the question. + doc_tokens: The list of tokens in the context obtained by splitting on + whitespace only. + orig_answer_text: Original text for the answer. + start_position: Starting index of the answer in `doc_tokens`. + end_position: Ending index of the answer in `doc_tokens`. + is_impossible: Whether the question is impossible to answer given the + context. Only used in SQuAD 2.0. + """ + + def __init__(self, + qas_id, + question_text, + doc_tokens, + orig_answer_text=None, + start_position=None, + end_position=None, + is_impossible=False): + self.qas_id = qas_id + self.question_text = question_text + self.doc_tokens = doc_tokens + self.orig_answer_text = orig_answer_text + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + + def __str__(self): + return self.__repr__() + + def __repr__(self): + s = "" + s += "qas_id: %s" % (tokenization.printable_text(self.qas_id)) + s += ", question_text: %s" % ( + tokenization.printable_text(self.question_text)) + s += ", doc_tokens: [%s]" % (" ".join(self.doc_tokens)) + if self.start_position: + s += ", start_position: %d" % (self.start_position) + if self.start_position: + s += ", end_position: %d" % (self.end_position) + if self.start_position: + s += ", is_impossible: %r" % (self.is_impossible) + return s + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, + unique_id, + example_index, + doc_span_index, + tokens, + token_to_orig_map, + token_is_max_context, + input_ids, + input_mask, + segment_ids, + paragraph_mask=None, + class_index=None, + start_position=None, + end_position=None, + is_impossible=None): + self.unique_id = unique_id + self.example_index = example_index + self.doc_span_index = doc_span_index + self.tokens = tokens + self.token_to_orig_map = token_to_orig_map + self.token_is_max_context = token_is_max_context + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + self.paragraph_mask = paragraph_mask + self.class_index = class_index + + +class FeatureWriter(object): + """Writes InputFeature to TF example file.""" + + def __init__(self, filename, is_training): + self.filename = filename + self.is_training = is_training + self.num_features = 0 + tf.io.gfile.makedirs(os.path.dirname(filename)) + self._writer = tf.io.TFRecordWriter(filename) + + def process_feature(self, feature): + """Write a InputFeature to the TFRecordWriter as a tf.train.Example.""" + self.num_features += 1 + + def create_int_feature(values): + feature = tf.train.Feature( + int64_list=tf.train.Int64List(value=list(values))) + return feature + + features = collections.OrderedDict() + features["unique_ids"] = create_int_feature([feature.unique_id]) + features["input_ids"] = create_int_feature(feature.input_ids) + features["input_mask"] = create_int_feature(feature.input_mask) + features["segment_ids"] = create_int_feature(feature.segment_ids) + + if feature.paragraph_mask is not None: + features["paragraph_mask"] = create_int_feature(feature.paragraph_mask) + if feature.class_index is not None: + features["class_index"] = create_int_feature([feature.class_index]) + + if self.is_training: + features["start_positions"] = create_int_feature([feature.start_position]) + features["end_positions"] = create_int_feature([feature.end_position]) + impossible = 0 + if feature.is_impossible: + impossible = 1 + features["is_impossible"] = create_int_feature([impossible]) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + self._writer.write(tf_example.SerializeToString()) + + def close(self): + self._writer.close() + + +def read_squad_examples(input_file, is_training, + version_2_with_negative, + translated_input_folder=None): + """Read a SQuAD json file into a list of SquadExample.""" + with tf.io.gfile.GFile(input_file, "r") as reader: + input_data = json.load(reader)["data"] + + if translated_input_folder is not None: + translated_files = tf.io.gfile.glob( + os.path.join(translated_input_folder, "*.json")) + for file in translated_files: + with tf.io.gfile.GFile(file, "r") as reader: + input_data.extend(json.load(reader)["data"]) + + def is_whitespace(c): + if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: + return True + return False + + examples = [] + for entry in input_data: + for paragraph in entry["paragraphs"]: + paragraph_text = paragraph["context"] + doc_tokens = [] + char_to_word_offset = [] + prev_is_whitespace = True + for c in paragraph_text: + if is_whitespace(c): + prev_is_whitespace = True + else: + if prev_is_whitespace: + doc_tokens.append(c) + else: + doc_tokens[-1] += c + prev_is_whitespace = False + char_to_word_offset.append(len(doc_tokens) - 1) + + for qa in paragraph["qas"]: + qas_id = qa["id"] + question_text = qa["question"] + start_position = None + end_position = None + orig_answer_text = None + is_impossible = False + if is_training: + + if version_2_with_negative: + is_impossible = qa["is_impossible"] + if (len(qa["answers"]) != 1) and (not is_impossible): + raise ValueError( + "For training, each question should have exactly 1 answer.") + if not is_impossible: + answer = qa["answers"][0] + orig_answer_text = answer["text"] + answer_offset = answer["answer_start"] + answer_length = len(orig_answer_text) + start_position = char_to_word_offset[answer_offset] + end_position = char_to_word_offset[answer_offset + answer_length - + 1] + # Only add answers where the text can be exactly recovered from the + # document. If this CAN'T happen it's likely due to weird Unicode + # stuff so we will just skip the example. + # + # Note that this means for training mode, every example is NOT + # guaranteed to be preserved. + actual_text = " ".join(doc_tokens[start_position:(end_position + + 1)]) + cleaned_answer_text = " ".join( + tokenization.whitespace_tokenize(orig_answer_text)) + if actual_text.find(cleaned_answer_text) == -1: + logging.warning("Could not find answer: '%s' vs. '%s'", + actual_text, cleaned_answer_text) + continue + else: + start_position = -1 + end_position = -1 + orig_answer_text = "" + + example = SquadExample( + qas_id=qas_id, + question_text=question_text, + doc_tokens=doc_tokens, + orig_answer_text=orig_answer_text, + start_position=start_position, + end_position=end_position, + is_impossible=is_impossible) + examples.append(example) + + return examples + + +def convert_examples_to_features(examples, + tokenizer, + max_seq_length, + doc_stride, + max_query_length, + is_training, + output_fn, + xlnet_format=False, + batch_size=None): + """Loads a data file into a list of `InputBatch`s.""" + + base_id = 1000000000 + unique_id = base_id + feature = None + for (example_index, example) in enumerate(examples): + query_tokens = tokenizer.tokenize(example.question_text) + + if len(query_tokens) > max_query_length: + query_tokens = query_tokens[0:max_query_length] + + tok_to_orig_index = [] + orig_to_tok_index = [] + all_doc_tokens = [] + for (i, token) in enumerate(example.doc_tokens): + orig_to_tok_index.append(len(all_doc_tokens)) + sub_tokens = tokenizer.tokenize(token) + for sub_token in sub_tokens: + tok_to_orig_index.append(i) + all_doc_tokens.append(sub_token) + + tok_start_position = None + tok_end_position = None + if is_training and example.is_impossible: + tok_start_position = -1 + tok_end_position = -1 + if is_training and not example.is_impossible: + tok_start_position = orig_to_tok_index[example.start_position] + if example.end_position < len(example.doc_tokens) - 1: + tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 + else: + tok_end_position = len(all_doc_tokens) - 1 + (tok_start_position, tok_end_position) = _improve_answer_span( + all_doc_tokens, tok_start_position, tok_end_position, tokenizer, + example.orig_answer_text) + + # The -3 accounts for [CLS], [SEP] and [SEP] + max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 + + # We can have documents that are longer than the maximum sequence length. + # To deal with this we do a sliding window approach, where we take chunks + # of the up to our max length with a stride of `doc_stride`. + _DocSpan = collections.namedtuple( # pylint: disable=invalid-name + "DocSpan", ["start", "length"]) + doc_spans = [] + start_offset = 0 + while start_offset < len(all_doc_tokens): + length = len(all_doc_tokens) - start_offset + if length > max_tokens_for_doc: + length = max_tokens_for_doc + doc_spans.append(_DocSpan(start=start_offset, length=length)) + if start_offset + length == len(all_doc_tokens): + break + start_offset += min(length, doc_stride) + + for (doc_span_index, doc_span) in enumerate(doc_spans): + tokens = [] + token_to_orig_map = {} + token_is_max_context = {} + segment_ids = [] + + # Paragraph mask used in XLNet. + # 1 represents paragraph and class tokens. + # 0 represents query and other special tokens. + paragraph_mask = [] + + # pylint: disable=cell-var-from-loop + def process_query(seg_q): + for token in query_tokens: + tokens.append(token) + segment_ids.append(seg_q) + paragraph_mask.append(0) + tokens.append("[SEP]") + segment_ids.append(seg_q) + paragraph_mask.append(0) + + def process_paragraph(seg_p): + for i in range(doc_span.length): + split_token_index = doc_span.start + i + token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index] + + is_max_context = _check_is_max_context(doc_spans, doc_span_index, + split_token_index) + token_is_max_context[len(tokens)] = is_max_context + tokens.append(all_doc_tokens[split_token_index]) + segment_ids.append(seg_p) + paragraph_mask.append(1) + tokens.append("[SEP]") + segment_ids.append(seg_p) + paragraph_mask.append(0) + + def process_class(seg_class): + class_index = len(segment_ids) + tokens.append("[CLS]") + segment_ids.append(seg_class) + paragraph_mask.append(1) + return class_index + + if xlnet_format: + seg_p, seg_q, seg_class, seg_pad = 0, 1, 2, 3 + process_paragraph(seg_p) + process_query(seg_q) + class_index = process_class(seg_class) + else: + seg_p, seg_q, seg_class, seg_pad = 1, 0, 0, 0 + class_index = process_class(seg_class) + process_query(seg_q) + process_paragraph(seg_p) + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < max_seq_length: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(seg_pad) + paragraph_mask.append(0) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + assert len(paragraph_mask) == max_seq_length + + start_position = 0 + end_position = 0 + span_contains_answer = False + + if is_training and not example.is_impossible: + # For training, if our document chunk does not contain an annotation + # we throw it out, since there is nothing to predict. + doc_start = doc_span.start + doc_end = doc_span.start + doc_span.length - 1 + span_contains_answer = (tok_start_position >= doc_start and + tok_end_position <= doc_end) + if span_contains_answer: + doc_offset = 0 if xlnet_format else len(query_tokens) + 2 + start_position = tok_start_position - doc_start + doc_offset + end_position = tok_end_position - doc_start + doc_offset + + if example_index < 20: + logging.info("*** Example ***") + logging.info("unique_id: %s", (unique_id)) + logging.info("example_index: %s", (example_index)) + logging.info("doc_span_index: %s", (doc_span_index)) + logging.info("tokens: %s", + " ".join([tokenization.printable_text(x) for x in tokens])) + logging.info( + "token_to_orig_map: %s", " ".join([ + "%d:%d" % (x, y) for (x, y) in six.iteritems(token_to_orig_map) + ])) + logging.info( + "token_is_max_context: %s", " ".join([ + "%d:%s" % (x, y) + for (x, y) in six.iteritems(token_is_max_context) + ])) + logging.info("input_ids: %s", " ".join([str(x) for x in input_ids])) + logging.info("input_mask: %s", " ".join([str(x) for x in input_mask])) + logging.info("segment_ids: %s", " ".join([str(x) for x in segment_ids])) + logging.info("paragraph_mask: %s", " ".join( + [str(x) for x in paragraph_mask])) + logging.info("class_index: %d", class_index) + if is_training: + if span_contains_answer: + answer_text = " ".join(tokens[start_position:(end_position + 1)]) + logging.info("start_position: %d", (start_position)) + logging.info("end_position: %d", (end_position)) + logging.info("answer: %s", tokenization.printable_text(answer_text)) + else: + logging.info("document span doesn't contain answer") + + feature = InputFeatures( + unique_id=unique_id, + example_index=example_index, + doc_span_index=doc_span_index, + tokens=tokens, + paragraph_mask=paragraph_mask, + class_index=class_index, + token_to_orig_map=token_to_orig_map, + token_is_max_context=token_is_max_context, + input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + start_position=start_position, + end_position=end_position, + is_impossible=not span_contains_answer) + + # Run callback + if is_training: + output_fn(feature) + else: + output_fn(feature, is_padding=False) + + unique_id += 1 + + if not is_training and feature: + assert batch_size + num_padding = 0 + num_examples = unique_id - base_id + if unique_id % batch_size != 0: + num_padding = batch_size - (num_examples % batch_size) + logging.info("Adding padding examples to make sure no partial batch.") + logging.info("Adds %d padding examples for inference.", num_padding) + dummy_feature = copy.deepcopy(feature) + for _ in range(num_padding): + dummy_feature.unique_id = unique_id + + # Run callback + output_fn(feature, is_padding=True) + unique_id += 1 + return unique_id - base_id + + +def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, + orig_answer_text): + """Returns tokenized answer spans that better match the annotated answer.""" + + # The SQuAD annotations are character based. We first project them to + # whitespace-tokenized words. But then after WordPiece tokenization, we can + # often find a "better match". For example: + # + # Question: What year was John Smith born? + # Context: The leader was John Smith (1895-1943). + # Answer: 1895 + # + # The original whitespace-tokenized answer will be "(1895-1943).". However + # after tokenization, our tokens will be "( 1895 - 1943 ) .". So we can match + # the exact answer, 1895. + # + # However, this is not always possible. Consider the following: + # + # Question: What country is the top exporter of electornics? + # Context: The Japanese electronics industry is the lagest in the world. + # Answer: Japan + # + # In this case, the annotator chose "Japan" as a character sub-span of + # the word "Japanese". Since our WordPiece tokenizer does not split + # "Japanese", we just use "Japanese" as the annotation. This is fairly rare + # in SQuAD, but does happen. + tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text)) + + for new_start in range(input_start, input_end + 1): + for new_end in range(input_end, new_start - 1, -1): + text_span = " ".join(doc_tokens[new_start:(new_end + 1)]) + if text_span == tok_answer_text: + return (new_start, new_end) + + return (input_start, input_end) + + +def _check_is_max_context(doc_spans, cur_span_index, position): + """Check if this is the 'max context' doc span for the token.""" + + # Because of the sliding window approach taken to scoring documents, a single + # token can appear in multiple documents. E.g. + # Doc: the man went to the store and bought a gallon of milk + # Span A: the man went to the + # Span B: to the store and bought + # Span C: and bought a gallon of + # ... + # + # Now the word 'bought' will have two scores from spans B and C. We only + # want to consider the score with "maximum context", which we define as + # the *minimum* of its left and right context (the *sum* of left and + # right context will always be the same, of course). + # + # In the example the maximum context for 'bought' would be span C since + # it has 1 left context and 3 right context, while span B has 4 left context + # and 0 right context. + best_score = None + best_span_index = None + for (span_index, doc_span) in enumerate(doc_spans): + end = doc_span.start + doc_span.length - 1 + if position < doc_span.start: + continue + if position > end: + continue + num_left_context = position - doc_span.start + num_right_context = end - position + score = min(num_left_context, num_right_context) + 0.01 * doc_span.length + if best_score is None or score > best_score: + best_score = score + best_span_index = span_index + + return cur_span_index == best_span_index + + +def write_predictions(all_examples, + all_features, + all_results, + n_best_size, + max_answer_length, + do_lower_case, + output_prediction_file, + output_nbest_file, + output_null_log_odds_file, + version_2_with_negative=False, + null_score_diff_threshold=0.0, + verbose=False): + """Write final predictions to the json file and log-odds of null if needed.""" + logging.info("Writing predictions to: %s", (output_prediction_file)) + logging.info("Writing nbest to: %s", (output_nbest_file)) + + all_predictions, all_nbest_json, scores_diff_json = ( + postprocess_output( + all_examples=all_examples, + all_features=all_features, + all_results=all_results, + n_best_size=n_best_size, + max_answer_length=max_answer_length, + do_lower_case=do_lower_case, + version_2_with_negative=version_2_with_negative, + null_score_diff_threshold=null_score_diff_threshold, + verbose=verbose)) + + write_to_json_files(all_predictions, output_prediction_file) + write_to_json_files(all_nbest_json, output_nbest_file) + if version_2_with_negative: + write_to_json_files(scores_diff_json, output_null_log_odds_file) + + +def postprocess_output(all_examples, + all_features, + all_results, + n_best_size, + max_answer_length, + do_lower_case, + version_2_with_negative=False, + null_score_diff_threshold=0.0, + xlnet_format=False, + verbose=False): + """Postprocess model output, to form predicton results.""" + + example_index_to_features = collections.defaultdict(list) + for feature in all_features: + example_index_to_features[feature.example_index].append(feature) + unique_id_to_result = {} + for result in all_results: + unique_id_to_result[result.unique_id] = result + + _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name + "PrelimPrediction", + ["feature_index", "start_index", "end_index", "start_logit", "end_logit"]) + + all_predictions = collections.OrderedDict() + all_nbest_json = collections.OrderedDict() + scores_diff_json = collections.OrderedDict() + + for (example_index, example) in enumerate(all_examples): + features = example_index_to_features[example_index] + + prelim_predictions = [] + # keep track of the minimum score of null start+end of position 0 + score_null = 1000000 # large and positive + min_null_feature_index = 0 # the paragraph slice with min mull score + null_start_logit = 0 # the start logit at the slice with min null score + null_end_logit = 0 # the end logit at the slice with min null score + for (feature_index, feature) in enumerate(features): + if feature.unique_id not in unique_id_to_result: + logging.info("Skip eval example %s, not in pred.", feature.unique_id) + continue + result = unique_id_to_result[feature.unique_id] + + # if we could have irrelevant answers, get the min score of irrelevant + if version_2_with_negative: + if xlnet_format: + feature_null_score = result.class_logits + else: + feature_null_score = result.start_logits[0] + result.end_logits[0] + if feature_null_score < score_null: + score_null = feature_null_score + min_null_feature_index = feature_index + null_start_logit = result.start_logits[0] + null_end_logit = result.end_logits[0] + for (start_index, start_logit, + end_index, end_logit) in _get_best_indexes_and_logits( + result=result, + n_best_size=n_best_size, + xlnet_format=xlnet_format): + # We could hypothetically create invalid predictions, e.g., predict + # that the start of the span is in the question. We throw out all + # invalid predictions. + if start_index >= len(feature.tokens): + continue + if end_index >= len(feature.tokens): + continue + if start_index not in feature.token_to_orig_map: + continue + if end_index not in feature.token_to_orig_map: + continue + if not feature.token_is_max_context.get(start_index, False): + continue + if end_index < start_index: + continue + length = end_index - start_index + 1 + if length > max_answer_length: + continue + prelim_predictions.append( + _PrelimPrediction( + feature_index=feature_index, + start_index=start_index, + end_index=end_index, + start_logit=start_logit, + end_logit=end_logit)) + + if version_2_with_negative and not xlnet_format: + prelim_predictions.append( + _PrelimPrediction( + feature_index=min_null_feature_index, + start_index=0, + end_index=0, + start_logit=null_start_logit, + end_logit=null_end_logit)) + prelim_predictions = sorted( + prelim_predictions, + key=lambda x: (x.start_logit + x.end_logit), + reverse=True) + + _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name + "NbestPrediction", ["text", "start_logit", "end_logit"]) + + seen_predictions = {} + nbest = [] + for pred in prelim_predictions: + if len(nbest) >= n_best_size: + break + feature = features[pred.feature_index] + if pred.start_index > 0 or xlnet_format: # this is a non-null prediction + tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)] + orig_doc_start = feature.token_to_orig_map[pred.start_index] + orig_doc_end = feature.token_to_orig_map[pred.end_index] + orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)] + tok_text = " ".join(tok_tokens) + + # De-tokenize WordPieces that have been split off. + tok_text = tok_text.replace(" ##", "") + tok_text = tok_text.replace("##", "") + + # Clean whitespace + tok_text = tok_text.strip() + tok_text = " ".join(tok_text.split()) + orig_text = " ".join(orig_tokens) + + final_text = get_final_text( + tok_text, orig_text, do_lower_case, verbose=verbose) + if final_text in seen_predictions: + continue + + seen_predictions[final_text] = True + else: + final_text = "" + seen_predictions[final_text] = True + + nbest.append( + _NbestPrediction( + text=final_text, + start_logit=pred.start_logit, + end_logit=pred.end_logit)) + + # if we didn't inlude the empty option in the n-best, inlcude it + if version_2_with_negative and not xlnet_format: + if "" not in seen_predictions: + nbest.append( + _NbestPrediction( + text="", start_logit=null_start_logit, + end_logit=null_end_logit)) + # In very rare edge cases we could have no valid predictions. So we + # just create a nonce prediction in this case to avoid failure. + if not nbest: + nbest.append( + _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) + + assert len(nbest) >= 1 + + total_scores = [] + best_non_null_entry = None + for entry in nbest: + total_scores.append(entry.start_logit + entry.end_logit) + if not best_non_null_entry: + if entry.text: + best_non_null_entry = entry + + probs = _compute_softmax(total_scores) + + nbest_json = [] + for (i, entry) in enumerate(nbest): + output = collections.OrderedDict() + output["text"] = entry.text + output["probability"] = probs[i] + output["start_logit"] = entry.start_logit + output["end_logit"] = entry.end_logit + nbest_json.append(output) + + assert len(nbest_json) >= 1 + + if not version_2_with_negative: + all_predictions[example.qas_id] = nbest_json[0]["text"] + else: + # pytype: disable=attribute-error + # predict "" iff the null score - the score of best non-null > threshold + if best_non_null_entry is not None: + if xlnet_format: + score_diff = score_null + scores_diff_json[example.qas_id] = score_diff + all_predictions[example.qas_id] = best_non_null_entry.text + else: + score_diff = score_null - best_non_null_entry.start_logit - ( + best_non_null_entry.end_logit) + scores_diff_json[example.qas_id] = score_diff + if score_diff > null_score_diff_threshold: + all_predictions[example.qas_id] = "" + else: + all_predictions[example.qas_id] = best_non_null_entry.text + else: + logging.warning("best_non_null_entry is None") + scores_diff_json[example.qas_id] = score_null + all_predictions[example.qas_id] = "" + # pytype: enable=attribute-error + + all_nbest_json[example.qas_id] = nbest_json + + return all_predictions, all_nbest_json, scores_diff_json + + +def write_to_json_files(json_records, json_file): + with tf.io.gfile.GFile(json_file, "w") as writer: + writer.write(json.dumps(json_records, indent=4) + "\n") + + +def get_final_text(pred_text, orig_text, do_lower_case, verbose=False): + """Project the tokenized prediction back to the original text.""" + + # When we created the data, we kept track of the alignment between original + # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So + # now `orig_text` contains the span of our original text corresponding to the + # span that we predicted. + # + # However, `orig_text` may contain extra characters that we don't want in + # our prediction. + # + # For example, let's say: + # pred_text = steve smith + # orig_text = Steve Smith's + # + # We don't want to return `orig_text` because it contains the extra "'s". + # + # We don't want to return `pred_text` because it's already been normalized + # (the SQuAD eval script also does punctuation stripping/lower casing but + # our tokenizer does additional normalization like stripping accent + # characters). + # + # What we really want to return is "Steve Smith". + # + # Therefore, we have to apply a semi-complicated alignment heruistic between + # `pred_text` and `orig_text` to get a character-to-charcter alignment. This + # can fail in certain cases in which case we just return `orig_text`. + + def _strip_spaces(text): + ns_chars = [] + ns_to_s_map = collections.OrderedDict() + for (i, c) in enumerate(text): + if c == " ": + continue + ns_to_s_map[len(ns_chars)] = i + ns_chars.append(c) + ns_text = "".join(ns_chars) + return (ns_text, ns_to_s_map) + + # We first tokenize `orig_text`, strip whitespace from the result + # and `pred_text`, and check if they are the same length. If they are + # NOT the same length, the heuristic has failed. If they are the same + # length, we assume the characters are one-to-one aligned. + tokenizer = tokenization.BasicTokenizer(do_lower_case=do_lower_case) + + tok_text = " ".join(tokenizer.tokenize(orig_text)) + + start_position = tok_text.find(pred_text) + if start_position == -1: + if verbose: + logging.info("Unable to find text: '%s' in '%s'", pred_text, orig_text) + return orig_text + end_position = start_position + len(pred_text) - 1 + + (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) + (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) + + if len(orig_ns_text) != len(tok_ns_text): + if verbose: + logging.info("Length not equal after stripping spaces: '%s' vs '%s'", + orig_ns_text, tok_ns_text) + return orig_text + + # We then project the characters in `pred_text` back to `orig_text` using + # the character-to-character alignment. + tok_s_to_ns_map = {} + for (i, tok_index) in six.iteritems(tok_ns_to_s_map): + tok_s_to_ns_map[tok_index] = i + + orig_start_position = None + if start_position in tok_s_to_ns_map: + ns_start_position = tok_s_to_ns_map[start_position] + if ns_start_position in orig_ns_to_s_map: + orig_start_position = orig_ns_to_s_map[ns_start_position] + + if orig_start_position is None: + if verbose: + logging.info("Couldn't map start position") + return orig_text + + orig_end_position = None + if end_position in tok_s_to_ns_map: + ns_end_position = tok_s_to_ns_map[end_position] + if ns_end_position in orig_ns_to_s_map: + orig_end_position = orig_ns_to_s_map[ns_end_position] + + if orig_end_position is None: + if verbose: + logging.info("Couldn't map end position") + return orig_text + + output_text = orig_text[orig_start_position:(orig_end_position + 1)] + return output_text + + +def _get_best_indexes_and_logits(result, + n_best_size, + xlnet_format=False): + """Generates the n-best indexes and logits from a list.""" + if xlnet_format: + for i in range(n_best_size): + for j in range(n_best_size): + j_index = i * n_best_size + j + yield (result.start_indexes[i], result.start_logits[i], + result.end_indexes[j_index], result.end_logits[j_index]) + else: + start_index_and_score = sorted(enumerate(result.start_logits), + key=lambda x: x[1], reverse=True) + end_index_and_score = sorted(enumerate(result.end_logits), + key=lambda x: x[1], reverse=True) + for i in range(len(start_index_and_score)): + if i >= n_best_size: + break + for j in range(len(end_index_and_score)): + if j >= n_best_size: + break + yield (start_index_and_score[i][0], start_index_and_score[i][1], + end_index_and_score[j][0], end_index_and_score[j][1]) + + +def _compute_softmax(scores): + """Compute softmax probability over raw logits.""" + if not scores: + return [] + + max_score = None + for score in scores: + if max_score is None or score > max_score: + max_score = score + + exp_scores = [] + total_sum = 0.0 + for score in scores: + x = math.exp(score - max_score) + exp_scores.append(x) + total_sum += x + + probs = [] + for score in exp_scores: + probs.append(score / total_sum) + return probs + + +def generate_tf_record_from_json_file(input_file_path, + vocab_file_path, + output_path, + translated_input_folder=None, + max_seq_length=384, + do_lower_case=True, + max_query_length=64, + doc_stride=128, + version_2_with_negative=False, + xlnet_format=False): + """Generates and saves training data into a tf record file.""" + train_examples = read_squad_examples( + input_file=input_file_path, + is_training=True, + version_2_with_negative=version_2_with_negative, + translated_input_folder=translated_input_folder) + tokenizer = tokenization.FullTokenizer( + vocab_file=vocab_file_path, do_lower_case=do_lower_case) + train_writer = FeatureWriter(filename=output_path, is_training=True) + number_of_examples = convert_examples_to_features( + examples=train_examples, + tokenizer=tokenizer, + max_seq_length=max_seq_length, + doc_stride=doc_stride, + max_query_length=max_query_length, + is_training=True, + output_fn=train_writer.process_feature, + xlnet_format=xlnet_format) + train_writer.close() + + meta_data = { + "task_type": "bert_squad", + "train_data_size": number_of_examples, + "max_seq_length": max_seq_length, + "max_query_length": max_query_length, + "doc_stride": doc_stride, + "version_2_with_negative": version_2_with_negative, + } + + return meta_data diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/squad_lib_sp.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/squad_lib_sp.py new file mode 100644 index 0000000000000000000000000000000000000000..021193d4114004adceb5a0197a46842cd9d4601b --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/squad_lib_sp.py @@ -0,0 +1,976 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run ALBERT on SQuAD 1.1 and SQuAD 2.0 using sentence piece tokenization. + +The file is forked from: + +https://github.com/google-research/ALBERT/blob/master/run_squad_sp.py +""" +import collections +import copy +import json +import math +import os + +from absl import logging +import numpy as np +import tensorflow as tf + +from official.nlp.bert import tokenization + + +class SquadExample(object): + """A single training/test example for simple sequence classification. + + For examples without an answer, the start and end position are -1. + """ + + def __init__(self, + qas_id, + question_text, + paragraph_text, + orig_answer_text=None, + start_position=None, + end_position=None, + is_impossible=False): + self.qas_id = qas_id + self.question_text = question_text + self.paragraph_text = paragraph_text + self.orig_answer_text = orig_answer_text + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + + def __str__(self): + return self.__repr__() + + def __repr__(self): + s = "" + s += "qas_id: %s" % (tokenization.printable_text(self.qas_id)) + s += ", question_text: %s" % ( + tokenization.printable_text(self.question_text)) + s += ", paragraph_text: [%s]" % (" ".join(self.paragraph_text)) + if self.start_position: + s += ", start_position: %d" % (self.start_position) + if self.start_position: + s += ", end_position: %d" % (self.end_position) + if self.start_position: + s += ", is_impossible: %r" % (self.is_impossible) + return s + + +class InputFeatures(object): + """A single set of features of data.""" + + def __init__(self, + unique_id, + example_index, + doc_span_index, + tok_start_to_orig_index, + tok_end_to_orig_index, + token_is_max_context, + tokens, + input_ids, + input_mask, + segment_ids, + paragraph_len, + class_index=None, + paragraph_mask=None, + start_position=None, + end_position=None, + is_impossible=None): + self.unique_id = unique_id + self.example_index = example_index + self.doc_span_index = doc_span_index + self.tok_start_to_orig_index = tok_start_to_orig_index + self.tok_end_to_orig_index = tok_end_to_orig_index + self.token_is_max_context = token_is_max_context + self.tokens = tokens + self.input_ids = input_ids + self.input_mask = input_mask + self.paragraph_mask = paragraph_mask + self.segment_ids = segment_ids + self.paragraph_len = paragraph_len + self.class_index = class_index + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + + +def read_squad_examples(input_file, + is_training, + version_2_with_negative, + translated_input_folder=None): + """Read a SQuAD json file into a list of SquadExample.""" + del version_2_with_negative + with tf.io.gfile.GFile(input_file, "r") as reader: + input_data = json.load(reader)["data"] + + if translated_input_folder is not None: + translated_files = tf.io.gfile.glob( + os.path.join(translated_input_folder, "*.json")) + for file in translated_files: + with tf.io.gfile.GFile(file, "r") as reader: + input_data.extend(json.load(reader)["data"]) + + examples = [] + for entry in input_data: + for paragraph in entry["paragraphs"]: + paragraph_text = paragraph["context"] + + for qa in paragraph["qas"]: + qas_id = qa["id"] + question_text = qa["question"] + start_position = None + orig_answer_text = None + is_impossible = False + + if is_training: + is_impossible = qa.get("is_impossible", False) + if (len(qa["answers"]) != 1) and (not is_impossible): + raise ValueError( + "For training, each question should have exactly 1 answer.") + if not is_impossible: + answer = qa["answers"][0] + orig_answer_text = answer["text"] + start_position = answer["answer_start"] + else: + start_position = -1 + orig_answer_text = "" + + example = SquadExample( + qas_id=qas_id, + question_text=question_text, + paragraph_text=paragraph_text, + orig_answer_text=orig_answer_text, + start_position=start_position, + is_impossible=is_impossible) + examples.append(example) + + return examples + + +def _convert_index(index, pos, m=None, is_start=True): + """Converts index.""" + if index[pos] is not None: + return index[pos] + n = len(index) + rear = pos + while rear < n - 1 and index[rear] is None: + rear += 1 + front = pos + while front > 0 and index[front] is None: + front -= 1 + assert index[front] is not None or index[rear] is not None + if index[front] is None: + if index[rear] >= 1: # pytype: disable=unsupported-operands + if is_start: + return 0 + else: + return index[rear] - 1 + return index[rear] + if index[rear] is None: + if m is not None and index[front] < m - 1: + if is_start: + return index[front] + 1 + else: + return m - 1 + return index[front] + if is_start: + if index[rear] > index[front] + 1: + return index[front] + 1 + else: + return index[rear] + else: + if index[rear] > index[front] + 1: + return index[rear] - 1 + else: + return index[front] + + +def convert_examples_to_features(examples, + tokenizer, + max_seq_length, + doc_stride, + max_query_length, + is_training, + output_fn, + do_lower_case, + xlnet_format=False, + batch_size=None): + """Loads a data file into a list of `InputBatch`s.""" + cnt_pos, cnt_neg = 0, 0 + base_id = 1000000000 + unique_id = base_id + max_n, max_m = 1024, 1024 + f = np.zeros((max_n, max_m), dtype=np.float32) + + for (example_index, example) in enumerate(examples): + + if example_index % 100 == 0: + logging.info("Converting %d/%d pos %d neg %d", example_index, + len(examples), cnt_pos, cnt_neg) + + query_tokens = tokenization.encode_ids( + tokenizer.sp_model, + tokenization.preprocess_text( + example.question_text, lower=do_lower_case)) + + if len(query_tokens) > max_query_length: + query_tokens = query_tokens[0:max_query_length] + + paragraph_text = example.paragraph_text + para_tokens = tokenization.encode_pieces( + tokenizer.sp_model, + tokenization.preprocess_text( + example.paragraph_text, lower=do_lower_case)) + + chartok_to_tok_index = [] + tok_start_to_chartok_index = [] + tok_end_to_chartok_index = [] + char_cnt = 0 + for i, token in enumerate(para_tokens): + new_token = token.replace(tokenization.SPIECE_UNDERLINE, " ") + chartok_to_tok_index.extend([i] * len(new_token)) + tok_start_to_chartok_index.append(char_cnt) + char_cnt += len(new_token) + tok_end_to_chartok_index.append(char_cnt - 1) + + tok_cat_text = "".join(para_tokens).replace(tokenization.SPIECE_UNDERLINE, + " ") + n, m = len(paragraph_text), len(tok_cat_text) + + if n > max_n or m > max_m: + max_n = max(n, max_n) + max_m = max(m, max_m) + f = np.zeros((max_n, max_m), dtype=np.float32) + + g = {} + + # pylint: disable=cell-var-from-loop + def _lcs_match(max_dist, n=n, m=m): + """Longest-common-substring algorithm.""" + f.fill(0) + g.clear() + + ### longest common sub sequence + # f[i, j] = max(f[i - 1, j], f[i, j - 1], f[i - 1, j - 1] + match(i, j)) + for i in range(n): + + # unlike standard LCS, this is specifically optimized for the setting + # because the mismatch between sentence pieces and original text will + # be small + for j in range(i - max_dist, i + max_dist): + if j >= m or j < 0: + continue + + if i > 0: + g[(i, j)] = 0 + f[i, j] = f[i - 1, j] + + if j > 0 and f[i, j - 1] > f[i, j]: + g[(i, j)] = 1 + f[i, j] = f[i, j - 1] + + f_prev = f[i - 1, j - 1] if i > 0 and j > 0 else 0 + if (tokenization.preprocess_text( + paragraph_text[i], lower=do_lower_case, + remove_space=False) == tok_cat_text[j] and f_prev + 1 > f[i, j]): + g[(i, j)] = 2 + f[i, j] = f_prev + 1 + + # pylint: enable=cell-var-from-loop + + max_dist = abs(n - m) + 5 + for _ in range(2): + _lcs_match(max_dist) + if f[n - 1, m - 1] > 0.8 * n: + break + max_dist *= 2 + + orig_to_chartok_index = [None] * n + chartok_to_orig_index = [None] * m + i, j = n - 1, m - 1 + while i >= 0 and j >= 0: + if (i, j) not in g: + break + if g[(i, j)] == 2: + orig_to_chartok_index[i] = j + chartok_to_orig_index[j] = i + i, j = i - 1, j - 1 + elif g[(i, j)] == 1: + j = j - 1 + else: + i = i - 1 + + if (all(v is None for v in orig_to_chartok_index) or + f[n - 1, m - 1] < 0.8 * n): + logging.info("MISMATCH DETECTED!") + continue + + tok_start_to_orig_index = [] + tok_end_to_orig_index = [] + for i in range(len(para_tokens)): + start_chartok_pos = tok_start_to_chartok_index[i] + end_chartok_pos = tok_end_to_chartok_index[i] + start_orig_pos = _convert_index( + chartok_to_orig_index, start_chartok_pos, n, is_start=True) + end_orig_pos = _convert_index( + chartok_to_orig_index, end_chartok_pos, n, is_start=False) + + tok_start_to_orig_index.append(start_orig_pos) + tok_end_to_orig_index.append(end_orig_pos) + + if not is_training: + tok_start_position = tok_end_position = None + + if is_training and example.is_impossible: + tok_start_position = 0 + tok_end_position = 0 + + if is_training and not example.is_impossible: + start_position = example.start_position + end_position = start_position + len(example.orig_answer_text) - 1 + + start_chartok_pos = _convert_index( + orig_to_chartok_index, start_position, is_start=True) + tok_start_position = chartok_to_tok_index[start_chartok_pos] + + end_chartok_pos = _convert_index( + orig_to_chartok_index, end_position, is_start=False) + tok_end_position = chartok_to_tok_index[end_chartok_pos] + assert tok_start_position <= tok_end_position + + def _piece_to_id(x): + return tokenizer.sp_model.PieceToId(x) + + all_doc_tokens = list(map(_piece_to_id, para_tokens)) + + # The -3 accounts for [CLS], [SEP] and [SEP] + max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 + + # We can have documents that are longer than the maximum sequence length. + # To deal with this we do a sliding window approach, where we take chunks + # of the up to our max length with a stride of `doc_stride`. + _DocSpan = collections.namedtuple( # pylint: disable=invalid-name + "DocSpan", ["start", "length"]) + doc_spans = [] + start_offset = 0 + + while start_offset < len(all_doc_tokens): + length = len(all_doc_tokens) - start_offset + if length > max_tokens_for_doc: + length = max_tokens_for_doc + doc_spans.append(_DocSpan(start=start_offset, length=length)) + if start_offset + length == len(all_doc_tokens): + break + start_offset += min(length, doc_stride) + + for (doc_span_index, doc_span) in enumerate(doc_spans): + tokens = [] + token_is_max_context = {} + segment_ids = [] + + # Paragraph mask used in XLNet. + # 1 represents paragraph and class tokens. + # 0 represents query and other special tokens. + paragraph_mask = [] + + cur_tok_start_to_orig_index = [] + cur_tok_end_to_orig_index = [] + + # pylint: disable=cell-var-from-loop + def process_query(seg_q): + for token in query_tokens: + tokens.append(token) + segment_ids.append(seg_q) + paragraph_mask.append(0) + tokens.append(tokenizer.sp_model.PieceToId("[SEP]")) + segment_ids.append(seg_q) + paragraph_mask.append(0) + + def process_paragraph(seg_p): + for i in range(doc_span.length): + split_token_index = doc_span.start + i + + cur_tok_start_to_orig_index.append( + tok_start_to_orig_index[split_token_index]) + cur_tok_end_to_orig_index.append( + tok_end_to_orig_index[split_token_index]) + + is_max_context = _check_is_max_context(doc_spans, doc_span_index, + split_token_index) + token_is_max_context[len(tokens)] = is_max_context + tokens.append(all_doc_tokens[split_token_index]) + segment_ids.append(seg_p) + paragraph_mask.append(1) + tokens.append(tokenizer.sp_model.PieceToId("[SEP]")) + segment_ids.append(seg_p) + paragraph_mask.append(0) + return len(tokens) + + def process_class(seg_class): + class_index = len(segment_ids) + tokens.append(tokenizer.sp_model.PieceToId("[CLS]")) + segment_ids.append(seg_class) + paragraph_mask.append(1) + return class_index + + if xlnet_format: + seg_p, seg_q, seg_class, seg_pad = 0, 1, 2, 3 + paragraph_len = process_paragraph(seg_p) + process_query(seg_q) + class_index = process_class(seg_class) + else: + seg_p, seg_q, seg_class, seg_pad = 1, 0, 0, 0 + class_index = process_class(seg_class) + process_query(seg_q) + paragraph_len = process_paragraph(seg_p) + + input_ids = tokens + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < max_seq_length: + input_ids.append(0) + input_mask.append(0) + segment_ids.append(seg_pad) + paragraph_mask.append(0) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + assert len(paragraph_mask) == max_seq_length + + span_is_impossible = example.is_impossible + start_position = None + end_position = None + if is_training and not span_is_impossible: + # For training, if our document chunk does not contain an annotation + # we throw it out, since there is nothing to predict. + doc_start = doc_span.start + doc_end = doc_span.start + doc_span.length - 1 + out_of_span = False + if not (tok_start_position >= doc_start and + tok_end_position <= doc_end): + out_of_span = True + if out_of_span: + # continue + start_position = 0 + end_position = 0 + span_is_impossible = True + else: + doc_offset = 0 if xlnet_format else len(query_tokens) + 2 + start_position = tok_start_position - doc_start + doc_offset + end_position = tok_end_position - doc_start + doc_offset + + if is_training and span_is_impossible: + start_position = class_index + end_position = class_index + + if example_index < 20: + logging.info("*** Example ***") + logging.info("unique_id: %s", (unique_id)) + logging.info("example_index: %s", (example_index)) + logging.info("doc_span_index: %s", (doc_span_index)) + logging.info("tok_start_to_orig_index: %s", + " ".join([str(x) for x in cur_tok_start_to_orig_index])) + logging.info("tok_end_to_orig_index: %s", + " ".join([str(x) for x in cur_tok_end_to_orig_index])) + logging.info( + "token_is_max_context: %s", " ".join( + ["%d:%s" % (x, y) for (x, y) in token_is_max_context.items()])) + logging.info( + "input_pieces: %s", + " ".join([tokenizer.sp_model.IdToPiece(x) for x in tokens])) + logging.info("input_ids: %s", " ".join([str(x) for x in input_ids])) + logging.info("input_mask: %s", " ".join([str(x) for x in input_mask])) + logging.info("segment_ids: %s", " ".join([str(x) for x in segment_ids])) + logging.info("paragraph_mask: %s", " ".join( + [str(x) for x in paragraph_mask])) + logging.info("class_index: %d", class_index) + + if is_training and span_is_impossible: + logging.info("impossible example span") + + if is_training and not span_is_impossible: + pieces = [ + tokenizer.sp_model.IdToPiece(token) + for token in tokens[start_position:(end_position + 1)] + ] + answer_text = tokenizer.sp_model.DecodePieces(pieces) + logging.info("start_position: %d", (start_position)) + logging.info("end_position: %d", (end_position)) + logging.info("answer: %s", (tokenization.printable_text(answer_text))) + + # With multi processing, the example_index is actually the index + # within the current process therefore we use example_index=None + # to avoid being used in the future. + # The current code does not use example_index of training data. + if is_training: + feat_example_index = None + else: + feat_example_index = example_index + + feature = InputFeatures( + unique_id=unique_id, + example_index=feat_example_index, + doc_span_index=doc_span_index, + tok_start_to_orig_index=cur_tok_start_to_orig_index, + tok_end_to_orig_index=cur_tok_end_to_orig_index, + token_is_max_context=token_is_max_context, + tokens=[tokenizer.sp_model.IdToPiece(x) for x in tokens], + input_ids=input_ids, + input_mask=input_mask, + paragraph_mask=paragraph_mask, + segment_ids=segment_ids, + paragraph_len=paragraph_len, + class_index=class_index, + start_position=start_position, + end_position=end_position, + is_impossible=span_is_impossible) + + # Run callback + if is_training: + output_fn(feature) + else: + output_fn(feature, is_padding=False) + + unique_id += 1 + if span_is_impossible: + cnt_neg += 1 + else: + cnt_pos += 1 + + if not is_training and feature: + assert batch_size + num_padding = 0 + num_examples = unique_id - base_id + if unique_id % batch_size != 0: + num_padding = batch_size - (num_examples % batch_size) + dummy_feature = copy.deepcopy(feature) + for _ in range(num_padding): + dummy_feature.unique_id = unique_id + + # Run callback + output_fn(feature, is_padding=True) + unique_id += 1 + + logging.info("Total number of instances: %d = pos %d neg %d", + cnt_pos + cnt_neg, cnt_pos, cnt_neg) + return unique_id - base_id + + +def _check_is_max_context(doc_spans, cur_span_index, position): + """Check if this is the 'max context' doc span for the token.""" + + # Because of the sliding window approach taken to scoring documents, a single + # token can appear in multiple documents. E.g. + # Doc: the man went to the store and bought a gallon of milk + # Span A: the man went to the + # Span B: to the store and bought + # Span C: and bought a gallon of + # ... + # + # Now the word 'bought' will have two scores from spans B and C. We only + # want to consider the score with "maximum context", which we define as + # the *minimum* of its left and right context (the *sum* of left and + # right context will always be the same, of course). + # + # In the example the maximum context for 'bought' would be span C since + # it has 1 left context and 3 right context, while span B has 4 left context + # and 0 right context. + best_score = None + best_span_index = None + for (span_index, doc_span) in enumerate(doc_spans): + end = doc_span.start + doc_span.length - 1 + if position < doc_span.start: + continue + if position > end: + continue + num_left_context = position - doc_span.start + num_right_context = end - position + score = min(num_left_context, num_right_context) + 0.01 * doc_span.length + if best_score is None or score > best_score: + best_score = score + best_span_index = span_index + + return cur_span_index == best_span_index + + +def write_predictions(all_examples, + all_features, + all_results, + n_best_size, + max_answer_length, + do_lower_case, + output_prediction_file, + output_nbest_file, + output_null_log_odds_file, + version_2_with_negative=False, + null_score_diff_threshold=0.0, + verbose=False): + """Write final predictions to the json file and log-odds of null if needed.""" + logging.info("Writing predictions to: %s", (output_prediction_file)) + logging.info("Writing nbest to: %s", (output_nbest_file)) + + all_predictions, all_nbest_json, scores_diff_json = ( + postprocess_output( + all_examples=all_examples, + all_features=all_features, + all_results=all_results, + n_best_size=n_best_size, + max_answer_length=max_answer_length, + do_lower_case=do_lower_case, + version_2_with_negative=version_2_with_negative, + null_score_diff_threshold=null_score_diff_threshold, + verbose=verbose)) + + write_to_json_files(all_predictions, output_prediction_file) + write_to_json_files(all_nbest_json, output_nbest_file) + if version_2_with_negative: + write_to_json_files(scores_diff_json, output_null_log_odds_file) + + +def postprocess_output(all_examples, + all_features, + all_results, + n_best_size, + max_answer_length, + do_lower_case, + version_2_with_negative=False, + null_score_diff_threshold=0.0, + xlnet_format=False, + verbose=False): + """Postprocess model output, to form predicton results.""" + + del do_lower_case, verbose + example_index_to_features = collections.defaultdict(list) + for feature in all_features: + example_index_to_features[feature.example_index].append(feature) + + unique_id_to_result = {} + for result in all_results: + unique_id_to_result[result.unique_id] = result + + _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name + "PrelimPrediction", + ["feature_index", "start_index", "end_index", "start_logit", "end_logit"]) + + all_predictions = collections.OrderedDict() + all_nbest_json = collections.OrderedDict() + scores_diff_json = collections.OrderedDict() + + for (example_index, example) in enumerate(all_examples): + features = example_index_to_features[example_index] + + prelim_predictions = [] + # keep track of the minimum score of null start+end of position 0 + score_null = 1000000 # large and positive + min_null_feature_index = 0 # the paragraph slice with min mull score + null_start_logit = 0 # the start logit at the slice with min null score + null_end_logit = 0 # the end logit at the slice with min null score + for (feature_index, feature) in enumerate(features): + if feature.unique_id not in unique_id_to_result: + logging.info("Skip eval example %s, not in pred.", feature.unique_id) + continue + result = unique_id_to_result[feature.unique_id] + + # if we could have irrelevant answers, get the min score of irrelevant + if version_2_with_negative: + if xlnet_format: + feature_null_score = result.class_logits + else: + feature_null_score = result.start_logits[0] + result.end_logits[0] + if feature_null_score < score_null: + score_null = feature_null_score + min_null_feature_index = feature_index + null_start_logit = result.start_logits[0] + null_end_logit = result.end_logits[0] + + doc_offset = 0 if xlnet_format else feature.tokens.index("[SEP]") + 1 + + for (start_index, start_logit, + end_index, end_logit) in _get_best_indexes_and_logits( + result=result, + n_best_size=n_best_size, + xlnet_format=xlnet_format): + # We could hypothetically create invalid predictions, e.g., predict + # that the start of the span is in the question. We throw out all + # invalid predictions. + if start_index - doc_offset >= len(feature.tok_start_to_orig_index): + continue + if end_index - doc_offset >= len(feature.tok_end_to_orig_index): + continue + if not feature.token_is_max_context.get(start_index, False): + continue + if end_index < start_index: + continue + length = end_index - start_index + 1 + if length > max_answer_length: + continue + prelim_predictions.append( + _PrelimPrediction( + feature_index=feature_index, + start_index=start_index - doc_offset, + end_index=end_index - doc_offset, + start_logit=start_logit, + end_logit=end_logit)) + + if version_2_with_negative and not xlnet_format: + prelim_predictions.append( + _PrelimPrediction( + feature_index=min_null_feature_index, + start_index=-1, + end_index=-1, + start_logit=null_start_logit, + end_logit=null_end_logit)) + prelim_predictions = sorted( + prelim_predictions, + key=lambda x: (x.start_logit + x.end_logit), + reverse=True) + + _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name + "NbestPrediction", ["text", "start_logit", "end_logit"]) + + seen_predictions = {} + nbest = [] + for pred in prelim_predictions: + if len(nbest) >= n_best_size: + break + feature = features[pred.feature_index] + if pred.start_index >= 0 or xlnet_format: # this is a non-null prediction + tok_start_to_orig_index = feature.tok_start_to_orig_index + tok_end_to_orig_index = feature.tok_end_to_orig_index + start_orig_pos = tok_start_to_orig_index[pred.start_index] + end_orig_pos = tok_end_to_orig_index[pred.end_index] + + paragraph_text = example.paragraph_text + final_text = paragraph_text[start_orig_pos:end_orig_pos + 1].strip() + if final_text in seen_predictions: + continue + + seen_predictions[final_text] = True + else: + final_text = "" + seen_predictions[final_text] = True + + nbest.append( + _NbestPrediction( + text=final_text, + start_logit=pred.start_logit, + end_logit=pred.end_logit)) + + # if we didn't inlude the empty option in the n-best, include it + if version_2_with_negative and not xlnet_format: + if "" not in seen_predictions: + nbest.append( + _NbestPrediction( + text="", start_logit=null_start_logit, + end_logit=null_end_logit)) + # In very rare edge cases we could have no valid predictions. So we + # just create a nonce prediction in this case to avoid failure. + if not nbest: + nbest.append( + _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) + + assert len(nbest) >= 1 + + total_scores = [] + best_non_null_entry = None + for entry in nbest: + total_scores.append(entry.start_logit + entry.end_logit) + if not best_non_null_entry: + if entry.text: + best_non_null_entry = entry + + probs = _compute_softmax(total_scores) + + nbest_json = [] + for (i, entry) in enumerate(nbest): + output = collections.OrderedDict() + output["text"] = entry.text + output["probability"] = probs[i] + output["start_logit"] = entry.start_logit + output["end_logit"] = entry.end_logit + nbest_json.append(output) + + assert len(nbest_json) >= 1 + + if not version_2_with_negative: + all_predictions[example.qas_id] = nbest_json[0]["text"] + else: + assert best_non_null_entry is not None + if xlnet_format: + score_diff = score_null + scores_diff_json[example.qas_id] = score_diff + all_predictions[example.qas_id] = best_non_null_entry.text + else: + # predict "" iff the null score - the score of best non-null > threshold + score_diff = score_null - best_non_null_entry.start_logit - ( + best_non_null_entry.end_logit) + scores_diff_json[example.qas_id] = score_diff + if score_diff > null_score_diff_threshold: + all_predictions[example.qas_id] = "" + else: + all_predictions[example.qas_id] = best_non_null_entry.text + + all_nbest_json[example.qas_id] = nbest_json + + return all_predictions, all_nbest_json, scores_diff_json + + +def write_to_json_files(json_records, json_file): + with tf.io.gfile.GFile(json_file, "w") as writer: + writer.write(json.dumps(json_records, indent=4) + "\n") + + +def _get_best_indexes_and_logits(result, + n_best_size, + xlnet_format=False): + """Generates the n-best indexes and logits from a list.""" + if xlnet_format: + for i in range(n_best_size): + for j in range(n_best_size): + j_index = i * n_best_size + j + yield (result.start_indexes[i], result.start_logits[i], + result.end_indexes[j_index], result.end_logits[j_index]) + else: + start_index_and_score = sorted(enumerate(result.start_logits), + key=lambda x: x[1], reverse=True) + end_index_and_score = sorted(enumerate(result.end_logits), + key=lambda x: x[1], reverse=True) + for i in range(len(start_index_and_score)): + if i >= n_best_size: + break + for j in range(len(end_index_and_score)): + if j >= n_best_size: + break + yield (start_index_and_score[i][0], start_index_and_score[i][1], + end_index_and_score[j][0], end_index_and_score[j][1]) + + +def _compute_softmax(scores): + """Compute softmax probability over raw logits.""" + if not scores: + return [] + + max_score = None + for score in scores: + if max_score is None or score > max_score: + max_score = score + + exp_scores = [] + total_sum = 0.0 + for score in scores: + x = math.exp(score - max_score) + exp_scores.append(x) + total_sum += x + + probs = [] + for score in exp_scores: + probs.append(score / total_sum) + return probs + + +class FeatureWriter(object): + """Writes InputFeature to TF example file.""" + + def __init__(self, filename, is_training): + self.filename = filename + self.is_training = is_training + self.num_features = 0 + tf.io.gfile.makedirs(os.path.dirname(filename)) + self._writer = tf.io.TFRecordWriter(filename) + + def process_feature(self, feature): + """Write a InputFeature to the TFRecordWriter as a tf.train.Example.""" + self.num_features += 1 + + def create_int_feature(values): + feature = tf.train.Feature( + int64_list=tf.train.Int64List(value=list(values))) + return feature + + features = collections.OrderedDict() + features["unique_ids"] = create_int_feature([feature.unique_id]) + features["input_ids"] = create_int_feature(feature.input_ids) + features["input_mask"] = create_int_feature(feature.input_mask) + features["segment_ids"] = create_int_feature(feature.segment_ids) + if feature.paragraph_mask is not None: + features["paragraph_mask"] = create_int_feature(feature.paragraph_mask) + if feature.class_index is not None: + features["class_index"] = create_int_feature([feature.class_index]) + + if self.is_training: + features["start_positions"] = create_int_feature([feature.start_position]) + features["end_positions"] = create_int_feature([feature.end_position]) + impossible = 0 + if feature.is_impossible: + impossible = 1 + features["is_impossible"] = create_int_feature([impossible]) + + tf_example = tf.train.Example(features=tf.train.Features(feature=features)) + self._writer.write(tf_example.SerializeToString()) + + def close(self): + self._writer.close() + + +def generate_tf_record_from_json_file(input_file_path, + sp_model_file, + output_path, + translated_input_folder=None, + max_seq_length=384, + do_lower_case=True, + max_query_length=64, + doc_stride=128, + xlnet_format=False, + version_2_with_negative=False): + """Generates and saves training data into a tf record file.""" + train_examples = read_squad_examples( + input_file=input_file_path, + is_training=True, + version_2_with_negative=version_2_with_negative, + translated_input_folder=translated_input_folder) + tokenizer = tokenization.FullSentencePieceTokenizer( + sp_model_file=sp_model_file) + train_writer = FeatureWriter( + filename=output_path, is_training=True) + number_of_examples = convert_examples_to_features( + examples=train_examples, + tokenizer=tokenizer, + max_seq_length=max_seq_length, + doc_stride=doc_stride, + max_query_length=max_query_length, + is_training=True, + output_fn=train_writer.process_feature, + xlnet_format=xlnet_format, + do_lower_case=do_lower_case) + train_writer.close() + + meta_data = { + "task_type": "bert_squad", + "train_data_size": number_of_examples, + "max_seq_length": max_seq_length, + "max_query_length": max_query_length, + "doc_stride": doc_stride, + "version_2_with_negative": version_2_with_negative, + } + + return meta_data diff --git a/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/tagging_data_lib.py b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/tagging_data_lib.py new file mode 100644 index 0000000000000000000000000000000000000000..f6b9c19744be9b6b3730e65dd24c1e19730f8e47 --- /dev/null +++ b/TensorFlow2x/ComputeVision/Classification/models-master/official/nlp/data/tagging_data_lib.py @@ -0,0 +1,426 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Library to process data for tagging task such as NER/POS.""" +import collections +import os + +from absl import logging +import tensorflow as tf + +from official.nlp.bert import tokenization +from official.nlp.data import classifier_data_lib + +# A negative label id for the padding label, which will not contribute +# to loss/metrics in training. +_PADDING_LABEL_ID = -1 + +# The special unknown token, used to substitute a word which has too many +# subwords after tokenization. +_UNK_TOKEN = "[UNK]" + + +class InputExample(object): + """A single training/test example for token classification.""" + + def __init__(self, + sentence_id, + sub_sentence_id=0, + words=None, + label_ids=None): + """Constructs an InputExample.""" + self.sentence_id = sentence_id + self.sub_sentence_id = sub_sentence_id + self.words = words if words else [] + self.label_ids = label_ids if label_ids else [] + + def add_word_and_label_id(self, word, label_id): + """Adds word and label_id pair in the example.""" + self.words.append(word) + self.label_ids.append(label_id) + + +def _read_one_file(file_name, label_list): + """Reads one file and returns a list of `InputExample` instances.""" + lines = tf.io.gfile.GFile(file_name, "r").readlines() + examples = [] + label_id_map = {label: i for i, label in enumerate(label_list)} + sentence_id = 0 + example = InputExample(sentence_id=0) + for line in lines: + line = line.strip("\n") + if line: + # The format is: \t