diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..418a32550ea167dcbeae65db8a5cb58b2dd5e1ab --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1 @@ +## Contributing to LiBai \ No newline at end of file diff --git a/LICENSE b/LICENSE index c7ab89cb12f4e6bb2e18eb8f072ea006d171c7b9..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 100644 --- a/LICENSE +++ b/LICENSE @@ -1,25 +1,201 @@ -BSD 2-Clause License - -Copyright (c) 2023, yuguo960516yuguo -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/changelog.md b/changelog.md new file mode 100644 index 0000000000000000000000000000000000000000..7eba8fc0d2e13361401bf1158f7fc93db8045af9 --- /dev/null +++ b/changelog.md @@ -0,0 +1,26 @@ +## Changelog + +### Beta 0.1.0 (22/03/2022) + +**New Features:** +- Support Data Parallelism +- Support 1D Tensor Parallelism +- Support Pipeline Parallelism +- Unified distributed Layers for both single-GPU and multi-GPU training +- `LazyConfig` system for more flexible syntax and no predefined structures +- Easy-to-use trainer and engine +- Support both CV and NLP data processing +- Mixed Precision Training +- Activation Checkpointing +- Gradient Accumulation +- Gradient Clipping +- Zero Redundancy Optimizer (ZeRO) + +**Supported Models:** +- Support 3D parallel [BERT](https://arxiv.org/abs/1810.04805) model +- Support 3D parallel [GPT-2](https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf) model +- Support 3D parallel [T5](https://arxiv.org/abs/1910.10683) model +- Support 3D parallel [Vision Transformer](https://arxiv.org/abs/2010.11929) +- Support Data parallel [Swin Transformer](https://arxiv.org/abs/2103.14030) model +- Support finetune task in [QQP project](/projects/QQP/) +- Support text classification task in [text classification project](/projects/text_classification/) diff --git a/configs/bert_classification.py b/configs/bert_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..709638e69c6dcef6db45980d095332c2cd04a88d --- /dev/null +++ b/configs/bert_classification.py @@ -0,0 +1,31 @@ +from libai.config import LazyCall +from libai.models.bert_model import BertForClassification +from .common.models.bert import cfg as bert_cfg +from .common.models.graph import graph +from .common.train import train +from .common.optim import optim +from .common.data.bert_dataset import tokenization, dataloader + +vocab_file = "./data_test/bert_data/bert-base-chinese-vocab.txt" +data_prefix = "./data_test/bert_data/loss_compara_content_sentence" + +dataloader.train.dataset[0].data_prefix = data_prefix +dataloader.train.dataset[0].indexed_dataset.data_prefix = data_prefix + +bert_cfg.num_labels = 2 +bert_cfg.classifier_dropout = 0.1 + +model = LazyCall(BertForClassification)(cfg=bert_cfg) +tokenization.tokenizer.vocab_file = vocab_file + +model.cfg.vocab_size = 21128 +model.cfg.intermediate_size = 3072 +model.cfg.num_attention_heads = 12 +model.cfg.hidden_layers = 12 +model.cfg.hidden_size = 768 + +train.amp.enabled = True +train.activation_checkpoint.enabled = True +train.dist.pipeline_num_layers = model.cfg.hidden_layers + +train.output_dir = "output/bert_classification_output" diff --git a/configs/bert_large_pretrain.py b/configs/bert_large_pretrain.py new file mode 100644 index 0000000000000000000000000000000000000000..eabf387fc4be550049be88905601b3de18a0ec0f --- /dev/null +++ b/configs/bert_large_pretrain.py @@ -0,0 +1,35 @@ +from libai.config import LazyCall +from libai.evaluation import PPLEvaluator +from .common.models.bert import pretrain_model as model +from .common.models.graph import graph +from .common.train import train +from .common.optim import optim +from .common.data.bert_dataset import dataloader, tokenization + +vocab_file = "./nlp_data/bert-base-chinese-vocab.txt" +data_prefix = "./nlp_data/data/loss_compara_content_sentence" + +tokenization.tokenizer.vocab_file = vocab_file +dataloader.train.dataset[0].data_prefix = data_prefix +dataloader.train.dataset[0].indexed_dataset.data_prefix = data_prefix + +# Bert-large model config +model.cfg.num_attention_heads = 16 +model.cfg.hidden_size = 768 +model.cfg.hidden_layers = 8 + +train.input_placement_device = "cpu" + +train.dist.pipeline_num_layers = model.cfg.hidden_layers + +train.train_micro_batch_size = 16 + +train.amp.enabled = True + +for ds in dataloader.train.dataset: + ds.max_seq_length = model.cfg.max_position_embeddings + +train.evaluation.evaluator = LazyCall(PPLEvaluator)() + +train.output_dir = "output/bert_output" + diff --git a/configs/common/data/bert_dataset.py b/configs/common/data/bert_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..d6f78b83c71a688ef17e5fb3939e63cfb5569bd0 --- /dev/null +++ b/configs/common/data/bert_dataset.py @@ -0,0 +1,44 @@ +from libai.config import LazyCall +from omegaconf import OmegaConf +from libai.data import build_nlp_test_loader, build_nlp_train_val_test_loader +from libai.data.datasets import BertDataset +from libai.data.data_utils import get_indexed_dataset + +from libai.tokenizer import BertTokenizer + + +tokenization = OmegaConf.create() + +tokenization.tokenizer = LazyCall(BertTokenizer)( + vocab_file="bert-base-chinese-vocab.txt", + do_lower_case=True, + do_chinese_wwm=True, +) +tokenization.append_eod = False +tokenization.make_vocab_size_divisible_by = 128 + +dataloader = OmegaConf.create() + +dataloader.train = LazyCall(build_nlp_train_val_test_loader)( + dataset=[ + LazyCall(BertDataset)( + name="bert", + data_prefix="/workspace/data/libai_dataset/loss_compara_content_sentence", + indexed_dataset=LazyCall(get_indexed_dataset)( + data_prefix="/workspace/data/libai_dataset/loss_compara_content_sentence", + data_impl="mmap", + skip_warmup=False, + ), + max_seq_length=512, + mask_lm_prob=0.15, + short_seq_prob=0.1, + binary_head=True, + seed=1234, + masking_style="bert-cn-wwm", + ), + ], + train_val_test_num_samples=None, # a hint for deferred assignment + splits=[[949.0, 50.0, 1.0]], + weights=[1.0], + num_workers=4, +) diff --git a/configs/common/data/cifar100.py b/configs/common/data/cifar100.py new file mode 100644 index 0000000000000000000000000000000000000000..0f7d85f1bf7281a20d9ba08b6f26da0203756278 --- /dev/null +++ b/configs/common/data/cifar100.py @@ -0,0 +1,79 @@ +from omegaconf import OmegaConf +from flowvision import transforms +from flowvision.data.mixup import Mixup +from flowvision.transforms import InterpolationMode +from flowvision.transforms.functional import str_to_interp_mode + +from libai.data.datasets import CIFAR100Dataset +from libai.data.build import build_image_train_loader, build_image_test_loader +from libai.config import LazyCall + +# mean and std of cifar100 dataset +CIFAR100_TRAIN_MEAN = (0.5070751592371323, 0.48654887331495095, 0.4409178433670343) +CIFAR100_TRAIN_STD = (0.2673342858792401, 0.2564384629170883, 0.27615047132568404) + +train_aug = LazyCall(transforms.Compose)( + transforms=[ + LazyCall(transforms.RandomResizedCrop)( + size=(224, 224), + scale=(0.08, 1.0), + ratio=(3.0 / 4.0, 4.0 / 3.0), + interpolation=str_to_interp_mode("bicubic"), + ), + LazyCall(transforms.RandomHorizontalFlip)(), + LazyCall(transforms.ToTensor)(), + LazyCall(transforms.Normalize)(mean=CIFAR100_TRAIN_MEAN, std=CIFAR100_TRAIN_STD), + ] +) + +test_aug = LazyCall(transforms.Compose)( + transforms=[ + LazyCall(transforms.Resize)( + size=256, + interpolation=InterpolationMode.BICUBIC, + ), + LazyCall(transforms.CenterCrop)( + size=224, + ), + LazyCall(transforms.ToTensor)(), + LazyCall(transforms.Normalize)( + mean=CIFAR100_TRAIN_MEAN, + std=CIFAR100_TRAIN_STD, + ), + ] +) + + +# Dataloader config +dataloader = OmegaConf.create() +dataloader.train = LazyCall(build_image_train_loader)( + dataset=[ + LazyCall(CIFAR100Dataset)( + root="./", + train=True, + download=True, + transform=train_aug, + ), + ], + num_workers=4, + mixup_func=LazyCall(Mixup)( + mixup_alpha=0.8, + cutmix_alpha=1.0, + prob=1.0, + switch_prob=0.5, + mode="batch", + num_classes=100, + ), +) + +dataloader.test = [ + LazyCall(build_image_test_loader)( + dataset=LazyCall(CIFAR100Dataset)( + root="./", + train=False, + download=True, + transform=test_aug, + ), + num_workers=4, + ) +] diff --git a/configs/common/data/gpt_dataset.py b/configs/common/data/gpt_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..1d13e9fbd9a0bdd647e058dc74b105a36cdee1cf --- /dev/null +++ b/configs/common/data/gpt_dataset.py @@ -0,0 +1,41 @@ +from libai.config import LazyCall +from omegaconf import OmegaConf +from libai.data import build_nlp_test_loader, build_nlp_train_val_test_loader +from libai.data.datasets import GPT2Dataset +from libai.data.data_utils import get_indexed_dataset + +from libai.tokenizer import GPT2Tokenizer + + +tokenization = OmegaConf.create() + +tokenization.tokenizer = LazyCall(GPT2Tokenizer)( + vocab_file="/workspace/data/gpt_dataset/gpt2-vocab.json", + merges_file="/workspace/data/gpt_dataset/gpt2-merges.txt", + do_lower_case=True, + do_chinese_wwm=True, +) +tokenization.append_eod = False +tokenization.make_vocab_size_divisible_by = 128 + +dataloader = OmegaConf.create() + +dataloader.train = LazyCall(build_nlp_train_val_test_loader)( + dataset=[ + LazyCall(GPT2Dataset)( + name="gpt-2", + data_prefix="/workspace/data/libai_dataset/loss_compara_content_sentence", + indexed_dataset=LazyCall(get_indexed_dataset)( + data_prefix="/workspace/data/libai_dataset/loss_compara_content_sentence", + data_impl="mmap", + skip_warmup=False, + ), + max_seq_length=1024, + seed=1234, + ), + ], + train_val_test_num_samples=None, # a hint for deferred assignment + splits=[[949.0, 50.0, 1.0]], + weights=[1.0], + num_workers=4, +) diff --git a/configs/common/data/imagenet.py b/configs/common/data/imagenet.py new file mode 100644 index 0000000000000000000000000000000000000000..67b8cbe344b8fe0c08c6f43034f3ca9b6782fdbe --- /dev/null +++ b/configs/common/data/imagenet.py @@ -0,0 +1,90 @@ +from omegaconf import OmegaConf +from flowvision import transforms +from flowvision.transforms import InterpolationMode +from flowvision.transforms.functional import str_to_interp_mode +from flowvision.data.constants import ( + IMAGENET_DEFAULT_MEAN, + IMAGENET_DEFAULT_STD, +) +from flowvision.data.auto_augment import rand_augment_transform +from flowvision.data.random_erasing import RandomErasing + +from libai.config import LazyCall +from libai.data.datasets import ImageNetDataset +from libai.data.build import build_image_train_loader, build_image_test_loader + +train_aug = LazyCall(transforms.Compose)( + transforms=[ + LazyCall(transforms.RandomResizedCrop)( + size=224, + scale=(0.08, 1.0), + ratio=(3.0 / 4.0, 4.0 / 3.0), + interpolation=InterpolationMode.BICUBIC, + ), + LazyCall(transforms.RandomHorizontalFlip)(p=0.5), + LazyCall(rand_augment_transform)( + config_str="rand-m9-mstd0.5-inc1", + hparams=dict( + translate_const=int(224 * 0.45), + img_mean=tuple([min(255, round(255 * x)) for x in IMAGENET_DEFAULT_MEAN]), + interpolation=str_to_interp_mode("bicubic"), + ), + ), + LazyCall(transforms.ToTensor)(), + LazyCall(transforms.Normalize)( + mean=IMAGENET_DEFAULT_MEAN, + std=IMAGENET_DEFAULT_STD, + ), + LazyCall(RandomErasing)( + probability=0.25, + mode="pixel", + max_count=1, + num_splits=0, + device="cpu", + ), + ] +) + + +test_aug = LazyCall(transforms.Compose)( + transforms=[ + LazyCall(transforms.Resize)( + size=256, + interpolation=InterpolationMode.BICUBIC, + ), + LazyCall(transforms.CenterCrop)( + size=224, + ), + LazyCall(transforms.ToTensor)(), + LazyCall(transforms.Normalize)( + mean=IMAGENET_DEFAULT_MEAN, + std=IMAGENET_DEFAULT_STD, + ), + ] +) + + +dataloader = OmegaConf.create() +dataloader.train = LazyCall(build_image_train_loader)( + dataset=[ + LazyCall(ImageNetDataset)( + root="./dataset", + train=True, + transform=train_aug, + ), + ], + num_workers=4, + mixup_func=None, +) + + +dataloader.test = [ + LazyCall(build_image_test_loader)( + dataset=LazyCall(ImageNetDataset)( + root="./dataset", + train=False, + transform=test_aug, + ), + num_workers=4, + ) +] diff --git a/configs/common/data/roberta_dataset.py b/configs/common/data/roberta_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..23623a448920ad6c1f17b564d6eb8eb598c7f45a --- /dev/null +++ b/configs/common/data/roberta_dataset.py @@ -0,0 +1,41 @@ +from libai.config import LazyCall +from omegaconf import OmegaConf +from libai.data import build_nlp_test_loader, build_nlp_train_val_test_loader +from libai.data.datasets import RobertaDataset +from libai.data.data_utils import get_indexed_dataset + +from libai.tokenizer import RobertaTokenizer + + +tokenization = OmegaConf.create() + +tokenization.tokenizer = LazyCall(RobertaTokenizer)( + vocab_file="roberta-vocab.json", merges_file="roberta-merges.txt" +) +tokenization.append_eod = False +tokenization.make_vocab_size_divisible_by = 128 + +dataloader = OmegaConf.create() + +dataloader.train = LazyCall(build_nlp_train_val_test_loader)( + dataset=[ + LazyCall(RobertaDataset)( + name="roberta", + data_prefix="/workspace/data/libai_dataset/loss_compara_content_sentence", + indexed_dataset=LazyCall(get_indexed_dataset)( + data_prefix="/workspace/data/libai_dataset/loss_compara_content_sentence", + data_impl="mmap", + skip_warmup=False, + ), + max_seq_length=512, + mask_lm_prob=0.15, + short_seq_prob=0.0, + seed=1234, + masking_style="bert", + ), + ], + train_val_test_num_samples=None, # a hint for deferred assignment + splits=[[949.0, 50.0, 1.0]], + weights=[1.0], + num_workers=4, +) diff --git a/configs/common/data/t5_dataset.py b/configs/common/data/t5_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..d8d04fb5afb908f5d823998c049cf1d3636f498e --- /dev/null +++ b/configs/common/data/t5_dataset.py @@ -0,0 +1,51 @@ +from libai.config import LazyCall +from omegaconf import OmegaConf +from libai.data import build_nlp_test_loader, build_nlp_train_val_test_loader +from libai.data.datasets import T5Dataset +from libai.data.data_utils import get_indexed_dataset + +from libai.tokenizer import BertTokenizer + + +tokenization = OmegaConf.create() + +tokenization.setup = True + +special_tokens = [] +for i in range(100): + special_tokens.append(f"") +tokenization.tokenizer = LazyCall(BertTokenizer)( + vocab_file="/workspace/data/libai_dataset/bert-base-chinese-vocab.txt", + do_lower_case=True, + do_chinese_wwm=True, + bos_token="[BOS]", + eos_token="[EOS]", + additional_special_tokens=special_tokens, +) +tokenization.append_eod = False +tokenization.make_vocab_size_divisible_by = 128 + +dataloader = OmegaConf.create() + +dataloader.train = LazyCall(build_nlp_train_val_test_loader)( + dataset=[ + LazyCall(T5Dataset)( + name="t5", + data_prefix="/workspace/data/libai_dataset/loss_compara_content_sentence", + indexed_dataset=LazyCall(get_indexed_dataset)( + data_prefix="/workspace/data/libai_dataset/" "/loss_compara_content_sentence", + data_impl="mmap", + skip_warmup=False, + ), + max_seq_length=512, + max_seq_length_dec=128, + masked_lm_prob=0.15, + short_seq_prob=0.1, + seed=1234, + ), + ], + train_val_test_num_samples=None, # a hint for deferred assignment + splits=[[949.0, 50.0, 1.0]], + weights=[1.0], + num_workers=4, +) diff --git a/configs/common/models/bert.py b/configs/common/models/bert.py new file mode 100644 index 0000000000000000000000000000000000000000..5986276b79b182c994c1057fa66e11ea00874073 --- /dev/null +++ b/configs/common/models/bert.py @@ -0,0 +1,32 @@ +from omegaconf import DictConfig +from libai.config import LazyCall +from libai.models import BertModel, BertForPreTraining + + +cfg = dict( + vocab_size=30522, + hidden_size=768, + hidden_layers=24, + num_attention_heads=12, + intermediate_size=4096, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + num_tokentypes=2, + add_pooling_layer=True, + initializer_range=0.02, + layernorm_eps=1e-5, + bias_gelu_fusion=True, + bias_dropout_fusion=True, + scale_mask_softmax_fusion=True, + apply_query_key_layer_scaling=True, + apply_residual_post_layernorm=False, + add_binary_head=True, + amp_enabled=False, +) + +cfg = DictConfig(cfg) + +bert_model = LazyCall(BertModel)(cfg=cfg) + +pretrain_model = LazyCall(BertForPreTraining)(cfg=cfg) diff --git a/configs/common/models/gpt.py b/configs/common/models/gpt.py new file mode 100644 index 0000000000000000000000000000000000000000..1e6757dac888272c3778b5cfbba21c9f174eac9d --- /dev/null +++ b/configs/common/models/gpt.py @@ -0,0 +1,31 @@ +from omegaconf import DictConfig +from libai.config import LazyCall +from libai.models import GPTModel, GPTForPreTraining + + +cfg = dict( + hidden_layers=6, + vocab_size=30522, + hidden_size=384, + ffn_hidden_size=1536, + num_attention_heads=12, + max_seq_length=1024, + embedding_dropout_prob=0, + attention_dropout_prob=0, + output_dropout_prob=0, + layernorm_epsilon=1e-5, + initializer_range=0.02, + use_scaled_init_for_output_weights=True, + bias_gelu_fusion=True, + bias_dropout_fusion=True, + scale_mask_softmax_fusion=True, + apply_query_key_layer_scaling=True, + apply_residual_post_layernorm=False, + amp_enabled=False, +) + +cfg = DictConfig(cfg) + +gpt_model = LazyCall(GPTModel)(cfg=cfg) + +pretrain_model = LazyCall(GPTForPreTraining)(cfg=cfg) diff --git a/configs/common/models/graph.py b/configs/common/models/graph.py new file mode 100644 index 0000000000000000000000000000000000000000..8be1bca9a0d5484c3f41d0f82a2f604dd5c045fc --- /dev/null +++ b/configs/common/models/graph.py @@ -0,0 +1,22 @@ +from omegaconf import DictConfig +from libai.config import LazyCall +from libai.models.utils import GraphBase + + +graph = dict( + # options for graph or eager mode + enabled=True, + debug=-1, # debug mode for graph + auto_parallel=dict( + enabled=False, + enable_auto_parallel_ignore_user_sbp_config=False, # ignore all .to_global() in graph + trunk_algo=True, # consider overlapping calculate time and transfer time + sbp_collector=False, # use proxy node when one node transfer to many nodes + ), + train_graph=LazyCall(GraphBase)( + is_train=True, + ), + eval_graph=LazyCall(GraphBase)(is_train=False), +) + +graph = DictConfig(graph) diff --git a/configs/common/models/resmlp/resmlpB_24.py b/configs/common/models/resmlp/resmlpB_24.py new file mode 100644 index 0000000000000000000000000000000000000000..5dc96fe1367da7bd8fb70e415ee6c24fe9418e22 --- /dev/null +++ b/configs/common/models/resmlp/resmlpB_24.py @@ -0,0 +1,12 @@ +from libai.config import LazyCall +from libai.models import ResMLP + +from .resmlp_12 import cfg + + +cfg.patch_size = 8 +cfg.embed_dim = 768 +cfg.depth = 24 +cfg.init_scale = 1e-6 + +model = LazyCall(ResMLP)(cfg=cfg) diff --git a/configs/common/models/resmlp/resmlp_12.py b/configs/common/models/resmlp/resmlp_12.py new file mode 100644 index 0000000000000000000000000000000000000000..e0328ce2d9f930022cadaf56347d2cc4494b24a7 --- /dev/null +++ b/configs/common/models/resmlp/resmlp_12.py @@ -0,0 +1,21 @@ +from omegaconf import DictConfig +from libai.config import LazyCall +from libai.models import ResMLP + + +cfg = dict( + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=384, + depth=12, + drop_rate=0.0, + drop_path_rate=0.05, + init_scale=0.1, + num_classes=1000, + loss_func=None, +) + +cfg = DictConfig(cfg) + +model = LazyCall(ResMLP)(cfg=cfg) diff --git a/configs/common/models/resmlp/resmlp_24.py b/configs/common/models/resmlp/resmlp_24.py new file mode 100644 index 0000000000000000000000000000000000000000..7a9b76a239fd9c0326551e35ca6a502a8e8e0d5a --- /dev/null +++ b/configs/common/models/resmlp/resmlp_24.py @@ -0,0 +1,10 @@ +from libai.config import LazyCall +from libai.models import ResMLP + +from .resmlp_12 import cfg + + +cfg.depth = 24 +cfg.init_scale = 1e-5 + +model = LazyCall(ResMLP)(cfg=cfg) diff --git a/configs/common/models/resmlp/resmlp_36.py b/configs/common/models/resmlp/resmlp_36.py new file mode 100644 index 0000000000000000000000000000000000000000..c216104fa6ab83eb56d7f7ad75a750e825c7aa7f --- /dev/null +++ b/configs/common/models/resmlp/resmlp_36.py @@ -0,0 +1,10 @@ +from libai.config import LazyCall +from libai.models import ResMLP + +from .resmlp_12 import cfg + + +cfg.depth = 36 +cfg.init_scale = 1e-6 + +model = LazyCall(ResMLP)(cfg=cfg) diff --git a/configs/common/models/roberta.py b/configs/common/models/roberta.py new file mode 100644 index 0000000000000000000000000000000000000000..9b4c18b048c7b8fe7bb7366915869a8f7b8076ab --- /dev/null +++ b/configs/common/models/roberta.py @@ -0,0 +1,34 @@ +from omegaconf import DictConfig +from libai.config import LazyCall +from libai.models import RobertaModel, RobertaForPreTraining, RobertaForCausalLM + + +cfg = dict( + vocab_size=50265, + hidden_size=768, + hidden_layers=12, + num_attention_heads=12, + intermediate_size=3072, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=514, + num_tokentypes=1, + add_pooling_layer=True, + initializer_range=0.02, + layernorm_eps=1e-5, + pad_token_id=1, + bias_gelu_fusion=True, + bias_dropout_fusion=True, + scale_mask_softmax_fusion=True, + apply_query_key_layer_scaling=True, + apply_residual_post_layernorm=False, + amp_enabled=False, +) + +cfg = DictConfig(cfg) + +roberta_model = LazyCall(RobertaModel)(cfg=cfg) + +roberta_causal_lm = LazyCall(RobertaForCausalLM)(cfg=cfg) + +pretrain_model = LazyCall(RobertaForPreTraining)(cfg=cfg) diff --git a/configs/common/models/swin/swin_base_patch4_window12_384.py b/configs/common/models/swin/swin_base_patch4_window12_384.py new file mode 100644 index 0000000000000000000000000000000000000000..14871173547b3ee9437a7cc4e495508b9c2e01ce --- /dev/null +++ b/configs/common/models/swin/swin_base_patch4_window12_384.py @@ -0,0 +1,13 @@ +from libai.config import LazyCall +from libai.models import SwinTransformer + +from .swin_tiny_patch4_window7_224 import cfg + + +cfg.img_size = 384 +cfg.embed_dim = 128 +cfg.depths = [2, 2, 18, 2] +cfg.num_heads = [4, 8, 16, 32] +cfg.drop_path_rate = 0.1 + +model = LazyCall(SwinTransformer)(cfg=cfg) diff --git a/configs/common/models/swin/swin_base_patch4_window7_224.py b/configs/common/models/swin/swin_base_patch4_window7_224.py new file mode 100644 index 0000000000000000000000000000000000000000..28ffa61018125fbdb0fb10f99282339083086c95 --- /dev/null +++ b/configs/common/models/swin/swin_base_patch4_window7_224.py @@ -0,0 +1,12 @@ +from libai.config import LazyCall +from libai.models import SwinTransformer + +from .swin_tiny_patch4_window7_224 import cfg + + +cfg.embed_dim = 128 +cfg.depths = [2, 2, 18, 2] +cfg.num_heads = [4, 8, 16, 32] +cfg.drop_path_rate = 0.5 + +model = LazyCall(SwinTransformer)(cfg=cfg) diff --git a/configs/common/models/swin/swin_large_patch4_window12_384.py b/configs/common/models/swin/swin_large_patch4_window12_384.py new file mode 100644 index 0000000000000000000000000000000000000000..900a065c55fa71db0c79feb3f2be50ea3c42ea75 --- /dev/null +++ b/configs/common/models/swin/swin_large_patch4_window12_384.py @@ -0,0 +1,14 @@ +from libai.config import LazyCall +from libai.models import SwinTransformer + +from .swin_tiny_patch4_window7_224 import cfg + + +cfg.img_size = 384 +cfg.embed_dim = 192 +cfg.depths = [2, 2, 18, 2] +cfg.num_heads = [6, 12, 24, 48] +cfg.window_size = 12 +cfg.drop_path_rate = 0.1 + +model = LazyCall(SwinTransformer)(cfg=cfg) diff --git a/configs/common/models/swin/swin_large_patch4_window7_224.py b/configs/common/models/swin/swin_large_patch4_window7_224.py new file mode 100644 index 0000000000000000000000000000000000000000..2f9de1f046ad6a01312d81c044c44da27c66a5a0 --- /dev/null +++ b/configs/common/models/swin/swin_large_patch4_window7_224.py @@ -0,0 +1,12 @@ +from libai.config import LazyCall +from libai.models import SwinTransformer + +from .swin_tiny_patch4_window7_224 import cfg + + +cfg.embed_dim = 192 +cfg.depths = [2, 2, 18, 2] +cfg.num_heads = [6, 12, 24, 48] +cfg.drop_path_rate = 0.1 + +model = LazyCall(SwinTransformer)(cfg=cfg) diff --git a/configs/common/models/swin/swin_small_patch4_window7_224.py b/configs/common/models/swin/swin_small_patch4_window7_224.py new file mode 100644 index 0000000000000000000000000000000000000000..af74e0bff642493172850d413c14f1121434a55c --- /dev/null +++ b/configs/common/models/swin/swin_small_patch4_window7_224.py @@ -0,0 +1,9 @@ +from libai.config import LazyCall +from libai.models import SwinTransformer +from .swin_tiny_patch4_window7_224 import cfg + + +cfg.depths = [2, 2, 18, 2] +cfg.drop_path_rate = 0.3 + +model = LazyCall(SwinTransformer)(cfg=cfg) diff --git a/configs/common/models/swin/swin_tiny_c24_patch4_window8_256.py b/configs/common/models/swin/swin_tiny_c24_patch4_window8_256.py new file mode 100644 index 0000000000000000000000000000000000000000..37f6ff828743a693433868a4251713f51c7e180e --- /dev/null +++ b/configs/common/models/swin/swin_tiny_c24_patch4_window8_256.py @@ -0,0 +1,10 @@ +from libai.config import LazyCall +from libai.models import SwinTransformer +from .swin_tiny_patch4_window7_224 import cfg + + +cfg.img_size = 256 +cfg.num_heads = [4, 8, 16, 32] +cfg.window_size = 8 + +model = LazyCall(SwinTransformer)(cfg=cfg) diff --git a/configs/common/models/swin/swin_tiny_patch4_window7_224.py b/configs/common/models/swin/swin_tiny_patch4_window7_224.py new file mode 100644 index 0000000000000000000000000000000000000000..e51511dc32c07f9f3334b3e4c44738d714818521 --- /dev/null +++ b/configs/common/models/swin/swin_tiny_patch4_window7_224.py @@ -0,0 +1,27 @@ +from omegaconf import DictConfig +from libai.config import LazyCall +from libai.models import SwinTransformer + + +cfg = dict( + img_size=224, + patch_size=4, + in_chans=3, + num_classes=1000, + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop_rate=0.0, + drop_path_rate=0.2, + ape=False, + patch_norm=True, + loss_func=None, +) + +cfg = DictConfig(cfg) + +model = LazyCall(SwinTransformer)(cfg=cfg) diff --git a/configs/common/models/swinv2/swinv2_base_patch4_window16_256.py b/configs/common/models/swinv2/swinv2_base_patch4_window16_256.py new file mode 100644 index 0000000000000000000000000000000000000000..efcd6f5791d23219cabfac0d8fdc0b78dfaf7541 --- /dev/null +++ b/configs/common/models/swinv2/swinv2_base_patch4_window16_256.py @@ -0,0 +1,10 @@ +from libai.config import LazyCall +from libai.models import SwinTransformerV2 +from .swinv2_tiny_patch4_window8_256 import cfg + +cfg.window_size = 16 +cfg.depths = [2, 2, 18, 2] +cfg.num_heads = [4, 8, 16, 32] +cfg.drop_path_rate = 0.5 + +model = LazyCall(SwinTransformerV2)(cfg=cfg) diff --git a/configs/common/models/swinv2/swinv2_base_patch4_window8_256.py b/configs/common/models/swinv2/swinv2_base_patch4_window8_256.py new file mode 100644 index 0000000000000000000000000000000000000000..2ffed5ea275894cc01d2b416b0d5a66a1dbea3a7 --- /dev/null +++ b/configs/common/models/swinv2/swinv2_base_patch4_window8_256.py @@ -0,0 +1,9 @@ +from libai.config import LazyCall +from libai.models import SwinTransformerV2 +from .swinv2_tiny_patch4_window8_256 import cfg + +cfg.depths = [2, 2, 18, 2] +cfg.num_heads = [4, 8, 16, 32] +cfg.drop_path_rate = 0.5 + +model = LazyCall(SwinTransformerV2)(cfg=cfg) diff --git a/configs/common/models/swinv2/swinv2_small_patch4_window16_256.py b/configs/common/models/swinv2/swinv2_small_patch4_window16_256.py new file mode 100644 index 0000000000000000000000000000000000000000..36a95538b4331a6c88f8df73d634942c64f7c3b8 --- /dev/null +++ b/configs/common/models/swinv2/swinv2_small_patch4_window16_256.py @@ -0,0 +1,9 @@ +from libai.config import LazyCall +from libai.models import SwinTransformerV2 +from .swinv2_tiny_patch4_window8_256 import cfg + +cfg.window_size = 16 +cfg.depths = [2, 2, 18, 2] +cfg.drop_path_rate = 0.3 + +model = LazyCall(SwinTransformerV2)(cfg=cfg) diff --git a/configs/common/models/swinv2/swinv2_small_patch4_window8_256.py b/configs/common/models/swinv2/swinv2_small_patch4_window8_256.py new file mode 100644 index 0000000000000000000000000000000000000000..c825eb80622f2a38982694b0661ea5fe3e9edb75 --- /dev/null +++ b/configs/common/models/swinv2/swinv2_small_patch4_window8_256.py @@ -0,0 +1,8 @@ +from libai.config import LazyCall +from libai.models import SwinTransformerV2 +from .swinv2_tiny_patch4_window8_256 import cfg + +cfg.depths = [2, 2, 18, 2] +cfg.drop_path_rate = 0.3 + +model = LazyCall(SwinTransformerV2)(cfg=cfg) diff --git a/configs/common/models/swinv2/swinv2_tiny_patch4_window16_256.py b/configs/common/models/swinv2/swinv2_tiny_patch4_window16_256.py new file mode 100644 index 0000000000000000000000000000000000000000..7eafc4bead4e163f96d3c398fcd87994b55348c2 --- /dev/null +++ b/configs/common/models/swinv2/swinv2_tiny_patch4_window16_256.py @@ -0,0 +1,7 @@ +from libai.config import LazyCall +from libai.models import SwinTransformerV2 +from .swinv2_tiny_patch4_window8_256 import cfg + +cfg.window_size = 16 + +model = LazyCall(SwinTransformerV2)(cfg=cfg) diff --git a/configs/common/models/swinv2/swinv2_tiny_patch4_window8_256.py b/configs/common/models/swinv2/swinv2_tiny_patch4_window8_256.py new file mode 100644 index 0000000000000000000000000000000000000000..76cd44fff6b651f2cec1de77c744fa3b00266ef4 --- /dev/null +++ b/configs/common/models/swinv2/swinv2_tiny_patch4_window8_256.py @@ -0,0 +1,26 @@ +from omegaconf import DictConfig +from libai.config import LazyCall +from libai.models import SwinTransformerV2 + +cfg = dict( + img_size=256, + patch_size=4, + in_chans=3, + num_classes=1000, + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=8, + mlp_ratio=4.0, + qkv_bias=True, + drop_rate=0.0, + drop_path_rate=0.2, + ape=False, + patch_norm=True, + pretrained_window_sizes=[0, 0, 0, 0], + loss_func=None, +) + +cfg = DictConfig(cfg) + +model = LazyCall(SwinTransformerV2)(cfg=cfg) diff --git a/configs/common/models/t5.py b/configs/common/models/t5.py new file mode 100644 index 0000000000000000000000000000000000000000..a23c637fd7b44a331ebbc5cc254f27e5ebd68418 --- /dev/null +++ b/configs/common/models/t5.py @@ -0,0 +1,30 @@ +from omegaconf import DictConfig +from libai.config import LazyCall +from libai.models import T5Model, T5ForPreTraining + + +cfg = dict( + vocab_size=30522, + hidden_size=768, + hidden_layers=6, + num_attention_heads=16, + intermediate_size=1536, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + embedding_dropout_prob=0.1, + initializer_range=0.02, + layernorm_eps=1e-5, + bias_gelu_fusion=True, + bias_dropout_fusion=True, + scale_mask_softmax_fusion=True, + apply_query_key_layer_scaling=True, + apply_residual_post_layernorm=False, + amp_enabled=False, +) + +cfg = DictConfig(cfg) + +t5_model = LazyCall(T5Model)(cfg=cfg) + +pretrain_model = LazyCall(T5ForPreTraining)(cfg=cfg) diff --git a/configs/common/models/vit/vit_base_patch16_224.py b/configs/common/models/vit/vit_base_patch16_224.py new file mode 100644 index 0000000000000000000000000000000000000000..dd102b65672cf4313661b1ea16b5d5ebf612315a --- /dev/null +++ b/configs/common/models/vit/vit_base_patch16_224.py @@ -0,0 +1,11 @@ +from libai.config import LazyCall +from libai.models import VisionTransformer + +from .vit_tiny_patch16_224 import cfg + + +cfg.patch_size = 16 +cfg.embed_dim = 768 +cfg.num_heads = 12 + +model = LazyCall(VisionTransformer)(cfg=cfg) diff --git a/configs/common/models/vit/vit_base_patch32_224.py b/configs/common/models/vit/vit_base_patch32_224.py new file mode 100644 index 0000000000000000000000000000000000000000..345b2d2e76b5ed497533fec5942ebe5b26190cbf --- /dev/null +++ b/configs/common/models/vit/vit_base_patch32_224.py @@ -0,0 +1,11 @@ +from libai.config import LazyCall +from libai.models import VisionTransformer + +from .vit_tiny_patch16_224 import cfg + + +cfg.patch_size = 32 +cfg.embed_dim = 768 +cfg.num_heads = 12 + +model = LazyCall(VisionTransformer)(cfg=cfg) diff --git a/configs/common/models/vit/vit_giant_patch14_224.py b/configs/common/models/vit/vit_giant_patch14_224.py new file mode 100644 index 0000000000000000000000000000000000000000..03a3db45b1bfdcf3621da92693d97f23caed9647 --- /dev/null +++ b/configs/common/models/vit/vit_giant_patch14_224.py @@ -0,0 +1,13 @@ +from libai.config import LazyCall +from libai.models import VisionTransformer + +from .vit_tiny_patch16_224 import cfg + + +cfg.patch_size = 14 +cfg.embed_dim = 1408 +cfg.mlp_ratio = 48 / 11 +cfg.depth = 40 +cfg.num_heads = 16 + +model = LazyCall(VisionTransformer)(cfg=cfg) diff --git a/configs/common/models/vit/vit_gigantic_patch14_224.py b/configs/common/models/vit/vit_gigantic_patch14_224.py new file mode 100644 index 0000000000000000000000000000000000000000..383e5abe7f330517c7cc2c710752c77f337c5941 --- /dev/null +++ b/configs/common/models/vit/vit_gigantic_patch14_224.py @@ -0,0 +1,13 @@ +from libai.config import LazyCall +from libai.models import VisionTransformer + +from .vit_tiny_patch16_224 import cfg + + +cfg.patch_size = 14 +cfg.embed_dim = 1664 +cfg.mlp_ratio = 64 / 13 +cfg.depth = 48 +cfg.num_heads = 16 + +model = LazyCall(VisionTransformer)(cfg=cfg) diff --git a/configs/common/models/vit/vit_huge_patch14_224.py b/configs/common/models/vit/vit_huge_patch14_224.py new file mode 100644 index 0000000000000000000000000000000000000000..50193780b42a84466372ed91db993c4a3bee79b3 --- /dev/null +++ b/configs/common/models/vit/vit_huge_patch14_224.py @@ -0,0 +1,12 @@ +from libai.config import LazyCall +from libai.models import VisionTransformer + +from .vit_tiny_patch16_224 import cfg + + +cfg.patch_size = 16 +cfg.embed_dim = 1280 +cfg.depth = 32 +cfg.num_heads = 16 + +model = LazyCall(VisionTransformer)(cfg=cfg) diff --git a/configs/common/models/vit/vit_large_patch16_224.py b/configs/common/models/vit/vit_large_patch16_224.py new file mode 100644 index 0000000000000000000000000000000000000000..0b046da5f720094c4c1af7d7cf80ded57edd0ff3 --- /dev/null +++ b/configs/common/models/vit/vit_large_patch16_224.py @@ -0,0 +1,12 @@ +from libai.config import LazyCall +from libai.models import VisionTransformer + +from .vit_tiny_patch16_224 import cfg + + +cfg.patch_size = 16 +cfg.embed_dim = 1024 +cfg.depth = 24 +cfg.num_heads = 16 + +model = LazyCall(VisionTransformer)(cfg=cfg) diff --git a/configs/common/models/vit/vit_large_patch32_224.py b/configs/common/models/vit/vit_large_patch32_224.py new file mode 100644 index 0000000000000000000000000000000000000000..80e0d6f3b1d51064c19d6cf0f8ba523f25bda27c --- /dev/null +++ b/configs/common/models/vit/vit_large_patch32_224.py @@ -0,0 +1,12 @@ +from libai.config import LazyCall +from libai.models import VisionTransformer + +from .vit_tiny_patch16_224 import cfg + + +cfg.patch_size = 32 +cfg.embed_dim = 1024 +cfg.depth = 24 +cfg.num_heads = 16 + +model = LazyCall(VisionTransformer)(cfg=cfg) diff --git a/configs/common/models/vit/vit_small_patch16_224.py b/configs/common/models/vit/vit_small_patch16_224.py new file mode 100644 index 0000000000000000000000000000000000000000..54f6ea0ce9347bae7337d28a9b380a6525f94134 --- /dev/null +++ b/configs/common/models/vit/vit_small_patch16_224.py @@ -0,0 +1,11 @@ +from libai.config import LazyCall +from libai.models import VisionTransformer + +from .vit_tiny_patch16_224 import cfg + + +cfg.patch_size = 16 +cfg.embed_dim = 384 +cfg.num_heads = 6 + +model = LazyCall(VisionTransformer)(cfg=cfg) diff --git a/configs/common/models/vit/vit_small_patch32_224.py b/configs/common/models/vit/vit_small_patch32_224.py new file mode 100644 index 0000000000000000000000000000000000000000..b94c9d1d1778101efcb9d242754f64c471e1fbdc --- /dev/null +++ b/configs/common/models/vit/vit_small_patch32_224.py @@ -0,0 +1,12 @@ +from omegaconf import DictConfig +from libai.config import LazyCall +from libai.models import VisionTransformer + +from .vit_tiny_patch16_224 import cfg + + +cfg.patch_size = 32 +cfg.embed_dim = 384 +cfg.num_heads = 6 + +model = LazyCall(VisionTransformer)(cfg=cfg) diff --git a/configs/common/models/vit/vit_tiny_patch16_224.py b/configs/common/models/vit/vit_tiny_patch16_224.py new file mode 100644 index 0000000000000000000000000000000000000000..de198028e801b1d878a6c04acf9c77de5ea6d171 --- /dev/null +++ b/configs/common/models/vit/vit_tiny_patch16_224.py @@ -0,0 +1,23 @@ +from omegaconf import DictConfig +from libai.config import LazyCall +from libai.models import VisionTransformer + + +cfg = dict( + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=192, + depth=12, + num_heads=3, + mlp_ratio=4.0, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.0, + num_classes=1000, + loss_func=None, +) + +cfg = DictConfig(cfg) + +model = LazyCall(VisionTransformer)(cfg=cfg) diff --git a/configs/common/optim.py b/configs/common/optim.py new file mode 100644 index 0000000000000000000000000000000000000000..7cd5f2cf05ef469aa80cf64715e4940736613731 --- /dev/null +++ b/configs/common/optim.py @@ -0,0 +1,20 @@ +import oneflow as flow + +from libai.optim import get_default_optimizer_params +from libai.config import LazyCall + +optim = LazyCall(flow.optim.AdamW)( + params=LazyCall(get_default_optimizer_params)( + # params.model is meant to be set to the model object, + # before instantiating the optimizer. + clip_grad_max_norm=1.0, + clip_grad_norm_type=2.0, + weight_decay_norm=0.0, + weight_decay_bias=0.0, + ), + lr=1e-4, + weight_decay=0.01, + betas=(0.9, 0.999), + eps=1e-8, + do_bias_correction=True, +) diff --git a/configs/common/train.py b/configs/common/train.py new file mode 100644 index 0000000000000000000000000000000000000000..8b5a6a02b10d925369904d1adaaa21316e16e18d --- /dev/null +++ b/configs/common/train.py @@ -0,0 +1,150 @@ +from omegaconf import DictConfig + +from libai.config import LazyCall +from libai.scheduler import WarmupCosineLR +from libai.evaluation import ClsEvaluator + +# fmt: off +train = dict( + + # Directory where output files are written + output_dir="./output", + + # `train_micro_batch_size` is number of samples per batch on each GPU. + # train_mini_batch_size = train_micro_batch_size * num_accumulation_steps. + # This is also the number of training samples per step (i.e. per iteration). + + # If we use 8 GPUs for data parallel groups, `train_micro_batch_size = 2` and + # `num_accumulation_steps = 4`, then each GPU will see 2 samples per batch and + # 8 samples per iteration. + # Total 64 samples will be trained per iteration across all GPUs. + + # global_batch_size = micro_batch_size * num_grad_acc * data_parallel_groups + train_micro_batch_size=32, + global_batch_size=None, + num_accumulation_steps=None, + + # The total training iterations + train_iter=10000, + # The total training epochs, will be scaled to training iterations automatically. + # The actual total training iterations will be calculated by the + # formula `max(train_iter, train_epoch * iter_per_epoch)`. + train_epoch=0, + consumed_train_samples=0, + consumed_valid_samples=0, + train_samples=None, + + # Fraction of lr-warmup-iters to use for warmup (as a float) + warmup_ratio=0, + + # The start iteration, usually needn't set it manually. + # It can be computed automatically when resuming training. + start_iter=0, + + # Enable automatic mixed precision for training which does not + # change model's inference behavior. + amp=dict(enabled=False), + + # Enable activation checkpointing to allow for training + # with larger models, sequences, and batch sizes. + # If enabled, checkpoint the input activations of each transformer layers by default. + activation_checkpoint=dict(enabled=False), + + # NCCL fusion threshold megabytes, set to 0 to + # compatible with previous version of OneFlow. + nccl_fusion_threshold_mb=16, + + # Maximum number of ops of NCCL fusion, set to 0 to + # compatible with previous version of OneFlow. + nccl_fusion_max_ops=24, + + # Enable ZeRO Optimization to allow for training with larger models. + # This optimization will reduce optimizer stages memory consumption + # as described in ZeRO https://arxiv.org/abs/1910.02054. + zero_optimization=dict( + enabled=False, + stage=1, + ), + + # Save a model checkpoint after every this number of iterations, + # and maximum number of checkpoint will be kept. + checkpointer=dict(period=5000, max_to_keep=100, save_model_after_n_epoch=None), + + # Options for evaluation + + # `test_micro_batch_size` is number of samples per batch on each GPU for testing. + # If we use 8 GPUs for data parallel groups and `test_micro_batch_size = 2`, then + # total 16 samples will be used per iteration across all GPUs. + test_micro_batch_size=32, + + # Enabled evaluation during training, after every `eval_period` number of iterations + # will perform the evaluation process. + # You can set the maximum evaluation iterations to run for validation/test. + # You can also set a customized evaluator for use. + evaluation=dict( + enabled=False, + # evaluator for calculating top-k acc + evaluator=LazyCall(ClsEvaluator)(topk=(1, 5)), + eval_period=50000, + eval_after_n_epoch=None, + eval_iter=1e5, # running steps for validation/test + + # Metrics to be used for best model checkpoint. + eval_metric="Acc@1", + eval_mode="max", + ), + + # Path to a checkpoint file to be loaded to the model for training or evaluation. + load_weight="", + + # Output log to console after every this number of iterations. + log_period=20, + + # lr_scheduler arguments + # See libai/scheduler/lr_scheduler.py for definition. + scheduler=LazyCall(WarmupCosineLR)( + # In DefaultTrainer we will automatically set `max_iter` + # and `warmup_iter` by the given train cfg. + warmup_factor=0.001, + alpha=0.01, + warmup_method="linear", + ), + + # Distributed arguments + # See https://libai.readthedocs.io/en/latest/tutorials/Getting%20Started.html for more detail. + dist=dict( + data_parallel_size=1, + tensor_parallel_size=1, + pipeline_parallel_size=1, + # users must set the `pipeline_num_layers` attribute when `pipeline_parallel_size > 1` + pipeline_num_layers=None, + # users could customize the number of layers in different stages + # by setting the `custom_pipeline_stage_id ` attribute which is used for + # manually balance calculation between stages when running pipeline parallelism + # e.g. you can set `custom_pipeline_stage_id=[0, 0, 0, 1]` + # for `pipeline_num_layers=4 and pipeline_parallel_size=2` + # which means the first 3 layers will be placed on stage0 and + # the last layer will be placed on stage1 + # NOTE: if it is None, LiBai will automatically set pipeline_stage_id + # `auto_pipeline_stage_id` and `actual_pipeline_stage_id` will be saved in `config.yaml` + custom_pipeline_stage_id=None, + ), + + # the device type of input tensors for model, defaults to "cuda". + # if you want to accelerate the model training when pipeline_parallel > 1 + # you can set `input_placement_device="cpu"` then call input_tensor.to_global() + # inside your model.forward() method + # see `libai/models/bert_model.py` as reference + input_placement_device="cuda", + + # set to `True` to enable rdma for improving speed of pipeline_parallel + rdma_enabled=True, + + # Set seed to positive to use a fixed seed. Note that a fixed seed increases + # reproducibility but does not guarantee fully deterministic behavior. + # Disabling all parallelism further increases reproducibility. + seed=1234, +) +# fmt: on + +train = DictConfig(train) diff --git a/configs/gpt2_pretrain.py b/configs/gpt2_pretrain.py new file mode 100644 index 0000000000000000000000000000000000000000..93cc1c2692ee71c1b0d83ecb5d0ae0e1aa4a11bb --- /dev/null +++ b/configs/gpt2_pretrain.py @@ -0,0 +1,42 @@ +from libai.config import LazyCall +from libai.evaluation import PPLEvaluator +from .common.models.gpt import pretrain_model as model +from .common.train import train +from .common.optim import optim +from .common.data.gpt_dataset import dataloader, tokenization + +from .common.models.graph import graph + +vocab_file = "./nlp_data/gpt2-vocab.json" +merge_files = "./nlp_data/gpt2-merges.txt" +data_prefix = "./nlp_data/data/loss_compara_content_sentence" + +tokenization.tokenizer.vocab_file = vocab_file +tokenization.tokenizer.merges_file = merge_files +dataloader.train.dataset[0].data_prefix = data_prefix +dataloader.train.dataset[0].indexed_dataset.data_prefix = data_prefix + +# GPT-2 model config +model.cfg.embedding_dropout_prob = 0.1 +model.cfg.attention_dropout_prob = 0.1 +model.cfg.num_attention_heads = 16 +model.cfg.hidden_size = 384 +model.cfg.ffn_hidden_size = 1536 +model.cfg.hidden_layers = 6 +model.cfg.max_seq_length = 1024 + +train.input_placement_device = "cpu" + +train.dist.pipeline_num_layers = model.cfg.hidden_layers + +for ds in dataloader.train.dataset: + ds.max_seq_length = model.cfg.max_seq_length + +optim.lr = 1.5e-4 + +train.train_micro_batch_size = 4 +train.amp.enabled = True + +train.evaluation.evaluator = LazyCall(PPLEvaluator)() + +train.output_dir = "./output/gpt2_output" diff --git a/configs/resmlp_imagenet.py b/configs/resmlp_imagenet.py new file mode 100644 index 0000000000000000000000000000000000000000..d78fc2c7e486754eddd2fbeef8ee6dc6dcf831fa --- /dev/null +++ b/configs/resmlp_imagenet.py @@ -0,0 +1,89 @@ +from libai.config import LazyCall +from .common.models.resmlp.resmlp_12 import model +from .common.models.graph import graph +from .common.train import train +from .common.optim import optim +from .common.data.imagenet import dataloader + +import oneflow as flow +import flowvision.transforms as transforms +from flowvision.transforms import InterpolationMode +from flowvision.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD +from flowvision.data import Mixup +from flowvision.loss.cross_entropy import SoftTargetCrossEntropy + +# Refine output dir +train.output_dir = "./output_resmlp" + +# Refine data path to imagenet +dataloader.train.dataset[0].root = "/path/to/imagenet" +dataloader.test[0].dataset.root = "/path/to/imagenet" + +# Refine test data augmentation for resmlp model +resmlp_test_aug = LazyCall(transforms.Compose)( + transforms=[ + LazyCall(transforms.Resize)( + size=int(224 / 0.9), + interpolation=InterpolationMode.BICUBIC, + ), + LazyCall(transforms.CenterCrop)( + size=224, + ), + LazyCall(transforms.ToTensor)(), + LazyCall(transforms.Normalize)( + mean=IMAGENET_DEFAULT_MEAN, + std=IMAGENET_DEFAULT_STD, + ), + ] +) +dataloader.test[0].dataset.transform = resmlp_test_aug + +# Refine model cfg for resmlp training on imagenet +model.cfg.num_classes = 1000 +model.cfg.loss_func = SoftTargetCrossEntropy() + +# Add Mixup Func +dataloader.train.mixup_func = LazyCall(Mixup)( + mixup_alpha=0.8, + cutmix_alpha=1.0, + prob=1.0, + switch_prob=0.5, + mode="batch", + num_classes=model.cfg.num_classes, +) + +# Refine optimizer cfg for resmlp model +optim._target_ = flow.optim.LAMB # use lamb optimizer +optim.lr = 5e-3 # default batch size equals to 256 * 8 = 2048 +optim.eps = 1e-8 +optim.weight_decay = 0.2 +optim.params.clip_grad_max_norm = None +optim.params.clip_grad_norm_type = None +optim.params.overrides = { + "alpha": {"weight_decay": 0.0}, + "beta": {"weight_decay": 0.0}, + "gamma_1": {"weight_decay": 0.0}, + "gamma_2": {"weight_decay": 0.0}, +} + +# Refine train cfg for resmlp model +train.train_micro_batch_size = 256 +train.test_micro_batch_size = 64 +train.train_epoch = 400 +train.warmup_ratio = 5 / 400 +train.evaluation.eval_period = 1000 +train.log_period = 1 + +# Scheduler +train.scheduler.warmup_factor = 0.001 +train.scheduler.alpha = 0.01 +train.scheduler.warmup_method = "linear" + +# Set fp16 ON +train.amp.enabled = True + +# Distributed Settings +train.dist.pipeline_num_layers = model.cfg.depth +train.dist.data_parallel_size = 1 +train.dist.tensor_parallel_size = 1 +train.dist.pipeline_parallel_size = 1 diff --git a/configs/roberta_pretrain.py b/configs/roberta_pretrain.py new file mode 100644 index 0000000000000000000000000000000000000000..9935a9716cf967824f37ec4fa50c13582eee8539 --- /dev/null +++ b/configs/roberta_pretrain.py @@ -0,0 +1,42 @@ +from libai.config import LazyCall +from libai.evaluation import PPLEvaluator +from .common.models.roberta import pretrain_model as model +from .common.models.graph import graph +from .common.train import train +from .common.optim import optim +from .common.data.roberta_dataset import dataloader, tokenization + + +vocab_file = "./data_test/roberta_data/roberta-vocab.json" +merge_files = "./data_test/roberta_data/roberta-merges.txt" +data_prefix = "./data_test/roberta_data/loss_compara_content_sentence" + +tokenization.tokenizer.vocab_file = vocab_file +tokenization.tokenizer.merges_file = merge_files +dataloader.train.dataset[0].data_prefix = data_prefix +dataloader.train.dataset[0].indexed_dataset.data_prefix = data_prefix + +# RoBERTa model config +model.cfg.num_attention_heads = 12 +model.cfg.hidden_size = 768 +model.cfg.hidden_layers = 8 + +train.input_placement_device = "cpu" + +# parallel strategy settings +train.dist.data_parallel_size = 8 +train.dist.tensor_parallel_size = 1 +train.dist.pipeline_parallel_size = 1 + +train.dist.pipeline_num_layers = model.cfg.hidden_layers + +train.train_micro_batch_size = 2 + +train.amp.enabled = True + +for ds in dataloader.train.dataset: + ds.max_seq_length = model.cfg.max_position_embeddings + +train.evaluation.evaluator = LazyCall(PPLEvaluator)() + +train.output_dir = "output/roberta_output" diff --git a/configs/swin_cifar100.py b/configs/swin_cifar100.py new file mode 100644 index 0000000000000000000000000000000000000000..1e687bc1fe5d62961fcf1ca8219ab3183eb31363 --- /dev/null +++ b/configs/swin_cifar100.py @@ -0,0 +1,58 @@ +from libai.config import LazyCall +from .common.models.swin.swin_tiny_patch4_window7_224 import model +from .common.models.graph import graph +from .common.train import train +from .common.optim import optim +from .common.data.cifar100 import dataloader + +from flowvision.data import Mixup +from flowvision.loss.cross_entropy import SoftTargetCrossEntropy + +# Add Mixup Func +dataloader.train.mixup_func = LazyCall(Mixup)( + mixup_alpha=0.8, + cutmix_alpha=1.0, + prob=1.0, + switch_prob=0.5, + mode="batch", + num_classes=100, +) + +# Refine model cfg for vit training on cifar100 +model.cfg.num_classes = 100 +model.cfg.loss_func = SoftTargetCrossEntropy() + +# Refine optimizer cfg for swin model +optim.lr = 5e-4 +optim.eps = 1e-8 +optim.weight_decay = 0.05 +optim.params.clip_grad_max_norm = None +optim.params.clip_grad_norm_type = None + +# Refine train cfg for swin model +train.train_micro_batch_size = 32 +train.num_accumulation_steps = 1 +train.test_micro_batch_size = 32 +train.train_epoch = 300 +train.warmup_ratio = 20 / 300 +train.evaluation.eval_period = 200 +train.log_period = 20 + +# Scheduler +train.scheduler.warmup_factor = 5e-7 +train.scheduler.alpha = 0.0 +train.scheduler.warmup_method = "linear" + +# parallel strategy settings +train.dist.data_parallel_size = 8 +train.dist.tensor_parallel_size = 1 +train.dist.pipeline_parallel_size = 1 +train.dist.pipeline_num_layers = sum(model.cfg.depths) +train.output_dir = "./output" + +# Set fp16 ON +train.amp.enabled = False +train.activation_checkpoint.enabled = False +# train.zero_optimization.enabled = True +# train.zero_optimization.stage = 1 +graph.enabled = False diff --git a/configs/swin_imagenet.py b/configs/swin_imagenet.py new file mode 100644 index 0000000000000000000000000000000000000000..de7f434a3c1eac26d835697a48e6f82d07bcced5 --- /dev/null +++ b/configs/swin_imagenet.py @@ -0,0 +1,49 @@ +from libai.config import LazyCall +from .common.models.swin.swin_tiny_patch4_window7_224 import model +from .common.models.graph import graph +from .common.train import train +from .common.optim import optim +from .common.data.imagenet import dataloader + +from flowvision.data import Mixup +from flowvision.loss.cross_entropy import SoftTargetCrossEntropy + +# Refine data path to imagenet +dataloader.train.dataset[0].root = "/path/to/imagenet" +dataloader.test[0].dataset.root = "/path/to/imagenet" + +# Add Mixup Func +dataloader.train.mixup_func = LazyCall(Mixup)( + mixup_alpha=0.8, + cutmix_alpha=1.0, + prob=1.0, + switch_prob=0.5, + mode="batch", + num_classes=1000, +) + +# Refine model cfg for vit training on imagenet +model.cfg.num_classes = 1000 +model.cfg.loss_func = SoftTargetCrossEntropy() +# Refine optimizer cfg for vit model +optim.lr = 1e-3 +optim.eps = 1e-8 +optim.weight_decay = 0.05 +optim.params.clip_grad_max_norm = None +optim.params.clip_grad_norm_type = None + +# Refine train cfg for vit model +train.train_micro_batch_size = 128 +train.test_micro_batch_size = 128 +train.train_epoch = 300 +train.warmup_ratio = 20 / 300 +train.eval_period = 1562 +train.log_period = 100 + +# Scheduler +train.scheduler.warmup_factor = 0.001 +train.scheduler.alpha = 0.01 +train.scheduler.warmup_method = "linear" + +# Set fp16 ON +train.amp.enabled = True diff --git a/configs/swinv2_cifar100.py b/configs/swinv2_cifar100.py new file mode 100644 index 0000000000000000000000000000000000000000..c6915858e42f8e8609349a51392f7648460b7de1 --- /dev/null +++ b/configs/swinv2_cifar100.py @@ -0,0 +1,126 @@ +from libai.config import LazyCall + +from .common.models.swinv2.swinv2_tiny_patch4_window8_256 import model +from .common.models.graph import graph +from .common.train import train +from .common.optim import optim +from .common.data.cifar100 import dataloader + +from flowvision import transforms +from flowvision.transforms import InterpolationMode +from flowvision.transforms.functional import str_to_interp_mode +from flowvision.data import Mixup +from flowvision.loss.cross_entropy import SoftTargetCrossEntropy + +CIFAR100_TRAIN_MEAN = (0.5070751592371323, 0.48654887331495095, 0.4409178433670343) +CIFAR100_TRAIN_STD = (0.2673342858792401, 0.2564384629170883, 0.27615047132568404) + +# Add Mixup Func +dataloader.train.mixup_func = LazyCall(Mixup)( + mixup_alpha=0.8, + cutmix_alpha=1.0, + prob=1.0, + switch_prob=0.5, + mode="batch", + num_classes=100, +) +dataloader.train.dataset[0].transform = LazyCall(transforms.Compose)( + transforms=[ + LazyCall(transforms.RandomResizedCrop)( + size=(256, 256), + scale=(0.08, 1.0), + ratio=(3.0 / 4.0, 4.0 / 3.0), + interpolation=str_to_interp_mode("bicubic"), + ), + LazyCall(transforms.RandomHorizontalFlip)(), + LazyCall(transforms.ToTensor)(), + LazyCall(transforms.Normalize)(mean=CIFAR100_TRAIN_MEAN, std=CIFAR100_TRAIN_STD), + ] +) +dataloader.test[0].dataset.transform = LazyCall(transforms.Compose)( + transforms=[ + LazyCall(transforms.Resize)( + size=256, + interpolation=InterpolationMode.BICUBIC, + ), + LazyCall(transforms.CenterCrop)( + size=256, + ), + LazyCall(transforms.ToTensor)(), + LazyCall(transforms.Normalize)( + mean=CIFAR100_TRAIN_MEAN, + std=CIFAR100_TRAIN_STD, + ), + ] +) + +# Refine model cfg for vit training on cifar100 +model.cfg.num_classes = 100 +model.cfg.loss_func = SoftTargetCrossEntropy() + +# Refine optimizer cfg for swinv2 model +optim.lr = 5e-4 +optim.eps = 1e-8 +optim.weight_decay = 0.05 + + +def check_keywords_in_name(name, keywords=()): + isin = False + for keyword in keywords: + if keyword in name: + isin = True + return isin + + +def set_weight_decay(model, skip_list=(), skip_keywords=()): + has_decay = [] + no_decay = [] + + for name, param in model.named_parameters(): + if not param.requires_grad: + continue # frozen weights + if ( + len(param.shape) == 1 + or name.endswith(".bias") + or (name in skip_list) + or check_keywords_in_name(name, skip_keywords) + ): + no_decay.append(param) + else: + has_decay.append(param) + return [{"params": has_decay}, {"params": no_decay, "weight_decay": 0.0}] + + +optim.params = LazyCall(set_weight_decay)( + model=model, + skip_list=("absolute_pos_embed"), + skip_keywords=("cpb_mlp", "logit_scale", "relative_position_bias_table"), +) +# Refine train cfg for swin model +train.train_micro_batch_size = 32 +train.num_accumulation_steps = 8 +train.test_micro_batch_size = 32 +train.train_epoch = 300 +train.warmup_ratio = 20 / 300 +train.evaluation.eval_period = 1562 +train.log_period = 10 + +# Scheduler +train.scheduler.warmup_factor = 5e-7 +train.scheduler.alpha = 0.0 +train.scheduler.warmup_method = "linear" + +# parallel strategy settings +train.dist.data_parallel_size = 1 +train.dist.tensor_parallel_size = 1 +train.dist.pipeline_parallel_size = 1 +train.dist.pipeline_num_layers = sum(model.cfg.depths) +train.output_dir = "./output" + +train.rdma_enabled = False +# Set fp16 ON +train.amp.enabled = False +train.activation_checkpoint.enabled = False +# train.zero_optimization.enabled = True +# train.zero_optimization.stage = 1 +graph.enabled = False diff --git a/configs/swinv2_imagenet.py b/configs/swinv2_imagenet.py new file mode 100644 index 0000000000000000000000000000000000000000..217e0d05f92021271717f80ada46532b6259843d --- /dev/null +++ b/configs/swinv2_imagenet.py @@ -0,0 +1,141 @@ +from libai.config import LazyCall +from .common.models.swinv2.swinv2_tiny_patch4_window8_256 import model +from .common.models.graph import graph +from .common.train import train +from .common.optim import optim +from .common.data.imagenet import dataloader + +from flowvision import transforms +from flowvision.data import Mixup +from flowvision.loss.cross_entropy import SoftTargetCrossEntropy +from flowvision.transforms import InterpolationMode +from flowvision.transforms.functional import str_to_interp_mode +from flowvision.data.constants import ( + IMAGENET_DEFAULT_MEAN, + IMAGENET_DEFAULT_STD, +) +from flowvision.data.auto_augment import rand_augment_transform +from flowvision.data.random_erasing import RandomErasing + + +# Refine data path to imagenet +dataloader.train.dataset[0].root = "/path/to/imagenet" +dataloader.test[0].dataset.root = "/path/to/imagenet" + +# Add Mixup Func +dataloader.train.mixup_func = LazyCall(Mixup)( + mixup_alpha=0.8, + cutmix_alpha=1.0, + prob=1.0, + switch_prob=0.5, + mode="batch", + num_classes=1000, +) + +dataloader.train.dataset[0].transform = LazyCall(transforms.Compose)( + transforms=[ + LazyCall(transforms.RandomResizedCrop)( + size=256, + scale=(0.08, 1.0), + ratio=(3.0 / 4.0, 4.0 / 3.0), + interpolation=InterpolationMode.BICUBIC, + ), + LazyCall(transforms.RandomHorizontalFlip)(p=0.5), + LazyCall(rand_augment_transform)( + config_str="rand-m9-mstd0.5-inc1", + hparams=dict( + translate_const=int(256 * 0.45), + img_mean=tuple([min(255, round(255 * x)) for x in IMAGENET_DEFAULT_MEAN]), + interpolation=str_to_interp_mode("bicubic"), + ), + ), + LazyCall(transforms.ToTensor)(), + LazyCall(transforms.Normalize)( + mean=IMAGENET_DEFAULT_MEAN, + std=IMAGENET_DEFAULT_STD, + ), + LazyCall(RandomErasing)( + probability=0.25, + mode="pixel", + max_count=1, + num_splits=0, + device="cpu", + ), + ] +) +dataloader.test[0].dataset.transform = LazyCall(transforms.Compose)( + transforms=[ + LazyCall(transforms.Resize)( + size=256, + interpolation=InterpolationMode.BICUBIC, + ), + LazyCall(transforms.CenterCrop)( + size=256, + ), + LazyCall(transforms.ToTensor)(), + LazyCall(transforms.Normalize)( + mean=IMAGENET_DEFAULT_MEAN, + std=IMAGENET_DEFAULT_STD, + ), + ] +) + + +# Refine model cfg for vit training on imagenet +model.cfg.num_classes = 1000 +model.cfg.loss_func = SoftTargetCrossEntropy() + +# Refine optimizer cfg for vit model +optim.lr = 1e-3 # The pytorch version is 1024 as the total batch size, 1e-3 as the learning rate +optim.eps = 1e-8 +optim.weight_decay = 0.05 + + +def check_keywords_in_name(name, keywords=()): + isin = False + for keyword in keywords: + if keyword in name: + isin = True + return isin + + +def set_weight_decay(model, skip_list=(), skip_keywords=()): + has_decay = [] + no_decay = [] + + for name, param in model.named_parameters(): + if not param.requires_grad: + continue # frozen weights + if ( + len(param.shape) == 1 + or name.endswith(".bias") + or (name in skip_list) + or check_keywords_in_name(name, skip_keywords) + ): + no_decay.append(param) + else: + has_decay.append(param) + return [{"params": has_decay}, {"params": no_decay, "weight_decay": 0.0}] + + +optim.params = LazyCall(set_weight_decay)( + model=model, + skip_list=("absolute_pos_embed"), + skip_keywords=("cpb_mlp", "logit_scale", "relative_position_bias_table"), +) + +# Refine train cfg for vit model +train.train_micro_batch_size = 128 +train.test_micro_batch_size = 128 +train.train_epoch = 300 +train.warmup_ratio = 20 / 300 +train.eval_period = 1562 +train.log_period = 100 +graph.enabled = False +train.rdma_enabled = True +# Scheduler +train.scheduler.warmup_factor = 0.001 +train.scheduler.alpha = 0.01 +train.scheduler.warmup_method = "linear" +# Set fp16 ON +train.amp.enabled = True diff --git a/configs/t5_large_pretrain.py b/configs/t5_large_pretrain.py new file mode 100644 index 0000000000000000000000000000000000000000..1d859fd8bb99758170a01b5bb76275a4b70a3361 --- /dev/null +++ b/configs/t5_large_pretrain.py @@ -0,0 +1,31 @@ +from libai.config import LazyCall +from libai.evaluation import PPLEvaluator +from .common.models.t5 import pretrain_model as model +from .common.train import train +from .common.optim import optim +from .common.data.t5_dataset import dataloader, tokenization + +from .common.models.graph import graph + +vocab_file = "./data_test/bert_data/bert-base-chinese-vocab.txt" +data_prefix = "./data_test/bert_data/loss_compara_content_sentence" + +tokenization.tokenizer.vocab_file = vocab_file +dataloader.train.dataset[0].data_prefix = data_prefix +dataloader.train.dataset[0].indexed_dataset.data_prefix = data_prefix + +# T5-large model config +model.cfg.num_attention_heads = 12 +model.cfg.hidden_size = 384 +model.cfg.hidden_layers = 6 + +train.input_placement_device = "cpu" + +train.dist.pipeline_num_layers = 2 * model.cfg.hidden_layers + +train.train_micro_batch_size = 16 +train.amp.enabled = True + +train.evaluation.evaluator = LazyCall(PPLEvaluator)() + +train.output_dir = "./output/t5_output" diff --git a/configs/vit_imagenet.py b/configs/vit_imagenet.py new file mode 100644 index 0000000000000000000000000000000000000000..91bdc914b5d6a2b6a7b973495bbb6f8e4080f2e3 --- /dev/null +++ b/configs/vit_imagenet.py @@ -0,0 +1,57 @@ +from libai.config import LazyCall +from .common.models.vit.vit_base_patch16_224 import model +from .common.models.graph import graph +from .common.train import train +from .common.optim import optim +from .common.data.imagenet import dataloader + +from flowvision.data import Mixup +from flowvision.loss.cross_entropy import SoftTargetCrossEntropy + +# Refine data path to imagenet +dataloader.train.dataset[0].root = "/path/to/imagenet" +dataloader.test[0].dataset.root = "/path/to/imagenet" + +# Refine model cfg for vit training on imagenet +model.cfg.num_classes = 1000 +model.cfg.loss_func = SoftTargetCrossEntropy() + +# Add Mixup Func +dataloader.train.mixup_func = LazyCall(Mixup)( + mixup_alpha=0.8, + cutmix_alpha=1.0, + prob=1.0, + switch_prob=0.5, + mode="batch", + num_classes=model.cfg.num_classes, +) + +# Refine optimizer cfg for vit model +optim.lr = 1e-3 # 5e-4 * 1024 (batchsize) / 512 +optim.eps = 1e-8 +optim.weight_decay = 0.05 +optim.params.clip_grad_max_norm = None +optim.params.clip_grad_norm_type = None +optim.params.overrides = {"pos_embed": {"weight_decay": 0.0}, "cls_token": {"weight_decay": 0.0}} + +# Refine train cfg for vit model +train.train_micro_batch_size = 128 +train.test_micro_batch_size = 128 +train.train_epoch = 300 +train.warmup_ratio = 5 / 300 +train.evaluation.eval_period = 1000 +train.log_period = 1 + +# Scheduler +train.scheduler.warmup_factor = 0.001 +train.scheduler.alpha = 0.01 +train.scheduler.warmup_method = "linear" + +# Set fp16 ON +train.amp.enabled = True + +# Distributed Settings +train.dist.pipeline_num_layers = model.cfg.depth +train.dist.data_parallel_size = 1 +train.dist.tensor_parallel_size = 1 +train.dist.pipeline_parallel_size = 1 diff --git a/dev/README.md b/dev/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1d5eb7bc7ee5fccff7e183f17b704b9310b1dd84 --- /dev/null +++ b/dev/README.md @@ -0,0 +1,5 @@ + +# Some scripts for developers to use, include: + +- `linter.sh`: lint the codebase before commit. +- `run_unittest.sh`: run unittest for codebase. diff --git a/dev/inference_test.sh b/dev/inference_test.sh new file mode 100644 index 0000000000000000000000000000000000000000..6f5cd2809843e5d5312c157ead5e22753dc4941b --- /dev/null +++ b/dev/inference_test.sh @@ -0,0 +1,15 @@ +#!/bin/bash -e + +# cd to libai project root +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +export TEST_OUTPUT=output_unittest +export ONEFLOW_TEST_DEVICE_NUM=4 + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/inference/test_text_generation.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/inference/test_text_classification.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/inference/test_image_classification.py + +rm -rf $TEST_OUTPUT diff --git a/dev/linter.sh b/dev/linter.sh new file mode 100644 index 0000000000000000000000000000000000000000..17be977cf3e456b1f3823f0b1902cd7e8ef9492d --- /dev/null +++ b/dev/linter.sh @@ -0,0 +1,40 @@ +#!/bin/bash -e + +# cd to libai project root +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +{ + black --version | grep -E "21\." > /dev/null +} || { + echo "Linter requires 'black==21.*' !" + exit 1 +} + +ISORT_VERSION=$(isort --version-number) +if [[ "$ISORT_VERSION" != 5.10.1 ]]; then + echo "Linter requires isort==5.10.1 !" + exit 1 +fi + +set -v + +echo "Running autoflake ..." +autoflake --remove-unused-variables --in-place --recursive . + +echo "Running isort ..." +isort . --atomic + +echo "Running black ..." +black -l 100 . + +echo "Running flake8 ..." +if [ -x "$(command -v flake8-3)" ]; then + flake8-3 . +else + python3 -m flake8 . +fi + +echo "Running clang-format ..." +find . -regex ".*\.\(cpp\|c\|cc\|cu\|cxx\|h\|hh\|hpp\|hxx\|tcc\|mm\|m\)" -print0 | xargs -0 clang-format -i + +command -v arc > /dev/null && arc lint \ No newline at end of file diff --git a/dev/model_loader_test.sh b/dev/model_loader_test.sh new file mode 100644 index 0000000000000000000000000000000000000000..cbe26d0a871979aa8b9894d7216fa954aeb7df90 --- /dev/null +++ b/dev/model_loader_test.sh @@ -0,0 +1,26 @@ +#!/bin/bash -e + +# cd to libai project root +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +export TEST_OUTPUT=output_unittest +export ONEFLOW_TEST_DEVICE_NUM=4 +export ONEFLOW_EP_CUDA_ENABLE_TF32_EXECUTION=0 + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/model_utils/test_bert_loader.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/model_utils/test_roberta_loader.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/model_utils/test_gpt_loader.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/model_utils/test_mt5_loader.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/model_utils/test_t5_loader.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/model_utils/test_swin_loader.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/model_utils/test_swinv2_loader.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/model_utils/test_vit_loader.py + +rm -rf $TEST_OUTPUT \ No newline at end of file diff --git a/dev/model_test.sh b/dev/model_test.sh new file mode 100644 index 0000000000000000000000000000000000000000..00fcfb71995cca466080a7da31dff419f7060dde --- /dev/null +++ b/dev/model_test.sh @@ -0,0 +1,25 @@ +#!/bin/bash -e + +# cd to libai project root +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +export TEST_OUTPUT=output_unittest +export ONEFLOW_TEST_DEVICE_NUM=4 + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_bert.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_roberta.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_gpt.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_t5.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_mt5.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_vit.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_swin.py + +python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_swinv2.py + +rm -rf $TEST_OUTPUT diff --git a/dev/run_unittest.sh b/dev/run_unittest.sh new file mode 100644 index 0000000000000000000000000000000000000000..782913b96ea1f275e9a31c5ff3c75dc35d40d78f --- /dev/null +++ b/dev/run_unittest.sh @@ -0,0 +1,6 @@ +#!/bin/bash -e + +# cd to libai project root +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +pytest --disable-warnings ./tests diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..64c85f9afd1c36ed36087f9fc7b25c93cf8729a7 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,35 @@ +FROM nvidia/cuda:11.1.1-cudnn8-devel-ubuntu18.04 +# use an older system (18.04) to avoid opencv incompatibility (issue#3524) + +# Comment it if you are not in China +RUN sed -i 's/security.ubuntu.com/mirrors.ustc.edu.cn/g' /etc/apt/sources.list +RUN sed -i 's/archive.ubuntu.com/mirrors.ustc.edu.cn/g' /etc/apt/sources.list + +ENV DEBIAN_FRONTEND noninteractive +RUN apt-get update && apt-get install -y \ + python3-opencv ca-certificates python3-dev git wget sudo ninja-build +RUN ln -sv /usr/bin/python3 /usr/bin/python + +# create a non-root user +ARG USER_ID=1000 +RUN useradd -m --no-log-init --system --uid ${USER_ID} appuser -g sudo +RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +USER appuser +WORKDIR /home/appuser + +ENV PATH="/home/appuser/.local/bin:${PATH}" +RUN wget https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ + python3 get-pip.py --user && \ + rm get-pip.py + +# install dependencies +# See https://pytorch.org/ for other options if you use a different version of CUDA +RUN pip install --user pybind11 cmake # cmake from apt-get is too old +RUN python3 -m pip install --user --pre oneflow -f https://staging.oneflow.info/branch/master/cu112 + +# install libai +RUN git clone https://github.com/Oneflow-Inc/libai.git libai_repo + +RUN pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --user -e libai_repo + +WORKDIR /home/appuser/libai_repo diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f970182075c69dca3d224ba33416a6e72d6b4a2b --- /dev/null +++ b/docker/README.md @@ -0,0 +1,23 @@ +## Use the container (with docker >= 19.03) + +```shell +cd docker/ +# Build: +docker build --build-arg USER_ID=$UID -t libai:v0 . +# Launch (require GPUs): +docker run --gpus all -it --name libai -v /home/user/:/workspace/ --ipc host --net host libai:v0 /bin/bash +``` + +You can also use the pre-built LiBai containers (the latest compatible version at time of publication can be pulled with `docker pull l1aoxingyu/libai:v0`). + +> NOTE: If your nvidia-driver cannot support `cuda111`, you can build the base image from `cuda102`. + +## Install new dependencies + +Add the following to `Dockerfile` to make persistent changes. + +```shell +RUN sudo apt-get update && sudo apt-get install -y vim +``` + +Or run them in the container to make temporary changes. diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..1f6715901ac50a05b2e45fd53fec4c2bdebb4e11 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +html: Makefile + @$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +clean: Makefile + @rm -rf build diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b035b86317608210b0e6536749b57607affdce8b --- /dev/null +++ b/docs/README.md @@ -0,0 +1,12 @@ +## Read LiBai Documentation +The latest documentation built from this directory is available at [libai.readthedocs.io](https://libai.readthedocs.io/). + + +## Build LiBai Documentation +1. Install LiBai according to [Installation](https://libai.readthedocs.io/en/latest/tutorials/get_started/Installation.html). +2. Install additional libraries and run `make html` for building the docs: +```bash +cd ${libai-path}/docs +pip install -r requirements.txt --user +make html +``` diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..cfdb9b951bf36941b33103ba7f6e4a1bc56d51e7 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,8 @@ +sphinx==3.5.4 +jinja2<3.1 +recommonmark==0.6.0 +sphinx-rtd-theme==1.0.0 +git+https://github.com/Oneflow-Inc/libai.git +--pre +-f https://staging.oneflow.info/branch/master/cpu +oneflow diff --git a/docs/source/_static/css/line_space.css b/docs/source/_static/css/line_space.css new file mode 100644 index 0000000000000000000000000000000000000000..ffa66038b41f6e32e502db40ef54efd67aeb76a6 --- /dev/null +++ b/docs/source/_static/css/line_space.css @@ -0,0 +1,9 @@ +.rst-content .section ol li>*, .rst-content .section ul li>* { + margin-top: 0px; + margin-bottom: 0px; +} + +.rst-content .section ol li>*, .rst-content .section ul li ul { + margin-top: 0px; + margin-bottom: 0px; +} diff --git a/docs/source/_templates/.gitkeep b/docs/source/_templates/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/docs/source/_templates/line_space.html b/docs/source/_templates/line_space.html new file mode 100644 index 0000000000000000000000000000000000000000..8407acc82655f32b109b8b665b7f04eca039ffd8 --- /dev/null +++ b/docs/source/_templates/line_space.html @@ -0,0 +1,2 @@ +{% extends "!line_space.html" %} +{% set css_files = css_files + [ "_static/css/line_space.css" ] %} diff --git a/docs/source/changelog.md b/docs/source/changelog.md new file mode 100644 index 0000000000000000000000000000000000000000..9cc603b09484200b690b2a7734245b0b5e35a2ea --- /dev/null +++ b/docs/source/changelog.md @@ -0,0 +1,46 @@ +## Changelog + +### v0.2.0 (07/07/2022) +**New Features:** +- Support evaluation enabled and set `eval_iter` +- Support customized sampler in `config.py` +- Support rdma for pipeline-model-parallel +- Support multi fused kernel + - fused_scale_mask_softmax_dropout + - fused_scale_tril_softmax_mask_scale + - fused_self_attention in branch `libai_bench` +- User Experience Optimization +- Optimization for training throughput, see [benchmark](https://libai.readthedocs.io/en/latest/tutorials/get_started/Benchmark.html) for more details + +**New Supported Models:** + +- Support 3D parallel [Roberta](https://arxiv.org/abs/1907.11692) model +- Support 2D parallel (data parallel + tensor model parallel) [SimCSE](https://arxiv.org/abs/2104.08821) model +- Support Data parallel [MAE](https://arxiv.org/abs/2111.06377) model +- Support Data parallel [MOCOV3](https://arxiv.org/abs/2104.02057) model + +### v0.1.0 (22/03/2022) + +**New Features:** +- Support Data Parallelism +- Support 1D Tensor Parallelism +- Support Pipeline Parallelism +- Unified distributed Layers for both single-GPU and multi-GPU training +- `LazyConfig` system for more flexible syntax and no predefined structures +- Easy-to-use trainer and engine +- Support both CV and NLP data processing +- Mixed Precision Training +- Activation Checkpointing +- Gradient Accumulation +- Gradient Clipping +- Zero Redundancy Optimizer (ZeRO) + +**Supported Models:** +- Support 3D parallel [BERT](https://arxiv.org/abs/1810.04805) model +- Support 3D parallel [GPT-2](https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf) model +- Support 3D parallel [T5](https://arxiv.org/abs/1910.10683) model +- Support 3D parallel [Vision Transformer](https://arxiv.org/abs/2010.11929) model +- Support Data parallel [Swin Transformer](https://arxiv.org/abs/2103.14030) model +- Support finetune task in [QQP project](https://github.com/Oneflow-Inc/libai/tree/main/projects/QQP) +- Support text classification task in [text classification project](https://github.com/Oneflow-Inc/libai/tree/main/projects/text_classification) + diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000000000000000000000000000000000000..f29963b561d6a5e4918fb4fec13509d034346842 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,71 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys + +import libai # noqa + +sys.path.insert(0, os.path.abspath(".")) + +# -- Project information ----------------------------------------------------- + +project = "libai" +copyright = "2021, OneFlow" +author = "OneFlow" + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "recommonmark", +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "Python" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +# html_theme = "furo" +html_theme = "sphinx_rtd_theme" + + +def setup(app): + app.add_css_file("css/line_space.css") + + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..24ae04dc14a7c1ea170b2256f9db81b72aea1751 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,15 @@ +.. libai documentation master file, created by + sphinx-quickstart on Mon Nov 29 10:26:07 2021. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to libai's documentation! +====================================== + +.. toctree:: + :maxdepth: 3 + + tutorials/index + notes/index + modules/index + changelog.md \ No newline at end of file diff --git a/docs/source/modules/index.rst b/docs/source/modules/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..bdb4bc2128453fc818bd8640865040c487cfa6b8 --- /dev/null +++ b/docs/source/modules/index.rst @@ -0,0 +1,17 @@ +API Documentation +================= + +.. toctree:: + :maxdepth: 2 + :caption: API Documentation + + libai.config + libai.data + libai.engine + libai.evaluation + libai.layers + libai.models + libai.optim + libai.scheduler + libai.tokenizer + libai.utils \ No newline at end of file diff --git a/docs/source/modules/libai.config.rst b/docs/source/modules/libai.config.rst new file mode 100644 index 0000000000000000000000000000000000000000..d9c589533d1add7d0cc8e18708a23e7103c1ff7e --- /dev/null +++ b/docs/source/modules/libai.config.rst @@ -0,0 +1,12 @@ +libai.config +############################## + +.. currentmodule:: libai.config +.. automodule:: libai.config + :members: + try_get_key, + get_config, + configurable, + instantiate, + LazyCall, + LazyConfig, diff --git a/docs/source/modules/libai.data.rst b/docs/source/modules/libai.data.rst new file mode 100644 index 0000000000000000000000000000000000000000..51c773c3041810e390ff8b0c62f4e7483c2a6104 --- /dev/null +++ b/docs/source/modules/libai.data.rst @@ -0,0 +1,48 @@ +libai.data +############################## + +libai.data.data_utils module +--------------------------------- +.. currentmodule:: libai.data +.. automodule:: libai.data.data_utils + :members: + IndexedCachedDataset, + IndexedDataset, + MMapIndexedDataset, + get_indexed_dataset, + + +libai.data.datasets module +--------------------------------- +.. currentmodule:: libai.data +.. automodule:: libai.data.datasets + :members: + MNISTDataset, + CIFAR10Dataset, + CIFAR100Dataset, + ImageNetDataset, + BertDataset, + RobertaDataset, + T5Dataset, + GPT2Dataset + + +libai.data.samplers module +--------------------------------- +.. currentmodule:: libai.data +.. automodule:: libai.data.samplers + :members: + CyclicSampler, + SingleRoundSampler, + +libai.data.build module +--------------------------------- +.. currentmodule:: libai.data +.. automodule:: libai.data.build + :members: + build_nlp_train_val_test_loader, + build_nlp_train_loader, + build_nlp_test_loader, + build_image_train_loader, + build_image_test_loader, + diff --git a/docs/source/modules/libai.engine.rst b/docs/source/modules/libai.engine.rst new file mode 100644 index 0000000000000000000000000000000000000000..7d52e76cbc1e6931ede766bc38c8e558b79a01d7 --- /dev/null +++ b/docs/source/modules/libai.engine.rst @@ -0,0 +1,38 @@ +libai.engine +############################## + +libai.engine.default module +--------------------------------- +.. currentmodule:: libai.engine +.. automodule:: libai.engine.default + :member-order: bysource + :members: + default_setup, + DefaultTrainer, + + +libai.engine.hooks module +--------------------------------- +.. currentmodule:: libai.engine +.. automodule:: libai.engine.hooks + :member-order: bysource + :members: + CallbackHook, + IterationTimer, + PeriodicWriter, + PeriodicCheckpointer, + BestCheckpointer, + EvalHook, + LRScheduler, + + +libai.engine.trainer module +--------------------------------- +.. currentmodule:: libai.engine +.. automodule:: libai.engine.trainer + :member-order: bysource + :members: + HookBase, + TrainerBase, + EagerTrainer, + GraphTrainer, \ No newline at end of file diff --git a/docs/source/modules/libai.evaluation.rst b/docs/source/modules/libai.evaluation.rst new file mode 100644 index 0000000000000000000000000000000000000000..bba8313af6076373d50d83d7b4ff49c88e0ffcc6 --- /dev/null +++ b/docs/source/modules/libai.evaluation.rst @@ -0,0 +1,14 @@ +libai.evaluation +############################## + +.. currentmodule:: libai.evaluation +.. automodule:: libai.evaluation + :members: + DatasetEvaluator, + inference_on_dataset, + print_csv_format, + flatten_results_dict, + ClsEvaluator, + PPLEvaluator, + RegEvaluator, + BLEUEvaluator, diff --git a/docs/source/modules/libai.layers.rst b/docs/source/modules/libai.layers.rst new file mode 100644 index 0000000000000000000000000000000000000000..2bb9518669c2fe7a7f599eaee20dc8c3432919d2 --- /dev/null +++ b/docs/source/modules/libai.layers.rst @@ -0,0 +1,21 @@ +libai.layers +############################## + +.. currentmodule:: libai.layers +.. automodule:: libai.layers + :members: + Embedding, + VocabEmbedding, + SinePositionalEmbedding, + PatchEmbedding, + drop_path, + DropPath, + build_activation, + Linear, + Linear1D, + MLP, + LayerNorm, + RMSLayerNorm, + MultiheadAttention, + TransformerLayer, + ParallelCrossEntropyLoss \ No newline at end of file diff --git a/docs/source/modules/libai.models.rst b/docs/source/modules/libai.models.rst new file mode 100644 index 0000000000000000000000000000000000000000..aaf2a4dd29695c7d550f11b4efddbf6fe1ca2fce --- /dev/null +++ b/docs/source/modules/libai.models.rst @@ -0,0 +1,83 @@ +libai.models +############################## +Supported models in LiBai(李白) + +- `VisionTransformer`_ +- `SwinTransformer`_ +- `SwinTransformerV2`_ +- `ResMLP`_ +- `BERT`_ +- `RoBERTa`_ +- `T5`_ +- `GPT-2`_ + +.. _VisionTransformer: https://arxiv.org/abs/2010.11929 +.. _SwinTransformer: https://arxiv.org/abs/2103.14030 +.. _SwinTransformerV2: https://arxiv.org/abs/2111.09883 +.. _ResMLP: https://arxiv.org/abs/2105.03404 +.. _BERT: https://arxiv.org/abs/1810.04805 +.. _RoBERTa: https://arxiv.org/abs/1907.11692 +.. _T5: https://arxiv.org/abs/1910.10683 +.. _GPT-2: https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf + + + +.. currentmodule:: libai.models + +.. automodule:: libai.models.build + :members: + build_model, + build_graph, + +VisionTransformer +----------------- +.. automodule:: libai.models.vision_transformer + :members: + VisionTransformer, + +SwinTransformer +--------------- +.. automodule:: libai.models.swin_transformer + :members: + SwinTransformer, + +SwinTransformerV2 +----------------- +.. automodule:: libai.models.swin_transformer_v2 + :members: + SwinTransformerV2, + +ResMLP +------- +.. automodule:: libai.models.resmlp + :members: + ResMLP, + +BERT +---- +.. automodule:: libai.models.bert_model + :members: + BertModel, + BertForPreTraining, + +RoBERTa +------- +.. automodule:: libai.models.roberta_model + :members: + RobertaModel, + RobertaForPreTraining, + RobertaForCausalLM, + +T5 +--- +.. automodule:: libai.models.t5_model + :members: + T5ForPreTraining, + T5Model, + +GPT-2 +----- +.. automodule:: libai.models.gpt_model + :members: + GPTForPreTraining, + GPTModel \ No newline at end of file diff --git a/docs/source/modules/libai.optim.rst b/docs/source/modules/libai.optim.rst new file mode 100644 index 0000000000000000000000000000000000000000..f54197ae3ae2113162492df3de431a1589009f14 --- /dev/null +++ b/docs/source/modules/libai.optim.rst @@ -0,0 +1,8 @@ +libai.optim +############################## + +.. currentmodule:: libai.optim +.. automodule:: libai.optim + :members: + build_optimizer, + get_default_optimizer_params, \ No newline at end of file diff --git a/docs/source/modules/libai.scheduler.rst b/docs/source/modules/libai.scheduler.rst new file mode 100644 index 0000000000000000000000000000000000000000..3b59cec3cb37574de85337b6a113e6bb465b4dc2 --- /dev/null +++ b/docs/source/modules/libai.scheduler.rst @@ -0,0 +1,11 @@ +libai.scheduler +############################## + +.. currentmodule:: libai.scheduler +.. automodule:: libai.scheduler + :members: + WarmupCosineAnnealingLR, + WarmupCosineLR, + WarmupExponentialLR, + WarmupMultiStepLR, + WarmupPolynomialLR \ No newline at end of file diff --git a/docs/source/modules/libai.tokenizer.rst b/docs/source/modules/libai.tokenizer.rst new file mode 100644 index 0000000000000000000000000000000000000000..cf91ad9934660be37d7def9c7864b93b1bd7df6d --- /dev/null +++ b/docs/source/modules/libai.tokenizer.rst @@ -0,0 +1,12 @@ +libai.tokenizer +############################## + +.. currentmodule:: libai.tokenizer +.. automodule:: libai.tokenizer + :member-order: bysource + :members: + BertTokenizer, + RobertaTokenizer, + GPT2Tokenizer, + T5Tokenizer, + PreTrainedTokenizer, diff --git a/docs/source/modules/libai.utils.rst b/docs/source/modules/libai.utils.rst new file mode 100644 index 0000000000000000000000000000000000000000..8aafbcdbc16accacef7ded8086b3a657c5d3483e --- /dev/null +++ b/docs/source/modules/libai.utils.rst @@ -0,0 +1,54 @@ +libai.utils +############################## + +libai.utils.distributed module +--------------------------------- +.. currentmodule:: libai.utils +.. automodule:: libai.utils.distributed + :members: + get_dist_util, + setup_dist_util, + ttol, + tton, + synchronize, + convert_to_distributed_default_setting, + get_nd_sbp, + get_layer_placement, + get_world_size, + get_num_nodes, + get_rank, + get_local_rank, + same_sbp, + get_data_parallel_size, + get_data_parallel_rank, + get_hidden_sbp + +libai.utils.events module +--------------------------------- +.. currentmodule:: libai.utils +.. automodule:: libai.utils.events + :members: + get_event_storage, + JSONWriter, + CommonMetricPrinter, + EventStorage, + + +libai.utils.logger module +--------------------------------- +.. currentmodule:: libai.utils +.. automodule:: libai.utils.logger + :members: + setup_logger, + log_first_n, + log_every_n, + log_every_n_seconds + + +libai.utils.checkpoint module +--------------------------------- +.. currentmodule:: libai.utils +.. automodule:: libai.utils.checkpoint + :members: + Checkpointer, + PeriodicCheckpointer diff --git a/docs/source/notes/FAQ.md b/docs/source/notes/FAQ.md new file mode 100644 index 0000000000000000000000000000000000000000..7ab6ddfae7f833decc5f6e26379b860cfa01f03c --- /dev/null +++ b/docs/source/notes/FAQ.md @@ -0,0 +1,45 @@ +# Frequently Asked Questions + +We list some common problems encountered by users and the corresponding solutions here. Feel free to enrich the list if you find any frequent issues and have ways to help others solve them. + +## Training + +- "Loss goes NaN or very large" + 1. Check if the dataset annotations are valid. Mask must be `{0, 1}` where `1` for tokens that are **not masked** and `0` for tokens that are **masked**. + 2. Check `initializer_range` in config file. It can be safely set to `0.02` in most cases. If the model size is very large, decreasing `initializer_range` is a good choice. For example, `initializer_range` can be set to `0.006` when training 175 billion parameter configuration GPT-3 model. + +- "AMP enabled goes NaN" + Set `ONEFLOW_DEBUG_KERNEL_SYNC_CHECK_NUMERICS=1` to check what triggers an overflow of the value range in fp16. + +- "GPU out of memory when validation" + Decrease `test_micro_batch_size` and use `--fast-dev-run` for quickly running through training and evaluation to check if memory is sufficient. + + +## Model + +- "`apply_query_key_layer_scaling` in MultiheadAttention" + As the number of attention heads increases, some of the GEMMS inside the self-attention layer become smaller and the number of elements in the self attention softmax also increases. + +- "QKV implementation is not consistent with Hugging Face in self attention" + ```python + # query_key_value：[batch_size, seq_len, 3*hidden_size] + + # QKV in LiBai + query_key_value = query_key_value.view(bsz, -1, self.num_heads, 3 * self.head_size) + query_key_value = query_key_value.permute(0, 2, 1, 3) + query, key, value = flow.chunk(query_key_value, chunks=3, dim=-1) + + # QKV in Huggingface + query, key, value = flow.chunk(query_key_value, chunks=3, dim=-1) + query = query.view(query.size(0), query.size(1), self.num_heads, -1).permute(0, 2, 1, 3) + key = key.view(key.size(0), key.size(1), self.num_heads, -1).permute(0, 2, 1, 3) + value = value.view(value.size(0), value.size(1), self.num_heads, -1).permute(0, 2, 1, 3) + ``` + In tensor parallelism, `chunk` dimension and `flow.sbp.split` dimension will be the same in Huggingface's implementation which will occur some unexpected behaviors (i.e., changing the tensor's SBP unexpectedly). + + We also provide a tutorial about how to load Huggingface weights correctly. Please refer to [How to use Huggingface's pretrained weights in LiBai](https://libai.readthedocs.io/en/latest/notes/How_to_implement_huggingface%27s_weights_in_LiBai.html) for more details. + +- "the order of layer normalization and the residual connection" + This is critical to enable the scaling of the BERT-style models beyond BERT-Large. The architecture with `apply_residual_post_layernorm=False` eliminates instabilities observed using the origin BERT architecture with `apply_residual_post_layernorm=True` and also has a lower training loss according to [Megatron-LM](https://arxiv.org/pdf/1909.08053.pdf). + +If you find some troubles hard to understand, feel free to open an issue to collect feedbacks in [OneFlow](https://github.com/Oneflow-Inc/oneflow). \ No newline at end of file diff --git a/docs/source/notes/How_to_build_vision_transformer_model_in_LiBai.md b/docs/source/notes/How_to_build_vision_transformer_model_in_LiBai.md new file mode 100644 index 0000000000000000000000000000000000000000..874741647859337b30e99ef7763c126f704f2576 --- /dev/null +++ b/docs/source/notes/How_to_build_vision_transformer_model_in_LiBai.md @@ -0,0 +1,433 @@ +# Detailed instruction on building Vision Transformer models in LiBai +It's easy for users to build the `transformer-based` models by using LiBai's built-in [layers](https://libai.readthedocs.io/en/latest/modules/libai.layers.html). Let's take a deep dive into the process of building a Vision Transformer model in LiBai. + +## Model Architecture +**Vision Transformer** was released in the paper [An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale](https://arxiv.org/abs/2010.11929) by Alexey Dosovitskiy, Lucas Beyer, Alexander Kolesnikov, Dirk Weissenborn, Xiaohua Zhai, Thomas Unterthiner, Mostafa Dehghani, Matthias Minderer, Georg Heigold, Sylvain Gelly, Jakob Uszkoreit, Neil Houlsby. + +A **Vision Transformer** model contains three parts: `Patch Embedding` + `Transformer Block` + `Linear Classification Head`, which can be summarized in the following picture: + +![](./assets/vision_transformer.png) + +## A simple Torch implementation of Vision Transformer +The following code shows the PyTorch implementation of ViT models modified from [timm.models.vision_transformer](https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py): + +```python +import torch +import torch.nn as nn + +from timm.models.layers import trunc_normal_, PatchEmbed, Mlp, DropPath + +""" +1. Build a self-attention module +""" +class Attention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, attn_drop=0.0, proj_drop=0.0): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x): + B, N, C = x.shape + qkv = ( + self.qkv(x) + .reshape(B, N, 3, self.num_heads, C // self.num_heads) + .permute(2, 0, 3, 1, 4) + ) + q, k, v = qkv[0], qkv[1], qkv[2] + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + +""" +2. Build a transformer block, which contains: + self-attention layer + mlp layer +""" +class Block(nn.Module): + def __init__( + self, + dim, + num_heads, + mlp_ratio=4.0, + qkv_bias=False, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + ): + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + attn_drop=attn_drop, + proj_drop=drop, + ) + # Use drop_path here + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_layer=act_layer, + drop=drop, + ) + + def forward(self, x): + x = x + self.drop_path(self.attn(self.norm1(x))) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + +""" +3. Build a Vision Transformer model which contains three parts: + patch embedding + transformer block + mlp classification head +""" +class VisionTransformer(nn.Module): + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=192, + depth=12, + num_heads=3, + mlp_ratio=4.0, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.0, + num_classes=1000, + ): + super().__init__() + self.num_classes = num_classes + + # patch embedding + self.patch_embed = PatchEmbed( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim + ) + num_patches = self.patch_embed.num_patches + + # cls token and position embedding + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.pos_embed = self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + 1, embed_dim)) + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth decay rule + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] + + # transformer block + self.blocks = nn.Sequential(*[ + Block( + dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=True, drop=drop_rate, + attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=nn.LayerNorm, act_layer=nn.GELU) + for i in range(depth)]) + + # classification head + self.norm = nn.LayerNorm(embed_dim) + self.head = nn.Linear(embed_dim, num_classes) + + # weight init + trunc_normal_(self.pos_embed, std=0.02) + trunc_normal_(self.cls_token, std=0.02) + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def forward_features(self, x): + # patch embedding + x = self.patch_embed(x) + + cls_token = self.cls_token.expand(x.shape[0], -1, -1) + x = torch.cat((cls_token, x), dim=1) + + # position embedding + pos_embed = self.pos_embed.expand(x.shape[0], -1, -1) + x = self.pos_drop(x + pos_embed) + + # transformer block + x = self.blocks(x) + return x + + def forward_head(self, x): + # only use cls token for classification + x = self.norm(x) + outcome = x[:, 0] + return self.head(outcome) + + def forward(self, x): + x = self.forward_features(x) + x = self.forward_head(x) + return x +``` +We have further decoupled the forward function into `forward_features` and `forward_head`: +- `forward_features`: extract the image features using the `patch_embed` layer and a stack of `transformer` blocks +- `forward_head`: take the `cls_token` of each sample and use `nn.Linear` for classification + +## Implement 3D parallel Vision Transformer in LiBai +In this section, we will show users how to use [libai.layers](https://libai.readthedocs.io/en/latest/modules/libai.layers.html) to build a 3D parallel Vision Transformer model with only 100+ lines of code, which is modified from [libai.models.vision_transformer](https://github.com/Oneflow-Inc/libai/blob/main/libai/models/vision_transformer.py) + +Here is the LiBai implementation of Vision Transformer models, and users only need to replace the PyTorch modules with the corresponding `libai.layers` as follows: + +```python +# LiBai's implementation of Vision Transformer +import oneflow as flow +import oneflow.nn as nn +from flowvision.layers.weight_init import trunc_normal_ + +import libai.utils.distributed as dist +from libai.config.config import configurable +from libai.layers import LayerNorm, Linear, PatchEmbedding, TransformerLayer + +""" +LiBai has already implemented: +1. PatchEmbedding Layer +2. Transformer Layer: Self-Attention + MLP + DropPath + LayerNorm +3. Linear Layer +We can directly build a Vision Transformer model with the built-in layers in LiBai as follows: +""" + +class VisionTransformer(nn.Module): + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=192, + depth=12, + num_heads=3, + mlp_ratio=4.0, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.0, + num_classes=1000, + loss_func=None + ): + super().__init__() + self.num_classes = num_classes + # patch embedding + self.patch_embed = PatchEmbedding( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + ) + num_patches = self.patch_embed.num_patches + + # cls token and position embedding with sbp signature + self.cls_token = nn.Parameter( + flow.zeros(1, 1, embed_dim, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0),) + ) + self.pos_embed = nn.Parameter( + flow.zeros(1, num_patches+1, embed_dim, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0),) + ) + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth decay rule + dpr = [x.item() for x in flow.linspace(0, drop_path_rate, depth)] + + # a stack of transformer block + ffn_size = int(embed_dim * mlp_ratio) + self.blocks = nn.Sequential( + *[ + TransformerLayer( + hidden_size=embed_dim, + ffn_hidden_size=ffn_size, + num_attention_heads=num_heads, + attention_dropout_prob=attn_drop_rate, + output_dropout_prob=drop_rate, + drop_path_prob=dpr[i], + layer_idx=i, + ) + for i in range(depth) + ] + ) + self.norm = LayerNorm(embed_dim, layer_idx=-1) + self.head = Linear(embed_dim, num_classes, layer_idx=-1) + + # implement loss function in nn.Module to match LiBai style + self.loss_func = nn.CrossEntropyLoss() if loss_func is None else loss_func + + # weight init function + trunc_normal_(self.pos_embed, std=0.02) + trunc_normal_(self.cls_token, std=0.02) + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, Linear): + trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def forward_features(self, x): + # patch embedding + x = self.patch_embed(x) + + cls_token = self.cls_token.expand( + x.shape[0], -1, -1 + ) # stole cls_tokens impl from Phil Wang, thanks + cls_token = cls_token.to_global(sbp=x.sbp, placement=cls_token.placement) + x = flow.cat((cls_token, x), dim=1) + + # position embedding + pos_embed = self.pos_embed.expand(x.shape[0], -1, -1) + pos_embed = pos_embed.to_global(sbp=x.sbp, placement=pos_embed.placement) + x = self.pos_drop(x + pos_embed) + + # transformer block + x = self.blocks(x) + return x + + def forward_head(self, x): + x = self.norm(x) + outcome = x[:, 0] + outcome = self.head(outcome) + return outcome + + def forward(self, images, labels=None): + x = self.forward_features(images) + x = self.forward_head(x) + + if labels is not None and self.training: + losses = self.loss_func(x, labels) + return {"losses": losses} + else: + return {"prediction_scores": x} + + @staticmethod + def set_pipeline_stage_id(model): + dist_utils = dist.get_dist_util() + + # Set pipeline parallelism stage_id + for module_block in model.modules(): + if isinstance(module_block.to(nn.Module), PatchEmbedding): + module_block.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(0)) + elif isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(module_block.layer_idx)) + + # Set pos_embed and cls_token stage id + model.pos_embed.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(0)) + model.cls_token.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(0)) + model.pos_drop.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(0)) + model.norm.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(-1)) + model.head.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(-1)) + model.loss_func.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(-1)) +``` + +## Details about LiBai's implementation of the Vision Transformer model + +**1. Replace nn.Module with libai.layers** + +LiBai has already implemented `PatchEmbedding`, `TransformerLayer`, `Linear`, `LayerNorm` layers, and users only need to replace the module in Torch Vision Transformer models to convert a Torch model into LiBai's style: + - `Block` -> `libai.layers.TransformerLayer` + - `nn.Linear` -> `libai.layers.Linear` + - `nn.LayerNorm` -> `libai.layers.LayerNorm` + - `PatchEmbed` -> `libai.layers.PatchEmbedding` + +**2. Manually set the SBP signature of `cls_token` and `pos_embed`** + +In order to fit different parallel modes in LiBai, users must manually set the [SBP signature](https://docs.oneflow.org/en/master/parallelism/02_sbp.html#spb-signature) for all the parameters and buffers of those layers not implemented in LiBai, like `cls_token` and `pos_embed` in Vision Transformer: +```python +import oneflow as flow +import oneflow.nn as nn +import libai.utils.distributed as dist + +self.cls_token = nn.Parameter( + flow.zeros( + 1, 1, embed_dim, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) +) +self.pos_embed = nn.Parameter( + flow.zeros( + 1, num_patches+1, embed_dim, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0),) +) +``` +- The SBP signature returned by `dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast])` means to broadcast `cls_token` and `pos_embed` across each GPU group. + +**3. Use the `to_global()` function to update the SBP signature of `cls_token` and `pos_embed` during forward function** + +In forward function, `cls_token` and `pos_embed` will be expanded to fit the input size. For efficiency, we can use the `to_global()` function to match the `cls_token` and `pos_embed` SBP signature with the input SBP signature like this: +```python +def forward_features(self, x): + cls_token = self.cls_token.expand( + x.shape[0], -1, -1 + ) + # use to_global to update the sbp signature of cls_token + cls_token = cls_token.to_global(sbp=x.sbp, placement=cls_token.placement) + x = flow.cat((cls_token, x), dim=1) + + # use to_global to update the sbp signature of pos_embed + pos_embed = self.pos_embed.expand(x.shape[0], -1, -1) + pos_embed = pos_embed.to_global(sbp=x.sbp, placement=pos_embed.placement) +``` + +**4. Manually set the stage id for pipeline parallel training** + +Most of the built-in layers in LiBai has the arg named `layer_idx` for pipeline parallel settings. To configure a **1F1B pipeline parallel** model, users should manually set the stage id for each layers in the model, which will automatically assign different layers on different stages and insert buffer in the process of forward & backward computation for 1F1B pipeline parallel training. With the help of `layer_idx`, we can simply get a pipeline parallel Vision Transformer model like: +```python +import libai.utils.distributed as dist + +""" +This is a staticmethod for class inherited from nn.Module, +""" +@staticmethod +def set_pipeline_stage_id(model): + dist_utils = dist.get_dist_util() + + # Set pipeline parallelism stage_id + for module_block in model.modules(): + # module_block.to(nn.Module) can get the original module + if isinstance(module_block.to(nn.Module), PatchEmbedding): + module_block.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(0)) + elif isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(module_block.layer_idx)) + + # Set pos_embed and cls_token stage id + model.pos_embed.to(nn.graph.GraphModule).set_stage(sdist_utils.get_layer_stage_id(0)) + model.cls_token.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(0)) + model.pos_drop.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(0)) + model.norm.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(-1)) + model.head.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(-1)) + model.loss_func.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(-1)) +``` + +Manually set the stage id: +- `PatchEmbedding` should be on the first stage +- Automatically assign the stage for `TransformerLayer` with `layer_idx` args +- `cls_token`, `pos_embed`, `pos_drop` should be on the first stage +- `norm`, `head` and `loss_func` should be on the last stage + +Please see [Write your own pipeline parallel model](https://libai.readthedocs.io/en/latest/tutorials/advanced_tutorials/customize_parallel.html#write-your-own-pipeline-parallel-model) for more details about the settings of pipeline parallel training in LiBai. diff --git a/docs/source/notes/How_to_load_huggingface's_pretrained_model_in_libai.md b/docs/source/notes/How_to_load_huggingface's_pretrained_model_in_libai.md new file mode 100644 index 0000000000000000000000000000000000000000..82a8a7044b637fb15481b12cfadfa438af4c2d51 --- /dev/null +++ b/docs/source/notes/How_to_load_huggingface's_pretrained_model_in_libai.md @@ -0,0 +1,116 @@ +# How to load pretrained model in LiBai +In this tutorial, we will introduce to users how to instantiate a pretrained oneflow model. + +## Steps +Firstly, Prepare pretrained model weights file, which can be the form of `OneFlow` or `HuggingFace`. +- `OneFlow`'s pretrained model weights saved using [`oneflow.save()`]. +- `Huggingface`'s pretrained model weights file(`pytorch_model.bin`) can be downloaded from https://huggingface.co/models. + +Secondly, Prepare config file. +> The config file is required when loading the `HuggingFace` model. +> `OneFlow`'s config file can be import directly from `configs/common/models`. +- `Huggingface`'s config file(`config.json`) can be downloaded from https://huggingface.co/models. + +Lastly, The structure of the pretrained model folder should be like: +```bash +# OneFlow pretrained model +$ tree pretrained_model_dir +path/to/pretrained_model_dir/ + └── oneflow_model + +# Huggingface pretrained model +$ tree pretrained_model_dir +path/to/pretrained_model_dir/ + ├── pytorch_model.bin + └── config.json +``` + +## Start Loading +You can load pretrained BERT as following: +```python +import libai +from libai.models.utils import BertLoaderHuggerFace, BertLoaderLiBai +from libai.config.configs.common.models.bert import cfg + + +# load huggingface weight +loader = BertLoaderHuggerFace( + model=libai.models.BertModel, + libai_cfg=cfg, + pretrained_model_path='path/to/huggingface_pretrained_model_directory', + hidden_dropout_prob=0, +) +bert = loader.load() + +# load libai weight +loader = BertLoaderLiBai( + model=libai.models.BertModel, + libai_cfg=cfg, + pretrained_model_path='path/to/libai_pretrained_model_directory', + hidden_dropout_prob=0, +) +bert = loader.load() +``` + + +# Use Custom ModelLoader + +## Model Loader for HuggerFace +If you want to define your own HuggerFace's model loader, you can inherit the base `ModelLoaderHuggerFace` in `libai.models.utils.model_utils.base_loader`. + +Then you need to overwrite the `_convert_state_dict` and `_load_config_from_json` method to load HuggingFace's pretrained model in LiBai. + +Finally, you need set `base_model_prefix_1` and `base_model_prefix_2` argument, which represent the base model name for HuggingFace and LiBai respectively. + +The following code shows how to use custom ModelLoaderHuggerFace: + +```python +from libai.models.utils import ModelLoaderHuggerFace + + +class ToyModelLoaderHuggerFace(ModelLoaderHuggerFace): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + + """NOTE: base_model_prefix_1 is ToyModel's prefix in Transformers. + base_model_prefix_2 is ToyModel's prefix in LiBai.""" + self.base_model_prefix_1 = "toy_model" + self.base_model_prefix_2 = "toy_model" + + def _convert_state_dict(self, flow_state_dict, cfg): + """Convert state_dict's keys to match model. + + Args: + flow_state_dict (OrderedDict): model state dict. + cfg (dict): model's default config dict in LiBai. + + Returns: + OrderedDict: flow state dict. + """ + ... + + def _load_config_from_json(self, config_file): + """load config from `config.json`, and update default config. + + Args: + config_file (str): Path of config file. + """ + ... +``` + +## Model Loader for LiBai +If you want to define your own LiBai's model loader, you can inherit the base `ModelLoaderLiBai` class in `libai.models.utils.model_utils.base_loader`. + +You just need to set `base_model_prefix_2` argument to load LiBai's pretrained model. + +The following code shows how to use custom ModelLoaderLiBai: + +```python +from libai.models.utils import ModelLoaderLiBai + + +class ToyModelLoaderLiBai(ModelLoaderLiBai): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + self.base_model_prefix_2 = "toy_model" +``` \ No newline at end of file diff --git a/docs/source/notes/How_to_use_distributed_inference_in_LiBai.md b/docs/source/notes/How_to_use_distributed_inference_in_LiBai.md new file mode 100644 index 0000000000000000000000000000000000000000..5e348692066fccfee32970216b87dfc133fa3823 --- /dev/null +++ b/docs/source/notes/How_to_use_distributed_inference_in_LiBai.md @@ -0,0 +1,150 @@ +# Detailed instruction for using distributed inference in LiBai + +If you want to using distributed inference in LiBai from pretrained `pytorch` model, you can refer to [DALLE2 inferecn doc](https://github.com/Oneflow-Inc/libai/blob/main/docs/source/notes/How_to_use_model_parallel_in_LiBai.md). And [Chinese doc for distributed inference](https://github.com/Oneflow-Inc/libai/discussions/386) is also available. + +Here we introduce how to use distributed infenrence in LiBai: + +## Check `model.py` + +check your `model.py` first: +1. Ensure There are `libai.layers` in your `model.py`: + ```python + # NOTE: you don't need to import all layers from libai, if you only use libai.layers.Linear + # in your `model.py`, you model will run model/pipeline parallel only in `Linear` layers + from libai.layers import ( + Linear, + LayerNorm, + ... + ) + ``` +2. If you want to run pipeline parallel in LiBai, you should additionally insert code `x = x.to_global(placement=target_tensor.placement)` in your `model.forward()`. +It is equal to torch code `x.to(cuda_device)`, which move tensor from gpuA to gpuB. There are many examples in LiBai: [example1](https://github.com/Oneflow-Inc/libai/blob/92dbe7c1b1496290e32e595f8473f9288ea1886e/projects/MT5/layers/attention_layer.py#L220), [example2](https://github.com/Oneflow-Inc/libai/blob/92dbe7c1b1496290e32e595f8473f9288ea1886e/projects/MT5/layers/attention_layer.py#L156) ... + + If you don't know where to insert code, you can run your code first, and the it will raise bug in the line which needed `to_global`. + for example: + + ```shell + File "libai/libai/layers/layer_norm.py", line 129, in forward + + return flow._C.rms_layer_norm(hidden_states, self.weight, self.l2norm_epsilon) RuntimeError: return flow._C.rms_layer_norm(hidden_states, self.weight, self.l2norm_epsilon)RuntimeErrorExpected all tensors to be on the same placement, but found at least two placements, oneflow.placement(type="cuda", ranks=[0, 1]) (positional 0) and oneflow.placement(type="cuda", ranks=[2, 3]) (positional 1)! + ``` + +## Build `config.py` + +If your model is Trained from LiBai, you can use the same `config.py` from training. refer to [Couplets](https://github.com/Oneflow-Inc/libai/tree/main/projects/Couplets#inference) for more details + +If your model is Trainer from other framework, you should build your own `inference_config.py`, you can refer to [`dalle2_config.py`](https://github.com/Oneflow-Inc/libai/blob/main/projects/DALLE2/configs/dalle2_config.py) and [`t5_inference_config.py `](https://github.com/Oneflow-Inc/libai/blob/main/projects/MT5/configs/t5_inference.py) + +## Refine `pipeline_inference.py` + +The base class [libai/inference/basic.py](https://github.com/Oneflow-Inc/libai/blob/main/libai/inference/basic.py) is already provided in `LiBai` , +Users only need to overload the functions they need. refer to [text_generation.py](https://github.com/Oneflow-Inc/libai/blob/main/libai/inference/text_generation.py) + +If your model is trained from `LiBai`, it will be easy to use, you can refer to [distribute_infer.py](https://github.com/Oneflow-Inc/libai/blob/main/projects/Couplets/distribute_infer.py) in [Couplets](https://github.com/Oneflow-Inc/libai/tree/main/projects/Couplets) + +If your model is trained from other framework, you need to build your own `model_loader` to load your model weights in LiBai, refer to [model_loader](https://libai.readthedocs.io/en/latest/notes/How_to_load_huggingface%27s_pretrained_model_in_libai.html) for more details + +Give a simple example, the function overloaded in `LiBai`: +```python +from libai.inference.basic import BasePipeline +from libai.utils import distributed as dist + +class MyPipeline(BasePipeline): + def _parse_parameters(self, **pipeline_parameters): + # By overloading this function, the input parameters in MyPipeline.__call__() hand out to preprocess/forward/postprocess stages of inference. + preprocess_params = { + "preprocess_param1": pipeline_parameters["preprocess_param1"], + "preprocess_param2": pipeline_parameters["preprocess_param2"], + } + forward_params = { + "forward_param": pipeline_parameters["forward_param"] + } + postprocess_params = { + "postprocess_param": pipeline_parameters["postprocess_param"] + } + return preprocess_params, forward_params, postprocess_params + + def load_pretrain_weight(self, libai_cfg_model, model_path, mode="myloader"): + # load your pretrain weight in this functor + # set your own "MyLoader" if your model is pretrained from other framework + # set mode to "libai" if your model is pretrained from libai + if mode == "myloader": + import MyLoader + + model_loader = MyLoader( + libai_cfg_model, + libai_cfg_model.cfg, + model_path, + ..., + ) + return model_loader.load() + else: + return super().load_pretrain_weight( + libai_cfg_model, + model_path, + mode=mode, + ) + + def preprocess(self, inputs, preprocess_param1, preprocess_param2, **kwargs): + ... + # model_input_dict: {"key1": flow.Tensor1, ...} + return model_input_dict + + def forward(self, model_input_dict, forward_param, **kwargs): + ... + model_output_dict = self.model(**model_input_dict) + return model_output_dict + + def postprocess(self, model_output_dict, postprocess_param, **kwargs): + ... + return out_dict + +if __name__ == "__main__": + pipeline = MyPipeline( + "path/to/myconfig.py", + data_parallel=1, + tensor_parallel=..., + pipeline_parallel=..., + pipeline_stage_id=..., + pipeline_num_layers=..., + model_path=..., + mode=..., + ) + out = pipeline( + input_text=..., + preprocess_param1=..., + preprocess_param2=..., + forward_param=..., + postprocess_param=..., + ) + if dist.is_main_process(): + print(out) +``` + +## Distributed run `pipeline_inference.py` + +To run model on 2 nodes with total 4 GPUs, + + in `node0`, run: + ```bash + NODE=2 NODE_RANK=1 ADDR=192.168.0.1 PORT=12345 bash tools/infer.sh pipeline_inference.py 2 + ``` + `NODE=2` means total number of nodes + + `NODE_RANK=0` means current node is node0 + + `ADDR=192.168.0.0` means the ip address of node0 + + `PORT=12345` means the port of node0 + + in `node1`, run: + ```bash + NODE=2 NODE_RANK=1 ADDR=192.168.0.1 PORT=12345 bash tools/infer.sh pipeline_inference.py 2 + ``` + `NODE=2` means total number of nodes + + `NODE_RANK=1` means current node is node1 + + `ADDR=192.168.0.0` means the ip address of node0 + + `PORT=12345` means the port of node0 diff --git a/docs/source/notes/How_to_use_huggingface's_weights_in_LiBai.md b/docs/source/notes/How_to_use_huggingface's_weights_in_LiBai.md new file mode 100644 index 0000000000000000000000000000000000000000..3534c217b32af0f7a023e07f96bc8587cc4b9efc --- /dev/null +++ b/docs/source/notes/How_to_use_huggingface's_weights_in_LiBai.md @@ -0,0 +1,45 @@ +# How to use Huggingface's pretrained weights in LiBai +The built-in layers in [LiBai](https://github.com/Oneflow-Inc/libai) adopts the structure which is more suitable for parallel training, therefore the implementation in LiBai may be a little bit different from that in Huggingface. In this tutorial, we will introduce to users how to correctly load Huggingface's pretrained weights into LiBai's model. Let's take BERT as an example. + + +## LiBai Transformer vs Huggingface Transformer +There are subtle differences in the BERT structure as shown in the following figure (left: LiBai, right: Huggingface), which can be summarized as: +- Location of layernorm: The location of layernorm is different, but the calculation order is the same. +- A different slicing way to get the `query`, `key` and `value` matrix. +- LiBai follows [Megatron-LM](https://github.com/NVIDIA/Megatron-LM) to use the order of the layernorm and the residual connections by default. Megatron-LM shows that this structure will eliminate instabilities and bring a lower training loss. LiBai can also support the original BERT architecture mentioned in [Paper](https://arxiv.org/pdf/1810.04805.pdf) by setting `apply_residual_post_layernorm=True`. + +![architecture](./assets/architecture.jpg) + + +## QKV slicing logic +LiBai's QKV slicing logic is different from that in Huggingface. +```python +# LiBai's QKV slicing logic +query_key_value = query_key_value.view(batch_size, -1, num_heads, 3 * head_size) +query_key_value = query_key_value.permute(0, 2, 1, 3) +query, key, value = flow.chunk(query_key_value, chunks=3, dim=-1) + +# Huggingface's QKV slicing logic +query, key, value = flow.chunk(query_key_value, chunks=3, dim=-1) +query = query.view(query.size(0), query.size(1), num_heads, -1).permute(0, 2, 1, 3) +key = key.view(key.size(0), key.size(1), num_heads, -1).permute(0, 2, 1, 3) +value = value.view(value.size(0), value.size(1), num_heads, -1).permute(0, 2, 1, 3) +``` + + +## How to correctly load QKV weights +- To correctly load Huggingface's transformer weights, you only need to rearrange the loaded weights as follows: + +```python +def convert_qkv_weight(cfg, qkv_weight, qkv_bias): + qkv_weight = qkv_weight.view([3, cfg.num_heads, cfg.head_size, cfg.hidden_size]) + qkv_weight = qkv_weight.permute(1, 0, 2, 3).contiguous().view(3*cfg.hidden_size, cfg.hidden_size) + qkv_bias = qkv_bias.view(3, cfg.num_heads, cfg.head_size) + qkv_bias = qkv_bias.permute(1,0,2).contiguous().view(-1) + return qkv_weight, qkv_bias +``` + +- For detailed examples, please refer to [load-huggingface-bert](https://github.com/Oneflow-Inc/libai/tree/test_bert_load_huggingface_weight/projects/test_bert_load_huggingface_weight). You can verify this by running: +```bash +bash test.sh +``` \ No newline at end of file diff --git a/docs/source/notes/How_to_use_model_parallel_in_LiBai.md b/docs/source/notes/How_to_use_model_parallel_in_LiBai.md new file mode 100644 index 0000000000000000000000000000000000000000..1b1de8ce0ed3f50b9e95bf3a7028b8b0867195cc --- /dev/null +++ b/docs/source/notes/How_to_use_model_parallel_in_LiBai.md @@ -0,0 +1,333 @@ +# Detailed instruction on using model parallel in LiBai +This document is a tutorial for users to learn how to transer a pytorch model to oneflow, and use model parallel in Libai for inference. We will first take the DALLE2 model for example, and then we will show how to use model parallel which can be easily done in libai. + +**Note**: the code of DALLE2 is adapted from [this repo](https://github.com/lucidrains/DALLE2-pytorch), which is an unofficial implementation. The final result may differ from the original generated images in the [paper](https://arxiv.org/abs/2204.06125). You can also try the model in [google colab](https://colab.research.google.com/github/LAION-AI/dalle2-laion/blob/main/notebooks/dalle2_laion_alpha.ipynb). + +## Transfer pytroch model to oneflow. +It's easy for user to tansfer a pytorch model into oneflow, since most of oneflow's api is consistent with pytorch. First we change `import torch` to `import oneflow as flow`, and then we can replace all `torch` in the code to `flow`. If the model can work correctly in the originally +pytorch codes, it's likely to be able to work correctly in oneflow. Sometimes the program may raise error like +``` +AttributeError: module 'oneflow' has no attribute 'xxx' +``` +try install the latest version of oneflow which might help, you can find more details [here](https://github.com/Oneflow-Inc/oneflow#install-oneflow). + + + +**1、Download the pytorch DALLE2 model**: + +As show in the [google colab](https://colab.research.google.com/github/LAION-AI/dalle2-laion/blob/main/notebooks/dalle2_laion_alpha.ipynb), we will use the version of 0.15.4, +``` +git clone -b v0.15.4 https://github.com/lucidrains/DALLE2-pytorch.git +``` +the pretrained model weights can be found in huggingface: [the prior weight](https://huggingface.co/zenglishuci/conditioned-prior/resolve/main/vit-l-14/prior_aes_finetune.pth) and [the decoder weight](https://huggingface.co/laion/DALLE2-PyTorch/resolve/main/decoder/1.5B_laion2B/latest.pth). +A simple inference script can be written as +```python +# inference_dalle2.py +import numpy as np +import torch +import os,sys +from dalle2_pytorch import tokenizer +from dalle2_pytorch import OpenAIClipAdapter, DALLE2, DiffusionPriorNetwork, DiffusionPrior, Unet, Decoder + +def generate_images_from_text(texts): + clip=OpenAIClipAdapter("ViT-L-14.pt").to("cuda") + + tokens = tokenizer.tokenize(text).to("cuda") + _, text_encodings, text_mask = clip.embed_text(tokens) + + prior_network = DiffusionPriorNetwork( + dim = 768, + depth = 24, + num_timesteps = 1000, + num_time_embeds = 1, + num_image_embeds=1, + num_text_embeds = 1, + dim_head = 64, + heads = 32, + ff_mult = 4, + attn_dropout = 0.05, + ff_dropout = 0.05, + normformer = True, + ) + + diffusion_prior = DiffusionPrior( + net = prior_network, + clip = clip, + image_embed_dim = 768, + timesteps = 1000, + cond_drop_prob = 0.1, + loss_type="l2", + condition_on_text_encodings = True + ) + state_dict = torch.load("prior_aes_finetune.pth", map_location="cpu")['ema_model'] + diffusion_prior.load_state_dict(state_dict, strict=True) + diffusion_prior.to("cuda") + + image_embed = diffusion_prior.sample(tokens, num_samples_per_batch = 2, cond_scale = 1.) + + unet = Unet( + dim = 320, + image_embed_dim = 768, + text_embed_dim = 768, + cond_dim = 512, + channels = 3, + dim_mults=(1, 2, 3, 4), + num_resnet_blocks = 4, + attn_heads = 8, + attn_dim_head = 64, + sparse_attn = True, + memory_efficient = True, + cond_on_text_encodings = True, # set to True for any unets that need to be conditioned on text encodings + self_attn = [False, True, True, True] + ) + + decoder = Decoder( + unet = (unet,), + image_sizes = [64], + clip = clip, + channels = 3, + timesteps = 1000, + loss_type = "l2", + beta_schedule = ["cosine"], + learned_variance = True + ) + state_dict = torch.load("latest.pth", map_location = "cpu") + + new_dict = {} + for k,v in state_dict.items(): + if 'clip.' in k: continue + if ('cross_attn' in k or 'fn.fn.' in k) and k.endswith(".g"): + k = k[:-1] + "gamma" + new_dict[k] = v + assert k in decoder.state_dict().keys(), k + decoder.load_state_dict(new_dict, strict=False) + decoder.to("cuda") + images = decoder.sample(image_embed = image_embed, text_encodings = text_encodings, text_mask = text_mask, cond_scale = 3.5) + return images + +def save_images(images): + import torchvision.transforms as T + to_pil = T.ToPILImage() + images = list(map(to_pil,images.unbind(dim = 0))) + for i,image in enumerate(images): + image.save(f"./result_{i}.png") + +def main(): + text = ["a dolphin in an astronaut suit on saturn, artstation"] + images = gen_text_and_img_emb(text) + save_images(images) + +if __name__ == "__main__": + main() +``` +run `python inference_dalle2.py`, this should work. + + + +## 2、Change the deep learning framework to oneflow. +As mentioned above, we replace all the `torch` symbol to `flow` by firstly change `import torch` to `import oneflow as flow` in all python files. +It should be noted that the original pytorch code also import other python packages using pytorch backend like [einops](https://github.com/arogozhnikov/einops)、[einops_ext](https://github.com/lucidrains/einops-exts)、[kornia](https://github.com/kornia/kornia) etc. which should also be modified at the same time. + +Fortunately, only a few api of these packages are used, we can take out the relevant code from the github repos and merge them in a separate file. + +For example, we can simplely create the einops_ext.py file adapted from [here](https://github.com/lucidrains/einops-exts/blob/main/einops_exts/einops_exts.py), then we can import einops_ext from the python file which use oneflow instead of python packages using torch. +```python +# einops_ext.py +import re +from oneflow import nn #here change `from torch improt nn` to `from oneflow import nn` +from functools import wraps, partial + +from einops import rearrange, reduce, repeat +``` + + + +## 3、Using Libai's api. +[LiBai](https://github.com/Oneflow-Inc/libai) is a large-scale open-source model training toolbox based on OneFlow. + +Libai provides many efficient api which can be easily used for distributed training and evaluation. It also supports some popular models under the projects folder such as [CLIP](https://github.com/Oneflow-Inc/libai/tree/main/projects/CLIP). To avoid duplication of work, we directly use the clip model implemented in Libai. The relavant code in the original pytorch code is the `OpenAIClipAdapter` class which can be written as follows: +```python +# _clip.py +import os +import sys +import oneflow as flow +import oneflow.nn.functional as F +from oneflow import nn +from collections import namedtuple + +def import_flow_clip(fn): + + def wrapper(*args, **kwargs): + sys.path.append(os.path.join(os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")), "CLIP")) + fn(*args, **kwargs) + sys.path.pop() + + return wrapper + +class BaseClipAdapter(nn.Module): + pass + +class OpenAIClipAdapter(BaseClipAdapter): + + @import_flow_clip + def __init__( + self, + name = 'ViT-L/14' + ): + import clip + openai_clip, preprocess = clip.load(name) + super().__init__(openai_clip) + +``` + +[DiffusionPrior](https://github.com/lucidrains/DALLE2-pytorch/blob/v0.15.4/dalle2_pytorch/dalle2_pytorch.py#L873) and [Decoder](https://github.com/lucidrains/DALLE2-pytorch/blob/v0.15.4/dalle2_pytorch/dalle2_pytorch.py#L1802) follow their original implementation. + + +**Using libai.layers** + +LiBai provides multiple parallelisms such as Data Parallelism, Tensor Parallelism, and Pipeline Parallelism. To experience these features, we will use libai.layers like Linear and LayerNorm: +```python +from libai.layers import Linear, LayerNorm +``` +the nn.Linear will be replace with `libai.layers.Linear`. + +**Compare the outputs** To make sure it is correctly modified from `torch` to `flow`, it's necessary to compare the outputs to see if they are the same after the change. A notable point here is that in the sampling stage, the noise are randomly generated, like +```python +noise = flow.randn(shape) +# or noise = torch.randn(shape) in torch code +``` +torch and oneflow will generate different numbers here even if they are set the same random seed. An alternate way is to make a transition through numpy: +```python +import numpy as np +np.random.seed(6666) +noise = flow.tensor(np.randn(shape)) +# or noise = torch.tensor(np.randn(shape)) in torch code +``` +When the model is fed the same input text, the output images by oneflow or torch code should be same. + +**LazyConfig and LazyCall** + +Oneflow provides LazyConfig system for more flexible syntax and no predefined structures, find more [here](https://libai.readthedocs.io/en/latest/tutorials/basics/Config_System.html). As for the DALLE2, the config file can be write as +```python +from omegaconf import DictConfig +from libai.config import LazyCall +from dalle2.models import DiffusionPrior, DiffusionPriorNetwork, Unet, Decoder, DALLE2 +from dalle2._clip import OpenAIClipAdapter + +clip = LazyCall(OpenAIClipAdapter)(name="./dalle2/model_weights/ViT-L-14.pt") + +prior = LazyCall(DiffusionPrior)( + net = LazyCall(DiffusionPriorNetwork)( + dim=768, + depth=24, + num_timesteps=1000, + max_text_len=77, + num_time_embeds=1, + num_image_embeds=1, + num_text_embeds=1, + dim_head=64, + heads=32, + ff_mult=4, + attn_dropout=0.05, + ff_dropout=0.05, + normformer=True, + ), + clip=clip, + image_embed_dim=768, + timesteps=1000, + cond_drop_prob=0.1, + loss_type="l2", + condition_on_text_encodings=True +) + +unet1 = LazyCall(Unet)( + dim=320, + image_embed_dim=768, + text_embed_dim=768, + cond_dim=512, + channels=3, + dim_mults=(1, 2, 3, 4), + num_resnet_blocks=4, + attn_heads=8, + attn_dim_head=64, + sparse_attn=True, + memory_efficient=True, + cond_on_text_encodings=True, # set to True for any unets that need to be conditioned on text encodings + self_attn=[False, True, True, True] +) + +decoder = LazyCall(Decoder)( + unet=(unet1,), + image_sizes=[64, ], + clip=None, + channels=3, + timesteps=1000, + loss_type="l2", + beta_schedule=["cosine"], + learned_variance=True +) + +dalle2_model = LazyCall(DALLE2)( + prior=prior, + decoder=decoder, + prior_weight_path='', + decoder_weight_path='' +) +``` + +## 4、Model parallel in Libai. +In order to achieve the model parallel inference under libai, we should set the parallel mode according to your needs. The default value of argument parallel is `data` in libai.layers.Linear, which means data parallel. To achieve model parallel, we need change the parallel to `col` or `row`. The most efficient way is to set the Linear layers in the col -> row -> col order. + +A transformer block contains a attention and a feedforward submodule, and each submodule exactly contains 2 Linear layers. +The attention module contains the qkv projection and out projection. Thus we set the qkv projejction as `col`, and the out projection as `row`: +```python +#attention +class Attention(nn.Module): + def __init__(self, *args, **kwargs): + super().__init__() + # 1、 qkv projection + self.to_q = Linear(dim, inner_dim, bias = False, parallel='col') + self.to_kv = Linear(dim, dim_head * 2, bias = False, parallel='col') + #2、 output projection + self.to_out = nn.Sequential( + Linear(inner_dim, dim, bias = False, parallel='row'), #'row' + LayerNorm(dim) + ) +``` +and feed forward contains in projection and out projection, the former will be set `col` and the later will be set `row`. +```python +def FeedForward( + dim, + mult = 4, + dropout = 0., + post_activation_norm = False +): + inner_dim = int(mult * dim) + return nn.Sequential( + LayerNorm(dim), + Linear(dim, inner_dim * 2, bias = False, parallel='col'), + SwiGLU(), + LayerNorm(inner_dim) if post_activation_norm else nn.Identity(), + nn.Dropout(dropout), + Linear(inner_dim, dim, bias = False, parallel='row') + ) +``` + + +for the single machine with 4 GPUs, the model parallel could be set like: +```python +import libai.utils.distributed as dist +dist.setup_dist_util( + DictConfig( + dict( + data_parallel_size=1, + tensor_parallel_size=4, + pipeline_parallel_size=1, + ) + ) +) +``` + + + +If you successfully complete the above steps, now you can have fun with the (unofficial) dalle2 model. \ No newline at end of file diff --git a/docs/source/notes/assets/architecture.jpg b/docs/source/notes/assets/architecture.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e69284112bb7e7ae21c02b97f5840122b19ea40f Binary files /dev/null and b/docs/source/notes/assets/architecture.jpg differ diff --git a/docs/source/notes/assets/vision_transformer.png b/docs/source/notes/assets/vision_transformer.png new file mode 100644 index 0000000000000000000000000000000000000000..9e217d6683ef06e75dd89efc429b0f952b3a103e Binary files /dev/null and b/docs/source/notes/assets/vision_transformer.png differ diff --git a/docs/source/notes/index.rst b/docs/source/notes/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..e9b88bb90fc736035deb9ce092c3adccfc9a7cc1 --- /dev/null +++ b/docs/source/notes/index.rst @@ -0,0 +1,13 @@ +Notes +============= + +.. toctree:: + :glob: + :maxdepth: 2 + + How_to_use_huggingface's_weights_in_LiBai.md + How_to_build_vision_transformer_model_in_LiBai.md + How_to_load_huggingface's_pretrained_model_in_libai.md + How_to_use_model_parallel_in_LiBai.md + How_to_use_distributed_inference_in_LiBai.md + FAQ.md diff --git a/docs/source/tutorials/advanced_tutorials/customize_dataloader.md b/docs/source/tutorials/advanced_tutorials/customize_dataloader.md new file mode 100644 index 0000000000000000000000000000000000000000..911fece7c8a245f7b32b8809f6e8c7bda5209281 --- /dev/null +++ b/docs/source/tutorials/advanced_tutorials/customize_dataloader.md @@ -0,0 +1,75 @@ +# How to Customize Dataloader + +Dataloader is the component that provides data to models. Dataloader usually (but not necessarily) takes raw information from [write dataloaders](https://libai.readthedocs.io/en/latest/tutorials/basics/Write_Dataloaders.html), and processes them into the format needed by the model. + +## How the Existing Dataloader Works + +LiBai contains a built-in data loading pipeline. It's beneficial to understand how it works, in case you need to write a custom one. + +LiBai provides some functions [build_{image,nlp}_{train,test}_loader](https://libai.readthedocs.io/en/latest/modules/libai.data.html#libai.data.build.build_nlp_train_loader) that create a default dataloader from a given config. Here is how `build_{image,nlp}_{train,test}_loader` work: + +1. It instantiates the `list[flow.utils.Dataset]` (e.g., `BertDataset`) by loading some dataset items with lightweight format. These dataset items are not yet ready to be used by the model (e.g., images are not loaded into memory, random augmentation have not been applied, etc.). + +2. The output format of dataset (`__getitem__(...)`) must be a dict whose keys must be consistent with argument names of the dataloader's consumer (usually the `model.forward(...)`). The role of the process is to transform the lightweight representation of a dataset item into a format that is ready for the model to consume (including, e.g., read images, perform random data augmentation and convert to oneflow Tensors). If you would like to perform custom transformations to data, you often want to rewrite it. Details about the dataset format can be found in [write dataloaders](https://libai.readthedocs.io/en/latest/tutorials/basics/Write_Dataloaders.html). + +3. The outputs of the dataset are simply batched with the following function. + +```python +def trivial_batch_collator(batch): + assert isinstance(batch[0], Instance), "batch[0] must be `instance` for trivial batch collator" + batch = Instance.stack(batch) + return batch +``` + +4. This batched data is the output of the dataloader. Typically, it's also the input of `get_batch`. After `get_batch(...)`, it becomes the input of `model.forward()`. `get_batch` simply changes the local tensors to global tensors with the given `sbp` and `placement` meta information. + + +```python +@classmethod +def get_batch(cls, data, mixup_func = None): + ... + ret_dict = {} + for key, value in data.get_fields().items(): + value.to_global() + ret_dict[key] = value.tensor + return ret_dict +``` + + +## Use Custom Dataloader + +If you use `DefaultTrainer`, you can overwrite its `build_train_loader` method to use your own dataloader which can be implemented with any tools you like. But you need to make sure that each rank is reading the data correctly under different parallelism circumstances. + +Then you need to overwrite `get_batch` method. `data` argument in `get_batch` is the output of your dataloader. You need to change the local tensors to global tensors manually, which means you should set the `sbp` and `placement` correctly. + +Here is an example. Process of rank0 gets all data and redistributes them into the other ranks. + +```python +@classmethod +def get_batch(cls, data, mixup_func=None): + if data is None: + # not rank0, set placeholders for data + # Note: make sure imgs and labels have the same shape and dtype on all ranks + imgs = flow.empty(16, 3, 224, 224, dtype=flow.float32) + labels = flow.empty(16, dtype=flow.int64) + else: + # rank0 + imgs, labels = data + dist.synchronize() + + imgs = imgs.to_global(spb=flow.sbp.broadcast, placement=flow.env.all_device_placement("cuda")) + imgs = imgs.to_global( + spb=dist.get_nd_sbp([flow.sbp.split(0), + flow.sbp.broadcast]), + placement=dist.get_layer_placement(0)) + + labels = labels.to_global(spb=flow.sbp.broadcast, placement=flow.env.all_device_placement("cuda")) + labels = labels.to_global( + spb=dist.get_nd_sbp([flow.sbp.split(0), + flow.sbp.broadcast]), + placement=dist.get_layer_placement(-1)) + return { + "images": imgs, + "labels": labels + } +``` diff --git a/docs/source/tutorials/advanced_tutorials/customize_parallel.md b/docs/source/tutorials/advanced_tutorials/customize_parallel.md new file mode 100644 index 0000000000000000000000000000000000000000..8773e0eecded200fa70bd5c3004c8175d24331cb --- /dev/null +++ b/docs/source/tutorials/advanced_tutorials/customize_parallel.md @@ -0,0 +1,242 @@ +# How to Customize Parallelism + +Common parallelisms have already been implemented in LiBai, such as data parallel, tensor parallel and pipeline parallel. But there is also a need for user customized parallel. In this tutorial, we will show you how to customize your own parallelism. + +## Define your own Parallel Model with LiBai.layers + +### Large-scale FC + +Suppose you have a huge fully-connected-layer for large-scale classification (e.g., 1000w classes), which makes it impossible to fit into a single GPU. + +Don't worry, with the help of `LiBai.layers`, you can write models in a familiar way that you used to write for a single GPU. Here is a simple example showing how to write a tensor-parallel fully-connected-layer with 2 GPUs. + +```python +# huge_fc_example.py +import oneflow as flow +from omegaconf import DictConfig +from oneflow import nn + +from libai.layers import Linear +from libai.utils import distributed as dist + +cfg = DictConfig(dict(data_parallel_size=1, tensor_parallel_size=2, pipeline_parallel_size=1)) +dist.setup_dist_util(cfg) + + +class Huge_FC(nn.Module): + def __init__(self): + super().__init__() + self.fc = Linear(2048, 32768, parallel="col") + + def forward(self, x): + return self.fc(x) + + +huge_fc = Huge_FC() + +x = flow.rand(32, 2048, sbp=flow.sbp.broadcast, placement=flow.placement("cuda", ranks=[0, 1])) +y = huge_fc(x) + +print(f"rank: {flow.env.get_rank()}, tensor shape: {y.to_local().shape}") +``` + +You can run this toy example with command line as follows: + +```shell +python3 -m oneflow.distributed.launch --nproc_per_node 2 huge_fc_example.py + +>> rank: 0, tensor shape: oneflow.Size([32, 16384]) +>> rank: 1, tensor shape: oneflow.Size([32, 16384]) +``` + +In the result, you can find that `y` has been split along with `axis=1` on 2 GPUs. + +### Large MLP models + +Suppose you have a huge MLP model which is very popular in transformer-based models, with a huge hidden size that makes it difficult to fit into a single GPU. + +You can then split the model weights across GPUs in a hybrid parallel mode while you can still write your model in a familiar way. + +Here is a simple example about the 2D parallel MLP in the LiBai context. + +```python +import oneflow as flow +from omegaconf import DictConfig +from oneflow import nn + +from libai.layers import Linear +from libai.utils import distributed as dist + +cfg = DictConfig(dict(data_parallel_size=2, tensor_parallel_size=2, pipeline_parallel_size=1)) +dist.setup_dist_util(cfg) + +# Write a Simple 2D Parallel MLP +class MLP_2D(nn.Module): + def __init__(self): + super().__init__() + self.linear_1 = Linear(in_features=1024, out_features=16384, parallel="col") + self.relu = nn.GELU() + self.linear_2 = Linear(in_features=16384, out_features=1024, parallel="row") + + def forward(self, x): + x = self.linear_1(x) + x = self.relu(x) + x = self.linear_2(x) + return x + +# define a model +mlp = MLP_2D() + +# define input with 2D sbp +x = flow.rand( + 32, + 1024, + sbp=dist.get_nd_sbp([flow.sbp.split(0), flow.sbp.broadcast]), + placement=dist.get_layer_placement(0) +) +y = mlp(x) + +print(f"rank: {flow.env.get_rank()}, tensor shape: {y.to_local().shape}") +``` + +You can run it with command line as follows: + +```shell +python3 -m oneflow.distributed.launch --nproc_per_node 4 huge_mlp_example.py + +>> rank: 2, tensor shape: oneflow.Size([16, 1024]) +>> rank: 3, tensor shape: oneflow.Size([16, 1024]) +>> rank: 1, tensor shape: oneflow.Size([16, 1024]) +>> rank: 0, tensor shape: oneflow.Size([16, 1024]) +``` + +From above, you can see that the data are split into 2 groups for data parallel, and weights are split into 2 groups for tensor model parallel. So this simple example just implements a 2D parallel. + +For your convenience, we provide some prevalent models such as BERT, GPT-2, and ViT in Mode Zoo. Feel free to customize them into different sizes to fit into your special needs. + +## Write your own Pipeline Parallel Model + +This tutorial describes how to use pipeline parallel in your own model. LiBai has two pipeline-parallel modes: naive pipeline parallel and (similar) 1F1B pipeline parallel introduced by [Megatron-LM](https://arxiv.org/abs/1909.08053). + +### Introduction of Naive Pipeline Parallel + +In LiBai, naive pipeline parallel can be implemented by setting layers and parameters `placement`. +You can easily configure their `placement` by `dist.get_layer_placement(idx)`. + +Here is an example for `placement` configuration. + +```python +# set a free tensor placement to first stage +self.pos_embed = nn.Parameter( + flow.zeros( + 1, + num_patches + 1, + embed_dim, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) +) + +# set a Linear placement to last stage +# set it manually +self.head = Linear(embed_dim, num_classes, layer_idx=-1).to_global(placement=dist.get_layer_placement(-1)) +# use `layer_idx` API +self.head = Linear(embed_dim, num_classes, layer_idx=-1) +``` + +After configuring models placement, add the input placement transition across different stages. LiBai sets a `layer_idx` attribute in each `nn.Module`, so you can simply add `to_global` in `forward` to implement input placement transition. + +```python +class MyModule(nn.Module): + def __init__(self, ... *, layer_idx): + ... + self.layer_idx = layer_idx + ... + + def forward(self, hidden_states): + hidden_states = hidden_states.to_global(placement=dist.get_layer_placement(self.layer_idx)) + ... +``` + +After configuring models and data placement, you only need to set the distributed configuration before training. + +```python +# set pipeline stages to 2 +train.dist.pipeline_parallel_size = 2 + +# set model layers for pipeline +train.dist.pipeline_num_layers = hidden_layers +``` + +### Introduction of 1F1B Pipeline Parallel + +First, we will introduce GPipe to you to get a better understanding of pipeline parallelism. In GPipe, when the forward passes of all microbatches finish, the backward passes would be executed (as shown in below). + +![gpipe](../assets/gpipe.png) + +1F1B performs one forward pass followed by one backward pass. Finally, at the end of a batch, complete backward passes for all remaining in-flight microbatches. In general, 1F1B is more efficient than GPipe. + +There are two schedules of 1F1B pipeline: the non-interleaved and the interleaved. The figures are shown below. + +![1f1b](../assets/1f1b.png) + +In LiBai, the non-interleaved schedule is supported currently, and this mode is more memory-efficient than GPipe. + +You only need to set models stage id except that placement configuration in naive pipeline parallel, and stage id can help create stashed buffers for activation. + +This example shows how to configure bert model stage id: + +```python +class BertForPreTraining(nn.Module): + def __init__(self, ...): + ... + def forward(self, ...): + ... + + @staticmethod + def set_pipeline_stage_id(model): + dist_utils = dist.get_dist_util() + + # Set pipeline parallelism stage_id + for module_block in model.modules(): + # module_block.to(nn.Module) can get the original module + if isinstance(module_block.to(nn.Module), BertEmbeddings): + module_block.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(0)) + elif isinstance(module_block.to(nn.Module), BertExtendedAttnMask): + module_block.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(0)) + elif isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(module_block.layer_idx)) + elif isinstance(module_block.to(nn.Module), BertPooler): + module_block.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(-1)) + elif isinstance(module_block.to(nn.Module), BertPreTrainingHeads): + module_block.to(nn.graph.GraphModule).set_stage(dist_utils.get_layer_stage_id(-1)) + + # Set the last layernorm stage id + model.bert.final_layernorm.config.stage_id = dist_utils.get_layer_stage_id(-1) +``` + +In `set_pipeline_stage_id`, `BertEmbeddings` and `BertExtendedAttnMask` are placed in the first stage, then each `TransformerLayer` is uniformly placed in each stages. At last, place `BertPooler` and `BertPreTrainingHeads` in the last stage. But don't forget to place the last `layernorm` in `BertEncoder` which does not belong to any `TransformerLayer` in the last stage. + +After adding the `set_pipeline_stage_id` function in a pre-defined `nn.Module`, `GraphBase` will invoke it automatically as below: + +```python +def set_pipeline_stage_id(self): + if hasattr(type(self.model.to(nn.Module)), "set_pipeline_stage_id"): + type(self.model.to(nn.Module)).set_pipeline_stage_id(self.model) +``` + +The last thing left is to set the training configuration as below: + +```python +# set pipeline stages to 2 +train.dist.pipeline_parallel_size = 2 + +# set model layers for pipeline +train.dist.pipeline_num_layers = hidden_layers + +# enable activation checkpointing +train.activation_checkpoint.enabled = True + +# enable gradient accumulation with 8 micro-batches +train.num_accumulation_steps = 8 +``` diff --git a/docs/source/tutorials/advanced_tutorials/index.rst b/docs/source/tutorials/advanced_tutorials/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..a96b411ee7e4e07f05c68f3b5c3d9ad380a2fb00 --- /dev/null +++ b/docs/source/tutorials/advanced_tutorials/index.rst @@ -0,0 +1,9 @@ +Advanced Tutorials +=================== + +.. toctree:: + :glob: + :maxdepth: 2 + + customize_dataloader.md + customize_parallel.md diff --git a/docs/source/tutorials/assets/1f1b.png b/docs/source/tutorials/assets/1f1b.png new file mode 100644 index 0000000000000000000000000000000000000000..7a29c8b2986b7b3fb6c578f918c8bcefcaa19f67 Binary files /dev/null and b/docs/source/tutorials/assets/1f1b.png differ diff --git a/docs/source/tutorials/assets/LiBai_Wechat.png b/docs/source/tutorials/assets/LiBai_Wechat.png new file mode 100644 index 0000000000000000000000000000000000000000..e78727e8bec0663ad652af91bb0bb7f21e392ddc Binary files /dev/null and b/docs/source/tutorials/assets/LiBai_Wechat.png differ diff --git a/docs/source/tutorials/assets/gpipe.png b/docs/source/tutorials/assets/gpipe.png new file mode 100644 index 0000000000000000000000000000000000000000..0408fd69f6dd3860f72f57ae8edf92c4bd0e5b86 Binary files /dev/null and b/docs/source/tutorials/assets/gpipe.png differ diff --git a/docs/source/tutorials/basics/Auto_Parallel.md b/docs/source/tutorials/basics/Auto_Parallel.md new file mode 100644 index 0000000000000000000000000000000000000000..579de8907401226251ca0df07fea3c8e28444992 --- /dev/null +++ b/docs/source/tutorials/basics/Auto_Parallel.md @@ -0,0 +1,74 @@ +# Auto Parallel Training + +LiBai supports **auto-parallel training** which means LiBai will automatically find **an efficient parallel training strategy** for a specific model during training. Users can try out auto-parallel training by the following steps. + +## Installation + +Install OneFlow nightly + +```shell +python3 -m pip install --pre oneflow -f https://staging.oneflow.info/branch/master/[PLATFORM] +``` + +- All available `[PLATFORM]`: + + + + + + + + + + + + + + + + + + + + + + + + +
Platform CUDA Driver VersionSupported GPUs
cu112 >= 450.80.02 GTX 10xx, RTX 20xx, A100, RTX 30xx
cu102 >= 440.33 GTX 10xx, RTX 20xx
cpu N/A N/A
+ + +## Train/Evaluate model in auto-parallel mode + +You can train your own model in auto-parallel mode by simply updating the config as follows: + +### Modify config file + +```python +# your config +from .common.models.graph import graph + +graph.auto_parallel.enabled = True +``` +Training model with auto-parallel on 4 GPUs: +```shell +bash ./tools/train.sh tools/train_net.py configs/your_own_config.py 4 +``` + +### Directly modify the training command line + +- auto-parallel training: + +```shell +bash ./tools/train.sh tools/train_net.py configs/your_own_config.py 4 graph.auto_parallel.enabled=True +``` + +- auto-parallel evaluation: + +```shell +bash ./tools/train.sh tools/train_net.py configs/your_own_config.py 4 --eval graph.auto_parallel.enabled=True +``` + +### More details with instructions and interface + +See [OneFlow Auto-Parallelism](https://oneflow.readthedocs.io/en/master/auto_parallel.html). diff --git a/docs/source/tutorials/basics/Build_New_Project_on_LiBai.md b/docs/source/tutorials/basics/Build_New_Project_on_LiBai.md new file mode 100644 index 0000000000000000000000000000000000000000..48ed9a9ad009785e11e79792ced533df7e70f0f9 --- /dev/null +++ b/docs/source/tutorials/basics/Build_New_Project_on_LiBai.md @@ -0,0 +1,87 @@ +# Build New Project on LiBai + +This is a basic guide to build new projects based on LiBai. The advantages of using LiBai to start a new project (such as paper reproduction and finetune task) are as follows: + +- Avoid redundant work. Developers can directly inherit many built-in modules from LiBai. +- Easily reproduce the experiments already run, because LiBai will save the configuration file automatically. +- Automatically output useful information during training time, such as remaining training time, current iter, throughput, loss information and current learning rate, etc. +- Set a few config params to enjoy distributed training techniques. + +## Introduction +Take the [Bert Finetune](https://github.com/Oneflow-Inc/libai/tree/main/projects/QQP) task as an example to introduce LiBai. + +The complete file structure of the project is: + +``` +projects/my_project +├── configs +│ └── config_custom.py +│ └── ... +├── dataset +│ ├── custom_dataset.py +│ └── ... +├── modeling +│ ├── custom_model.py +│ └── ... +├── README.md +``` + +To start a new project based on LiBai step by step: + +Step 1. Prepare an independent config file (such as [config.py](https://github.com/Oneflow-Inc/libai/blob/main/projects/QQP/configs/config_qqp.py)) which contains: + +- The relevant parameters of the task. +- The pre-defined related Class, such as `Model`, `Optimizer`, `Scheduler`, `Dataset`. +- You can inherit the default config in `configs/common` and rewrite it, which can greatly reduce the workload. +- Related class defined with LazyCall which returns a dict instead of calling the object. + +Step 2. Prepare a model file (such as [model.py](https://github.com/Oneflow-Inc/libai/blob/main/projects/QQP/modeling/model.py)) : +- Build related models in this file. The construction method is similar to OneFlow. +- Because Libai will set up a static diagram by default, the calculation of loss needs to be inside the model. +- The function `forward` must return a dict. +- When defining a tensor in the model, you need to use `to_global`. Turn tensor into a global pattern. +- When defining layers, you can import them directly from `libai.layers`, because it has already pre-defined the SBP signature. + +Step 3. Prepare a dataset file (such as [dataset.py](https://github.com/Oneflow-Inc/libai/tree/main/projects/QQP/dataset)) : +- Build `Dataset` in this file. The construction method is similar to OneFlow. +- The difference is that you need to use `DistTensorData` and `Instance`. +- The shape of each batch must be global. +- In `__getitem__` function, the `key` returned by the method must be consistent with the parameter name of the `forward` function in the `model`. + + +## Main Function Entry +[tools/train_net.py](https://github.com/Oneflow-Inc/libai/blob/main/tools/train_net.py) is the default main function entry provided in LiBai. + + +## Build Config +The `config.py` in LiBai is special, which takes the form of lazyconfig and will be saved as `.yaml` at runtime. The config has several necessary fields, such as `train`, `model`, `optim`, `lr_scheduler`, `graph`. For more information, please refer to [Config_System.md](https://libai.readthedocs.io/en/latest/tutorials/Config_System.html). + +> All imported modules must take LiBai as the root directory. Otherwise, the saved `yaml` file cannot save the correct path of the module, resulting in an error when reading `yaml`, and the experiment cannot be reproduced. + +After building the `config.py`, if you want to get the corresponding fields in the project, you need to access like `cfg.my_cfg.***`. + +## Start Training +The `train.sh` file contains some parameters, such as `GPUS`, `NODE`, etc. + +```bash +#!/usr/bin/env bash +FILE=$1 +CONFIG=$2 +GPUS=$3 +NODE=${NODE:-1} +NODE_RANK=${NODE_RANK:-0} +ADDR=${ADDR:-127.0.0.1} +PORT=${PORT:-12345} + +python3 -m oneflow.distributed.launch \ +--nproc_per_node $GPUS --nnodes $NODE --node_rank $NODE_RANK --master_addr $ADDR --master_port $PORT \ +$FILE --config-file $CONFIG ${@:4} +``` + +After building the above modules, you can start training with single GPU. + +> Config can support both `py` files and generated `yaml` files. + +```bash +bash projects/my_projects/train.sh tools/train_net.py projects/my_projects/config.py 1 +``` diff --git a/docs/source/tutorials/basics/Config_System.md b/docs/source/tutorials/basics/Config_System.md new file mode 100644 index 0000000000000000000000000000000000000000..f7531f12c3af53077e38de88df7f1ad3da76cd38 --- /dev/null +++ b/docs/source/tutorials/basics/Config_System.md @@ -0,0 +1,395 @@ +# Config System + +Given that the traditional yacs-based config system or python argparse command-line options suffer from providing enough flexibility for the development of new project, we borrowed the [lazy config system](https://detectron2.readthedocs.io/en/latest/tutorials/lazyconfigs.html) design from detectron2 which forms the non-intrusive config system for LiBai. + +You can refer to the [d2 tutorial](https://detectron2.readthedocs.io/en/latest/tutorials/lazyconfigs.html) for more details about the syntax and basic usage of lazy config. This section shows some examples of usage in LiBai. + +## Configs in LiBai + +LiBai defines a standard set of config namespaces for later use. This set of namespaces must be kept if you want to perform the complete training and evaluation process of LiBai. + +In summary, this set of namespaces is `model, graph, train, optim, dataloader, tokenization(optional)`. The details are as follows. + +### model + +This is the configuration for model definition. You can refer to `configs/common/models` for more examples. + +A model config file can be loaded like this: + +```python +# bert.py: +from libai.config import LazyCall +from libai.models import BertModel + +# define a model with lazycall +bert_model = LazyCall(BertModel)( + vocab_size=30522, + hidden_size=768, + hidden_layers=24, + num_attention_heads=12, + intermediate_size=4096, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + num_tokentypes=2, + add_pooling_layer=True, + initializer_range=0.02, + layernorm_eps=1e-5, + bias_gelu_fusion=True, + bias_dropout_fusion=True, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=True, + add_binary_head=True, + amp_enabled=False, +) + +# my_config.py: +from bert import bert_model as model +assert model.hidden_size == 768 +model.hidden_layers = 12 # change hidden layers +``` + +After defining the model config in a python file, you can `import` it in the global scope of the config file. Note that you need to rename it as `model` regardless of the name used in the model config. + +You can access and change all keys in the model config after import. + +### graph + +This is the configuration for static `nn.Graph` mode. For more information about the static graph mode, refer to the official [nn.Graph docs](https://docs.oneflow.org/master/basics/08_nn_graph.html). + +LiBai has already defined a `GraphBase` class for almost all models to use. You can simply turn on this option to convert eager mode to graph mode. + +The graph config can be found in [graph.py](https://github.com/Oneflow-Inc/libai/blob/main/configs/common/models/graph.py), and two useful options are shown as follows: + +```python +# Turn on graph mode, if set to `False`, will use eager mode. +graph.enabled = True + +# Set graph debug level, -1 means no debug info, and 0,1,2,3 can be +# set for different debug levels. +# More information can be found in nn.Graph documents. +graph.debug = -1 +``` + +### train + +This is the configuration for training and evaluation. The default training config can be found in `configs/common/train.py`. + +The convention of training / test specific parameters is as follows: + +```python +from libai.config import LazyCall + +train = dict( + + # Directory where output files are written + output_dir="./output", + + # `train_micro_batch_size` is number of samples per batch on each GPU. + # train_mini_batch_size = train_micro_batch_size * num_accumulation_steps. + # This is also the number of training samples per step (i.e. per iteration). + + # If we use 8 GPUs for data parallel groups, `train_micro_batch_size = 2` and + # `num_accumulation_steps = 4`, then each GPU will see 2 samples per batch and + # 8 samples per iteration. + # Total 64 samples will be trained per iteration across all GPUs. + + # global_batch_size = micro_batch_size * num_grad_acc * data_parallel_groups + train_micro_batch_size=32, + global_batch_size=None, + num_accumulation_steps=None, + + # The total training iterations + train_iter=10000, + # The total training epochs, will be scaled to training iterations automatically. + # The actual total training iterations will be calculated by the + # formula `max(train_iter, train_epoch * iter_per_epoch)`. + train_epoch=0, + consumed_train_samples=0, + consumed_valid_samples=0, + train_samples=None, + + # Fraction of lr-warmup-iters to use for warmup (as a float) + warmup_ratio=0, + + # The start iteration, usually needn't set it manually. + # It can be computed automatically when resuming training. + start_iter=0, + + # Enable automatic mixed precision for training which does not + # change model's inference behavior. + amp=dict(enabled=False), + + # Enable activation checkpointing to allow for training + # with larger models, sequences, and batch sizes. + # If enabled, checkpoint the input activations of each transformer layers by default. + activation_checkpoint=dict(enabled=False), + + # NCCL fusion threshold megabytes, set to 0 to + # compatible with previous version of OneFlow. + nccl_fusion_threshold_mb=16, + + # Maximum number of ops of NCCL fusion, set to 0 to + # compatible with previous version of OneFlow. + nccl_fusion_max_ops=24, + + # Enable ZeRO Optimization to allow for training with larger models. + # This optimization will reduce optimizer stages memory consumption + # as described in ZeRO https://arxiv.org/abs/1910.02054. + zero_optimization=dict( + enabled=False, + stage=1, + ), + + # Save a model checkpoint after every this number of iterations, + # and maximum number of checkpoint will be kept. + checkpointer=dict(period=5000, max_to_keep=100), + + # Options for evaluation + + # `test_micro_batch_size` is number of samples per batch on each GPU for testing. + # If we use 8 GPUs for data parallel groups and `test_micro_batch_size = 2`, then + # total 16 samples will be used per iteration across all GPUs. + test_micro_batch_size=32, + + # Enabled evaluation during training, after every `eval_period` number of iterations + # will perform the evaluation process. + # You can set the maximum evaluation iterations to run for validation/test. + # You can also set a customized evaluator for use. + evaluation=dict( + enabled=True, + # evaluator for calculating top-k acc + evaluator=LazyCall(ClsEvaluator)(topk=(1, 5)), + eval_period=5000, + eval_iter=1e9, # running steps for validation/test + + # Metrics to be used for best model checkpoint. + eval_metric="Acc@1", + eval_mode="max", + ), + + # Path to a checkpoint file to be loaded to the model for training or evaluation. + load_weight="", + + # Output log to console after every this number of iterations. + log_period=20, + + # lr_scheduler arguments + # See libai/scheduler/lr_scheduler.py for definition. + scheduler=LazyCall(WarmupCosineLR)( + # In DefaultTrainer we will automatically set `max_iter` + # and `warmup_iter` by the given train cfg. + warmup_factor=0.001, + alpha=0.01, + warmup_method="linear", + ), + + # Distributed arguments + # See https://libai.readthedocs.io/en/latest/tutorials/basics/Distributed_Configuration.html for more details. + dist=dict( + data_parallel_size=1, + tensor_parallel_size=1, + pipeline_parallel_size=1, + # users must set the `pipeline_num_layers` attribute when `pipeline_parallel_size > 1` + pipeline_num_layers=None, + # users could customize the number of layers in different stages + # by setting the `custom_pipeline_stage_id ` attribute which is used for + # manually balance calculation between stages when running pipeline parallelism + # e.g. you can set `custom_pipeline_stage_id=[0, 0, 0, 1]` + # for `pipeline_num_layers=4 and pipeline_parallel_size=2` + # which means the first 3 layers will be placed on stage0 and + # the last layer will be placed on stage1 + # NOTE: if it is None, LiBai will automatically set pipeline_stage_id + # `auto_pipeline_stage_id` and `actual_pipeline_stage_id` will be saved in `config.yaml` + custom_pipeline_stage_id=None, + ), + + # the device type of input tensors for model, defaults to "cuda". + # if you want to accelerate the model training when pipeline_parallel > 1 + # you can set `input_placement_device="cpu"` then call input_tensor.to_global() + # inside your model.forward() method + # see `libai/models/bert_model.py` as reference + input_placement_device="cuda", + + # set to `True` to enable rdma for improving speed of pipeline_parallel + rdma_enabled=True, + + # Set seed to positive to use a fixed seed. Note that a fixed seed increases + # reproducibility but does not guarantee fully deterministic behavior. + # Disabling all parallelism further increases reproducibility. + seed=1234, +) +``` +**Note:** ``warmup_ratio`` is the ratio of warmup iterations of the total training iterations, and the real ``warmup iterations`` will be calculated by ``wramup_ratio * train_iter`` automatically. + +**Example:** If you need to train 300 epochs with 5 warmup epochs, update the config as follows: +```python +# config.py +train.train_epoch = 300 +train.warmup_ratio = 5 / 300 +``` +If you need to train 1000 iters with 200 warmup iters, set the training config like this: +```python +# config.py +train.train_iter = 1000 +train.warmup_ratio = 200 / 1000 +``` + + +### optim + +This is the configuration for optimizer. The default configuration can be found in `configs/common/optim.py`. + +LiBai utilizes the function `get_default_optimizer_params`, which needs the `nn.Module` as the argument and returns the parameter groups. + +With `LazyConfig`, you can set other arguments in advance and pass the `model` argument later. For more details, refer to [API docs of libai optim](../libai.optim.html#libai.optim.get_default_optimizer_params). + +```python +# optim.py: +import oneflow as flow +from libai.config import LazyCall +from libai.optim import get_default_optimizer_params + +optim = LazyCall(flow.optim.AdamW)( + params=LazyCall(get_default_optimizer_params)( + # params.model is meant to be set to the model object, + # before instantiating the optimizer. + clip_grad_max_norm=1.0, + clip_grad_norm_type=2.0, + weight_decay_norm=0.0, + weight_decay_bias=0.0, + ), + lr=1e-4, + weight_decay=0.01, + betas=(0.9, 0.999), + eps=1e-8, + do_bias_correction=True, +) + +# my_config.py: +import oneflow as flow +optim._target_ = flow.optim.SGD + +# Remove the incompatible arguments in optim +del optim.do_bias_correction + +# Set the need arguments +optim.momentum = 0.9 +``` + +### dataloader + +This is the configuration for dataset/dataloader. This component provides data to the model. A dataloader usually takes raw information and processes it into the format required by the model. + +See example datasets in `configs/common/data/`, including `cifar100`, `imagenet`, `bert_dataset` and so on. You can also define your customized dataset config as you like. + +Take `bert_dataset.py` as an example: + +```python +# bert_dataset.py: +from libai.config import LazyCall +from omegaconf import OmegaConf +from libai.data import build_nlp_test_loader, build_nlp_train_val_test_loader +from libai.data.datasets import BertDataset +from libai.data.data_utils import get_indexed_dataset + +dataloader = OmegaConf.create() + +dataloader.train = LazyCall(build_nlp_train_val_test_loader)( + dataset=[ + LazyCall(BertDataset)( + data_prefix="/your/data_prefix/path", + indexed_dataset=LazyCall(get_indexed_dataset)( + data_prefix="/your/data_prefix/path", + data_impl="mmap", + skip_warmup=False, + ), + max_seq_length=512, + mask_lm_prob=0.15, + short_seq_prob=0.1, + ), + ], + splits=[[949.0, 50.0, 1.0]], + weights=[1.0], + num_workers=4, +) + +# my_config.py: +dataloader.train.dataset[0].max_seq_length = 256 +dataloader.train.num_workers = 2 +``` + +LiBai provides two functions `build_nlp_train_val_test_loader` and `build_image_train_loader` to create a default train data loader from a given config. It takes the list of `dataset_class`(e.g., `BertDataset`) and combines them using `flow.utils.data.dataset.ConcatDataset`. + +It is recommended to check out [API docs of libai.data](../libai.data.html#libai.data.build.build_nlp_train_loader) to learn more about the APIs of `build_nlp_train_val_test_loader`. + +### tokenization (optional) + +You need to configure a tokenizer if you want to train a NLP task. Each NLP dataset has its own tokenizer config in the corresponding data config file. + +Here we use: + +```python +# bert_dataset.py: +from libai.config import LazyCall +from omegaconf import OmegaConf +from libai.tokenizer import BertTokenizer + +tokenization = OmegaConf.create() + +tokenization.tokenizer = LazyCall(BertTokenizer)( + vocab_file="bert-base-chinese-vocab.txt", + do_lower_case=True, + do_chinese_wwm=True, +) +tokenization.append_eod = False +tokenization.make_vocab_size_divisible_by = 128 + +# my_config.py: +tokenization.tokenizer.do_lower_case = False +``` + +Tokenization config must contain a tokenizer(e.g., `BertTokenizer`). `append_eod` and `make_vocab_size_divisible_by` are not necessary. + +`make_vocab_size_divisible_by` is used for padding the vocab size to be divisible by this value. This is added for computational efficiency for tensor parallelism. + +## Get the Default Config + +You don't need to rewrite all contents in config every time. You can import a config file as a python file or use function [`get_config`](../libai.config.html#libai.config.get_config). + +If you build LiBai from source, you can get all default config files in `configs/*`. Then you can import the config files as follows: + +```python +# import config +from .common.models.bert import pretrain_model as model +from .common.models.graph import graph +from .common.train import train +from .common.optim import optim +from .common.data.bert_dataset import dataloader, tokenization + +# modify it +train.train_iter = 100 +... +``` + +If you install LiBai by `pip`, you can use `get_config` function to get all default config files as follows: + +```python +from libai.config import get_config +# get config +model = get_config("common/models/bert.py").pretrain_model +graph = get_config("common/models/graph.py").graph +train = get_config("common/train.py").train +optim = get_config("common/optim.py").optim +dataloader = get_config("common/data/bert_dataset.py").dataloader +tokenization = get_config("common/data/bert_dataset.py").tokenization + +# modify it +train.train_iter = 100 +... +``` + +## LazyConfig Best Practices + +1. Treat the configs you write as actual "code": Avoid copying them or duplicating them. Import the common parts between configs. +2. Keep the configs you write simple: Don't include keys that do not affect the experimental setting. diff --git a/docs/source/tutorials/basics/Distributed_Configuration.md b/docs/source/tutorials/basics/Distributed_Configuration.md new file mode 100644 index 0000000000000000000000000000000000000000..ff4835d8e4014dbe32537c9b1a43e40242b95de0 --- /dev/null +++ b/docs/source/tutorials/basics/Distributed_Configuration.md @@ -0,0 +1,163 @@ +# Distributed Configuration + +In LiBai, you can try out different parallel strategies by simply changing the distributed config in [training config file](https://github.com/Oneflow-Inc/libai/blob/main/configs/common/train.py). +```python +# Distributed arguments +dist=dict( + data_parallel_size=1, + tensor_parallel_size=1, + pipeline_parallel_size=1, + + # users must set the `pipeline_num_layers` attribute when `pipeline_parallel_size > 1` + pipeline_num_layers=None, + # users could customize the number of layers in different stages + # by setting the `custom_pipeline_stage_id ` attribute which is used for + # manually balance calculation between stages when running pipeline parallelism + # e.g. you can set `custom_pipeline_stage_id=[0, 0, 0, 1]` + # for `pipeline_num_layers=4 and pipeline_parallel_size=2` + # which means the first 3 layers will be placed on stage0 and + # the last layer will be placed on stage1 + # NOTE: if it is None, LiBai will automatically set pipeline_stage_id + # `auto_pipeline_stage_id` and `actual_pipeline_stage_id` will be saved in `config.yaml` + custom_pipeline_stage_id=None, +) +``` +For example, you can set `data_parallel_size=2` which will automatically split the input data into two groups for data parallel training. + +## Distributed Setting Example +Here are some simple examples for you to understand the basic configuration of LiBai's distributed settings. LiBai's **BERT** model supports three parallelism techniques (**data parallel training**, **tensor parallel training** and **pipeline parallel training**). Take 1 node with 8 GPUs as an example. If you do not change any default settings, LiBai will execute **data parallel training** by default. You can try out different combinations of parallelism training techniques by updating [bert config file](https://github.com/Oneflow-Inc/libai/blob/main/configs/bert_large_pretrain.py) as follows: + +#### **Pure Data Parallel Training on 8 GPUs** + +In this example, the model is replicated on 8 GPUs, and each replica handles only part of the input data during iteration. +```python +from .common.train import train +... + +train.dist.data_parallel_size = 8 +``` + +#### **Pure Tensor Parallel Training on 8 GPUs** + +In this example, the weight of the layers in the model will be split into 8 parts for tensor parallel training on 8 GPUs. +```python +from .common.train import train +... + +train.dist.tensor_parallel_size = 8 +``` + +**Note:** This only works for models built with ``libai.layers``. + +#### **Pure Pipeline Parallel Training on 8 GPUs** + +In this example, 8 GPUs will be split into 8 stages, and different layers of the model will be put on different stages automatically for pipeline parallel training. +```python +from .common.train import train +... + +train.dist.pipeline_parallel_size = 8 + +train.dist.pipeline_num_layers = model.cfg.hidden_layers +``` + +**Note:** +- `train.dist.pipeline_num_layers` must be set consistent with the model layers. If unset, it will use the default value `1000`, +which might trigger unexpected behavior. + +- For models which have been configured with pipeline parallelism(e.g., BERT, GPT-2, T5 and ViT), you can simply update the distributed config to execute pipeline parallel training on them. If you need to train your own model with pipeline parallel strategy, please refer to [Write Models](https://libai.readthedocs.io/en/latest/tutorials/basics/Write_Models.html) for more details about configuring your own model with pipeline parallelism. + +#### **Data Parallel + Tensor Parallel for 2D Parallel Training on 8 GPUs** + +In this example, 8 GPUs will be split into **2 groups**, and each group contains **4 GPUs**. The input data will be split into 2 parts by chunking in the batch dimension for data parallel training between 2 groups. The model is replicated between **2 data-parellel groups**. Within each group, the weight of each layers will be splited across **4 GPUs** for tensor parallel training. + +```python +from .common.train import train +... + +train.dist.data_parallel_size = 2 +train.dist.tensor_parallel_size = 4 +``` +Here we provide a specific example for you to understand this. We number 8 GPUs from 0 to 7, e.g., ``[0, 1, 2, 3, 4, 5, 6, 7]``, and for ``data parallel + tensor parallel``, 8 GPUs will be split into 2 groups as ``[[0, 1, 2, 3], [4, 5, 6, 7]]``, ``GPU: [0, 1, 2, 3]`` as group-0 and ``GPU: [4, 5, 6, 7]`` as group-1. The model is replicated between group-0 and group-1. In group-0, the model will execute tensor parallel between ``GPU: [0, 1, 2, 3]``. In group-1, the model will execute tensor parallel between ``GPU: [4, 5, 6, 7]``, and each group only handle a portion of the input data for data parallel training. + +#### **Data Parallel + Pipeline Parallel for 2D Parallel Training on 8 GPUs** + +In this example, 8 GPUs will be split into **4 stages**. Each stage contains **2 GPUs** which will be split into **2 data-parallel groups**. Each stage only contains a portion of the model. The weight of the layers put on the specific stage is replicated on **2 data-parallel groups**. Each group handles a portion of the input data. +```python +from .common.train import train +... + +train.dist.data_parallel_size = 2 +train.dist.pipeline_parallel_size = 4 + +train.dist.pipeline_num_layers = model.cfg.hidden_layers +``` + +#### **Tensor Parallel + Pipeline Parallel for 2D Parallel Training on 8 GPUs** + +In this example, 8 GPUs will be split into **4 stages**, each stage contains **2 GPUs** as a **group**. And different layers in the model will be put on different stages automatically for pipeline parallel training. The weight of the layers put on the specific stage will be split into 2 parts for tensor parallel training within the group. + +```python +from .common.train import train +... + +train.dist.tensor_parallel_size = 2 +train.dist.pipeline_parallel_size = 4 + +train.dist.pipeline_num_layers = model.cfg.hidden_layers +``` + +#### **Data Parallel + Tensor Parallel + Pipeline Parallel for 3D Parallel Training on 8 GPUs** + +In this example, 8 GPUs will also be split into **2 stages**, and different layers in the model will be put on different stages for pipeline parallel training. Each stage only contains a portion of the whole model, and each stage will be split into **2 groups**. In each stage, the model will be replicated between **2 data-parallel groups**, and each **data-parallel group** contains **2 GPUs**. The input data will be split into 2 parts by chunking in the batch dimension for data-parallel training between **2 data-parallel groups**. Within each group, the weight of each layer will be split across **2 GPUs** for tensor parallel training. + +```python +from .common.train import train +... + +train.dist.data_parallel_size = 2 +train.dist.tensor_parallel_size = 2 +train.dist.pipeline_parallel_size = 2 + +train.dist.pipeline_num_layers = model.cfg.hidden_layers +``` + + +**Note:** `train.dist.data_parallel_size` will be automatically calculated by `(gpu_nums / (tensor_parallel_size * pipeline_parallel_size))` if only `train.dist.tensor_parallel_size` and `train.dist.pipeline_parallel_size` are set. For example: + +```python +from .common.train import train +... +# only set tensor_parallel_size and pipeline_parallel_size +train.dist.tensor_parallel_size = 2 +train.dist.pipeline_parallel_size = 2 + +train.dist.pipeline_num_layers = model.cfg.hidden_layers +``` +And the `data_parallel_size` will be automatically set to `(8 / (2 * 2)) = 2` + + +#### **Set `custom_pipeline_stage_id` for Load Balance** +In most cases, the transformer layers of common models have the same computational overhead, so there is no need to set `custom_pipeline_stage_id`. + +But when transformer layers have unbalanced computational overhead, you can set `custom_pipeline_stage_id` for manually balance the compuation between stages in pipeline_parallelism + +For example: +```python +train.dist.pipeline_parallel_size = 4 +train.dist.pipeline_num_layers = 24 +train.dist.custom_pipeline_stage_id = [0]*6 + [1]*7 + [2]*7 + [3]*4 +``` +It means you have `[6, 7, 7, 4]` layers separately located in `stage0`~`stage3`. +Modify `custom_pipeline_stage_id` according to your own needs. + +## Update Distributed Config with Command Line +You can also control the parallelization strategy by **command line** parameters as follows: + +```bash +bash tools/train.sh tools/train_net.py configs/bert_large_pretrain.py \ +8 \ # num of gpus +train.dist.data_parallel_size=2 \ +train.dist.tensor_parallel_size=2 \ +train.dist.pipeline_parallel_size=2 \ +``` \ No newline at end of file diff --git a/docs/source/tutorials/basics/Evaluation.md b/docs/source/tutorials/basics/Evaluation.md new file mode 100644 index 0000000000000000000000000000000000000000..d3c60ee0da6ec4eab73da47c428e45ee70caff0b --- /dev/null +++ b/docs/source/tutorials/basics/Evaluation.md @@ -0,0 +1,91 @@ +# Evaluation +Evaluation is a process that takes a number of inputs/outputs pairs and calculates them to get metrics. You can always use the model directly and parse its inputs/outputs manually to perform evaluation. Alternatively, evaluation can be implemented in LiBai using the `DatasetEvaluator` interface. + +LiBai includes a few `DatasetEvaluator` that computes metrics like top-N accuracy, PPL(Perplexity), etc. You can also implement your own `DatasetEvaluator` that performs some other jobs using the inputs/outputs pairs. For example, to count how many instances are detected on the validation set: +``` Python +class Counter(DatasetEvaluator): + def reset(self): + self.count = 0 + def process(self, inputs, outputs): + for output in outputs: + self.count += len(output["instances"]) + def evaluate(self): + # save self.count somewhere, or print it, or return it. + return {"count": self.count} +``` + +## Customize Evaluator using DatasetEvaluator +`DatasetEvaluator` is the Base class for a dataset evaluator. This class accumulates information of the inputs/outputs (by `process`) after every batch inference, and produces evaluation results in the end (by `evaluate`). The input is from the `trainer.get_batch()`, which converts the outputs of `dataset.__getitem__()` to dict. The output is from the dict return of `model.forward()`. + +Firstly, declare a new evaluator class that inherits the `DatasetEvaluator` and overwrites its `process` and `evaluation` functions to satisfy the needs. + +For example, declare a `MyEvaluator` class in `libai/evaluator/myevaluator.py`: +``` Python +class MyEvaluator(DatasetEvaluator): + def __init__(self): + self._predictions = [] + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + # the key of inputs/outputs can be customized + pred_logits = outputs["prediction_scores"] + labels = inputs["labels"] + + # measure accuracy + preds = pred_logits.cpu().topk(1)[1].squeeze(1).numpy() + labels = labels.cpu().numpy() + + self._predictions.append({"preds": preds, "labels": labels}) + + def evaluate(self): + correct = 0.0 + all_sample = 0.0 + for pred in self._predictions: + preds = pred["preds"] + labels = pred["labels"] + correct += (preds==labels).sum() + all_sample += len(preds) + self._results = OrderedDict() + self._results["acc"] = correct/all_sample + return copy.deepcopy(self._results) +``` + +Secondly, import the customized class and set the evaluation in config: +``` Python +from libai.evaluation.myevaluator import MyEvaluator +evaluation=dict( + enabled=True, + # evaluator for calculating top-k acc + evaluator=LazyCall(MyEvaluator)(), + eval_period=5000, + eval_iter=1e9, # running steps for validation/test + # Metrics to be used for best model checkpoint. + eval_metric="acc", # your returned metric key in MyEvaluator.evaluate() + eval_mode="max", # set `max` or `min` for saving best model according to your metric +) +``` + +## Run Evaluator Manually +To check your evaluator code outside `LiBai`, use the methods of evaluators manually: +``` Python +def get_all_inputs_outputs(): + for data in data_loader: + yield data, model(data) + +evaluator.reset() +for inputs, outputs in get_all_inputs_outputs(): + evaluator.process(inputs, outputs) +eval_results = evaluator.evaluate() +``` + +Evaluators can also be used with `inference_on_dataset`. For example: +``` Python +eval_results = inference_on_dataset( + model, + data_loader, + evaluator, + ... +) +``` \ No newline at end of file diff --git a/docs/source/tutorials/basics/Features.md b/docs/source/tutorials/basics/Features.md new file mode 100644 index 0000000000000000000000000000000000000000..825b0259add6684b0399345e0a6cd0681f1ebd96 --- /dev/null +++ b/docs/source/tutorials/basics/Features.md @@ -0,0 +1,138 @@ +# Features in LiBai + +LiBai provides many features out of the box. This section shows how to configure them step by step. + +## Automatic Mixed Precision Training + +AMP stands for automatic mixed precision training. To enable AMP in LiBaiYou, add `train.amp.enabled=True` in your configuration file . + +### Usage + +```python +# import config +from .common.train import train + +# get config +from libai.config import get_config +train = get_config("common/train.py").train + +# enable amp +train.amp.enabled = True +# disable amp +train.amp.enabled = False +``` + +## Gradient Clipping + +Gradient clipping is a technique that tackles exploding gradients. The idea of gradient clipping is very simple: the gradient will be rescaled down if it gets too large. + +LiBai supports gradient clipping in a convenient way, and you don't have to implement it by yourself. You just need to add some settings to your configuration file to enable gradient clipping. + +**Note:** We do not recommend writing gradient clipping by yourself, because naive gradient clipping may fail when using tensor parallel or pipeline parallel. + +### Usage + +```python +# import config +from .common.optim import optim + +# get config +from libai.config import get_config +optim = get_config("common/optim.py").optim + +# enable gradient clipping +optim.params.clip_grad_max_norm = 1.0 +optim.params.clip_grad_norm_type = 2.0 + +# disable gradient clipping +optim.params.clip_grad_max_norm = None +optim.params.clip_grad_norm_type = None +``` + +`clip_grad_max_norm` and `clip_grad_norm_type` can be checked in [API docs of oneflow](https://oneflow.readthedocs.io/en/master/nn.html#oneflow.nn.utils.clip_grad_norm_). + +## Gradient Accumulation + +Gradient accumulation is a common strategy to train large-scale models when memory becomes the bottleneck. This technique splits the mini-batch into several micro-batches, then performs normal forward and backward operations. Models will only be updated after accumulating the gradients of all these micro-batches. + +Besides, when training with pipeline parallel, gradient accumulation makes different stages executed in different micro-batch in parallel. Therefore, the calculation of each stage can be overlapped. + +### Usage + +```python +# import config +from .common.train import train + +# get config +from libai.config import get_config +train = get_config("common/train.py").train + +# enable grad accumulation for 4 steps +train.num_accumulation_steps = 4 + +# disable grad accumulation +train.num_accumulation_steps = None +``` + +## Activation Checkpointing + +To reduce GPU memory usage and deploy a large model to a training system, LiBai supports activation checkpointing. LiBai uses a Transformer layer as the unit of checkpointing, because the activation size bloats in the middle of a Transformer layer, so checkpointing the input of a Transformer layer is storage-efficient. + +LiBai supports [activation checkpointing](https://arxiv.org/abs/1604.06174) by `set_activation_checkpoint` in `GraphBase`. So models using `libai.layers.TransformerLayer` support activation checkpointing by default. If you want to set activation checkpointing for customized layers, you need to override this function. + +```python +def set_activation_checkpoint(self): + for module_block in self.model.modules(): + if isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(nn.graph.GraphModule).activation_checkpointing = True +``` + +### Usage + +```python +# import config +from .common.train import train + +# get config +from libai.config import get_config +train = get_config("common/train.py").train + +# enable activation checkpointing +train.activation_checkpoint.enabled = True + +# disable activation checkpointing +train.activation_checkpoint.enabled = False +``` + +## ZeRO + +Unlike normal data parallelism, where model states and gradients are replicated across data-parallel processes, Zero Redundancy Optimizer (ZeRO) partitions them across data-parallel processes, which can reduce memory footprint significantly. + +- Level 1: The optimizer states (e.g., for Adam optimizer, 32-bit weights, and the first and second moment estimates) are partitioned across the processes so that each process will only update its own partition. + +- Level 2: The reduced 32-bit gradients for updating the model weights are also partitioned so that each process retains only the gradients corresponding to its portion of the optimizer states. + +> **Note:** ZeRO only supports data parallel and pipeline parallel, or the combination of them. If you use tensor parallel in your training, make sure ZeRO is disabled. + +### Usage + +```python +# import config +from .common.train import train + +# get config +from libai.config import get_config +train = get_config("common/train.py").train + +# enable zero +train.zero_optimization.enabled = True + +# enable zero for level-1 +train.zero_optimization.stage = 1 + +# enable zero for level-2 +train.zero_optimization.stage = 2 + +# disable zero +train.zero_optimization.enabled = False +``` \ No newline at end of file diff --git a/docs/source/tutorials/basics/Load_and_Save_Checkpoint.md b/docs/source/tutorials/basics/Load_and_Save_Checkpoint.md new file mode 100644 index 0000000000000000000000000000000000000000..312d230951714d7b8f80273f04288a581df73f2b --- /dev/null +++ b/docs/source/tutorials/basics/Load_and_Save_Checkpoint.md @@ -0,0 +1,52 @@ +# Load and Save a Checkpoint in LiBai + +Instead of directly using [`flow.save()`](https://oneflow.readthedocs.io/en/master/oneflow.html?highlight=save#oneflow.save) and [`flow.load()`](https://oneflow.readthedocs.io/en/master/oneflow.html?highlight=oneflow.load#oneflow.load), LiBai provides the [`checkpoint module`](https://libai.readthedocs.io/en/latest/modules/libai.utils.html#module-libai.utils.checkpoint) to deal with the complex situations when saving/loading model. + + +Typically, you don't need to write the relative code to load/save weights trained from LiBai when using LiBai's `DefaultTrainer` and `LazyConfig`. For more details, see [Training & Evaluation in Command Line](https://libai.readthedocs.io/en/latest/tutorials/basics/Train_and_Eval_Command_Line.html) which introduces `weight load` and `resume training` settings in `config.py` or in command line for standard training. + +Here we introduce how to load&save weights according to your custom needs. Suppose you have a model trained with LiBai. + +```shell +# your model directory +output/finetune_qqp +├── config.yaml +├── last_checkpoint +├── log.txt +├── log.txt.rank1 +├── log.txt.rank2 +├── log.txt.rank3 +├── metrics.json +├── model_0000009 +│ ├── graph +│ ├── lr_scheduler +│ └── model +├── model_0000019 +│ ├── graph +│ ├── lr_scheduler +│ └── model +├── model_best +│ ├── graph +│ ├── lr_scheduler +│ └── model +└── model_final + ├── graph + ├── lr_scheduler + └── model +``` + +The following code shows how to load/save model weights: +```python +from libai.utils.checkpoint import Checkpointer +from path.to.your.build_model import build_model + +model = build_model(cfg.model) +# load model weights +Checkpointer(model).load(path_to_model) # path_to_model should be "output/finetune_qqp/model_final" + +# save model weights +checkpointer = Checkpointer(model, save_dir="output/") +checkpointer.save("model_999") # save to output/model_999 +``` + +You can also save other informations (e.g. `optim`, `scheduler`) other than model weights by using `checkpointer`. See [libai.utils.checkpoint](https://libai.readthedocs.io/en/latest/modules/libai.utils.html#module-libai.utils.checkpoint) for more details. \ No newline at end of file diff --git a/docs/source/tutorials/basics/Preprocessing_Dataset.md b/docs/source/tutorials/basics/Preprocessing_Dataset.md new file mode 100644 index 0000000000000000000000000000000000000000..d0ea526a071516991e55473464b46270f046893c --- /dev/null +++ b/docs/source/tutorials/basics/Preprocessing_Dataset.md @@ -0,0 +1,38 @@ +# Preprocessing Dataset + +If you use LiBai's `Dataset` to training NLP model, you can preprocess the training data. + +This tutorial introduces how to preprocess your own training data, let's take training `Bert` as an example. + +First, You need to store the training data in loose JSON format file, which contains one text sample per line, For example: + +```bash +{"chapter": "Chapter One", "text": "April Johnson had been crammed inside an apartment", "type": "April", "background": "novel"} +{"chapter": "Chapter Two", "text": "He couldn't remember their names", "type": "Dominic", "background": "novel"} +``` + +You can set the `--json-keys` argument to select the specific data of per sample, and the other keys will not be used. + +Then, Process the JSON file into a binary format for training. To conver the json into mmap, cached index file, or the lazy loader format use `toos/preprocess_data.py`. Set the `--dataset-impl` flag to `mmap`, `cached`, or `lazy` respectively. You can run the following code to prepare you own dataset for training BERT: + +```bash +#!/bin/bash + +IMPL=lazy +KEYS=text + +python tools/preprocess_data.py \ + --input path/to/test_sample_cn.json \ + --json-keys ${KEYS} \ + --vocab-file path/to/bert-base-chinese-vocab.txt \ + --dataset-impl ${IMPL} \ + --tokenizer-name BertTokenizer \ + --do-lower-case \ + --do-chinese-wwm \ + --split-sentences \ + --output-prefix cn_samples_${IMPL} \ + --workers 1 \ + --log-interval 2 +``` + +Further command line arguments are described in the source file [`preprocess_data.py`](https://github.com/Oneflow-Inc/libai/blob/main/tools/preprocess_data.py). diff --git a/docs/source/tutorials/basics/Train_and_Eval_Command_Line.md b/docs/source/tutorials/basics/Train_and_Eval_Command_Line.md new file mode 100644 index 0000000000000000000000000000000000000000..8401cd5c27b7f998d52754db0bbd4646a32ad87d --- /dev/null +++ b/docs/source/tutorials/basics/Train_and_Eval_Command_Line.md @@ -0,0 +1,95 @@ +# Training & Evaluation in Command Line + +LiBai provides multiple arguments covering a variety of situations. + +## Training + +LiBai provides `tools/train.sh` and `tools/train_net.py` for launching training & eval task. + +You can modify `tools/train_net.py` according to your own needs. + +### Training & Evaluation + +To completely train and test, run: + +```shell +bash tools/train.sh \ +tools/train_net.py \ +path_to_your_config.py \ # config.py for your task +4 # number of gpus +``` + +### Training & Partial Evaluation + +If the evaluation process is time consuming, you can set the parameter `train.evaluation.eval_iter` in your `config.py` to a smaller number such as 20, which can make the evaluation process faster by using only part of the testset. You can also set the parameter by the command line directly : + +**Note:** the eval metric will be calculated by part of testing dataset. + +```shell +bash tools/train.sh \ +tools/train_net.py \ +path_to_your_config.py \ # config.py for your task +4 \ # number of gpus +train.evaluation.eval_iter=20 # set eval_iter for testing +``` + +### Training & No Evaluation + +To train without evaluation, set `train.evaluation.enabled=False` in your `config.py` or in the command line: + +```shell +bash tools/train.sh \ +tools/train_net.py \ +path_to_your_config.py \ # config.py for your task +4 \ # number of gpus +train.evaluation.enabled=False # set no evaluation +``` + +### Resume Training + +To resume training, set `--resume` in the command line, and set `train.output_dir` in your `config.py` or in the command line + +For example, if your training is interrupted unexpectly, and your lastest model path is `output/demo/model_0000019/`, then set `train.output_dir=output/demo` to resume trainig: + +```shell +bash tools/train.sh \ +tools/train_net.py \ +path_to_your_config.py \ # config.py for your task +4 \ # number of gpus +--resume \ # set resume training +train.output_dir=path/task # set resume path, it should be parent directory of model path +``` + + +## Evaluation + +To evaluate your model without training, set `--eval-only` in your command line, and set `train.load_weight`. + +Besides, `train.evaluation.eval_iter=20` will be valid in `--eval-only` if you set it. You can set `eval_iter` according to your own needs. + +```shell +bash tools/train.sh \ +tools/train_net.py \ +path_to_your_config.py \ # config.py for your task +4 \ # number of gpus +--eval-only \ # set eval without train +train.load_weight=path/task/model_final # set model path +``` + +## Quickly check in the respective loop + +To find out whether there are any bugs in your program, pass `--fast-dev-run` to the command line, which will change config settings to: +```python +train.train_epoch = 0 +train.train_iter = 20 +train.evaluation.eval_period = 10 +train.log_period = 1 +``` +Besides, `train.evaluation.eval_iter=20` will be valid in `--fast-dev-run` if you set it. You can set `eval_iter` according to your own needs. +```shell +bash tools/train.sh \ +tools/train_net.py \ +path_to_your_config.py \ # config.py for your task +4 \ # number of gpus +--fast-dev-run # set for quickly check +``` diff --git a/docs/source/tutorials/basics/Training.md b/docs/source/tutorials/basics/Training.md new file mode 100644 index 0000000000000000000000000000000000000000..3e8c3eec50c12d439ccb05c45c9719456e6373e4 --- /dev/null +++ b/docs/source/tutorials/basics/Training.md @@ -0,0 +1,177 @@ +# Training + +To run training, we highly recommend using the standardized `trainer` in LiBai. + +## Trainer Abstraction + +LiBai provides a standardized `trainer` abstraction with a hook system to help simplify the standard training behavior. + +`DefaultTrainer` is initialized from the lazy config system, used by `tools/train_net.py` and many scripts. It includes many standard default behaviors that you might want to opt in, including default configurations for the optimizer, learning rate scheduler, logging, evaluation, model checkpointing, etc. + +For simple customizations (e.g. change optimizer, evaluator, LR scheduler, data loader, etc.), you can just modify the corresponding configuration in `config.py` according to your own needs (refer to [Config_System](https://libai.readthedocs.io/en/latest/tutorials/Config_System.html#configs-in-libai)). + +## Customize a DefaultTrainer + +For complicated customizations, we recommend you to overwrite function in [DefaultTrainer](https://github.com/Oneflow-Inc/libai/blob/main/libai/engine/default.py). + +In `DefaultTrainer`, the training process consists of `run_step in trainer` and `hooks` which can be modified according to your own needs. + +The following code indicates how `run_step` and `hooks` work during training: +```python +class DefaultTrainer(TrainerBase): + def train(self, start_iter: int, max_iter: int): + ... + + with EventStorage(self.start_iter) as self.storage: + try: + self.before_train() # in hooks + for self.iter in range(start_iter, max_iter): + self.before_step() # in hooks + self.run_step() # in self._trainer + self.after_step() # in hooks + self.iter += 1 + except Exception: + logger.exception("Exception during training:") + raise + finally: + self.after_train() # in hooks + +``` + +Refer to `tools/train_net.py` to rewrite `tools/my_train_net.py` with your modified `_trainer` and `hooks`. The next subsection will introduce how to modify them. + +```python +# tools/my_train_net.py + +import ... +from libai.engine import DefaultTrainer +from path_to_myhook import myhook +from path_to_mytrainer import _mytrainer + +class MyTrainer(DefaultTrainer): + def __init__(self, cfg): + super().__init__(cfg) + + # add your _trainer according to your own needs + # NOTE: run_step() is overwrited in your _trainer + self._trainer = _mytrainer() + + def build_hooks(self): + ret = [ + hooks.IterationTimer(), + hooks.LRScheduler(), + hooks.PeriodicCheckpointer(self.checkpointer, self.cfg.train.checkpointer.period), + ] + # add your hook according to your own needs + # NOTE: all hooks will be called sequentially + ret.append(myhook()) + + ... + + if dist.is_main_process(): + ret.append(hooks.PeriodicWriter(self.build_writers(), self.cfg.train.log_period)) + return ret + +logger = logging.getLogger("libai." + __name__) + +def main(args): + ... + + trainer = MyTrainer(cfg) + return trainer.train() + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + main(args) +``` + +Using ``trainer & hook`` system means there will always be some non-standard behaviors which is hard to support in LiBai, especially for research. Therefore, we intentionally keep the ``trainer & hook`` system minimal, rather than powerful. + +### Customize Hooks in Trainer + +You can customize your own hooks for some extra tasks during training. + +[HookBase](https://github.com/Oneflow-Inc/libai/blob/ffe5ca0e46544d1cbb4fbe88d9185f96c0dc2c95/libai/engine/trainer.py#L28) in `libai/engine/trainer.py` provides a standard behavior for you to use hook. You can overwirte its function according to your own needs. Please refer to [libai/engine/hooks.py](https://github.com/Oneflow-Inc/libai/blob/main/libai/engine/hooks.py) for more details. +```python +class HookBase: + def before_train(self): + """ + Called before the first iteration. + """ + + def after_train(self): + """ + Called after the last iteration. + """ + + def before_step(self): + """ + Called before each iteration. + """ + + def after_step(self): + """ + Called after each iteration. + """ +``` + +Depending on the functionality of the hook, you can specify what the hook will do at each stage of the training in ``before_train``, ``after_train``, ``before_step``, ``after_step``. For example, to print `iter` in trainer during training: + +```python +class InfoHook(HookBase): + def before_train(self): + logger.info(f"start training at {self.trainer.iter}") + + def after_train(self): + logger.info(f"end training ad {self.trainer.iter}") + + def after_step(self): + if self.trainer.iter % 100 == 0: + logger.info(f"iteration {self.trainer.iter}!") +``` + +Then you can import your `hook` in `tools/my_train_net.py` + +### Modify train_step in Trainer + +LiBai provides `EagerTrainer` and `GraphTrainer` in `libai/engine/trainer.py` by default. `EagerTrainer` is used in `eager` mode, while `GraphTrainer` is used in `graph` mode, and the mode is determined by the `graph.enabled` parameter in your `config.py`. + +> For more details about `eager` and `graph` mode, please refer to [oneflow doc](https://docs.oneflow.org/en/master/basics/08_nn_graph.html). + +For example, using a temp variable to keep the model's output in run_step: + +```python +class MyEagerTrainer(EagerTrainer): + + def __init__(self, model, data_loader, optimizer, grad_acc_steps=1): + super().__init__(model, data_loader, optimizer, grad_acc_steps) + self.previous_output = None + + def run_step(self, get_batch: Callable): + ... + loss_dict = self.model(**data) + self.previous_output = loss_dict + ... +``` + +Then you can set your `MyEagerTrainer` as `self.trainer` in `tools/my_train_net.py` + +## Logging of Metrics + +During training, the trainer put metrics to a centralized [EventStorage](https://libai.readthedocs.io/en/latest/modules/libai.utils.html#module-libai.utils.events). The following code can be used to access it and log metrics to it: + +```python +from libai.utils.events import get_event_storage + +# inside the model: +if self.training: + value = # compute the value from inputs + storage = get_event_storage() + storage.put_scalar("some_accuracy", value) + +``` + +See [EventStorage](https://libai.readthedocs.io/en/latest/modules/libai.utils.html#module-libai.utils.events) for more details. + +Metrics are then written to various destinations with EventWriter. Metrics information will be written to `{cfg.train.output_dir}/metrics.json`. DefaultTrainer enables a few EventWriter with default configurations. See above for how to customize them. \ No newline at end of file diff --git a/docs/source/tutorials/basics/Write_Dataloaders.md b/docs/source/tutorials/basics/Write_Dataloaders.md new file mode 100644 index 0000000000000000000000000000000000000000..bcecf6320b9f1142b5756c05677f8e2dc6f302c8 --- /dev/null +++ b/docs/source/tutorials/basics/Write_Dataloaders.md @@ -0,0 +1,66 @@ +# Write Dataloaders + +This tutorial explains how the dataset APIs work, and how to customize your own datasets with them. + +## Build Common Dataloaders + +To build dataloaders in LiBai, we highly recommend users to use the default `build_nlp_train_val_test_loader`, `build_nlp_train_loader`, `build_nlp_test_loader`, `build_image_train_loader` and `build_image_test_loader` which are defined in [`libai/data/build.py`](https://github.com/Oneflow-Inc/libai/blob/main/libai/data/build.py) for most of the common cases. + +The only thing you need to do is to write pytorch style `Dataset`, and return `Instance` structure in `__getitem__`. The `Instance` structure stores the attributes of an instance (e.g., image, tokens) as "fields", and the `DistTensorData` structure provides a standard `to_global()`(called in `get_batch()`) function to convert local tensors to global tensors. + +The returned instance by `__getitem__` function must contain the same keys with the `args` passed in `forward` function of the `model`. The following shows an example: + +**NOTE:** Set `placement_idx=-1` in `DistTensorData` when the `tensor` is **only** used in `loss_function`, it is used for pipeline parallel training. + +```python +# my_dataset.py +import numpy as np +import oneflow as flow + +from libai.data.structures import DistTensorData, Instance + +class MyDataset(flow.utils.data.Dataset): + + ... + + def __getitem__(self, idx): + text = np.array(self.dataset[idx], dtype=np.long) + # transfer to flow.tensor + input_ids = flow.tensor(text[:-1], dtype=flow.long) + lm_labels = flow.tensor(text[1:2], dtype=flow.long) + # attention_mask must be a [0, 1] metric + attention_mask = flow.tensor(text[2:3], dtype=flow.long) + loss_mask = flow.tensor(text[3:], dtype=flow.long) + # the keys (`input_ids` ... `labels`) should be same as the parameter name of model.forward() + sample = Instance( + input_ids=DistTensorData(input_ids), + # attention_mask must be a [0, 1] metric + attention_mask=DistTensorData(attention_mask), + loss_mask=DistTensorData(lm_labels, placement_idx=-1), + labels=DistTensorData(lm_labels, placement_idx=-1), + ) + return sample + +# my_model.py +import oneflow.nn as nn + +class MyModel(nn.Module): + ... + + # the parameters' name is the same as the returned key in __getitem__ + def forward(self, input_ids, attention_mask, loss_mask, labels): + ... +``` + +In particular, the values of `attention_mask` can only be `0` or `1` if you need to generate your own `attention_mask`. Because LiBai has already processed `attention_mask` in [`libai/layers/attention.py`](https://github.com/Oneflow-Inc/libai/blob/main/libai/layers/attention.py) as follows: + +```python +attention_scores = flow.mul(attention_scores, attention_mask) +attention_scores = attention_scores - 10000.0 * (1 - attention_mask) +attention_weights = flow.softmax(attention_scores, dim=-1) +``` + +After finishing your `MyDataset`, set `dataloader` in your `config.py` depending on your needs. If you have only one training dataset for nlp task and want to split it into `train`, `valid` and `test` datasets automatically, you can choose `build_nlp_train_val_test_loader`, the evaluation will be calculated in `valid` and `test` dataset. + +Otherwise, you can choose `build_nlp_train_loader` && `build_nlp_test_loader` or `build_image_train_loader` && `build_image_test_loader` in `config.py` according to your own needs. +see [`libai/data/build.py`](https://github.com/Oneflow-Inc/libai/blob/main/libai/data/build.py) for more details. \ No newline at end of file diff --git a/docs/source/tutorials/basics/Write_Models.md b/docs/source/tutorials/basics/Write_Models.md new file mode 100644 index 0000000000000000000000000000000000000000..cda870e050aff96fd25157d61e090fdde8063aff --- /dev/null +++ b/docs/source/tutorials/basics/Write_Models.md @@ -0,0 +1,59 @@ +# Write Models + +This section introduces how to implement a new model entirely from scratch and make it compatible with LiBai. + + +## Construct Models in LiBai + +LiBai uses [LazyConfig](https://libai.readthedocs.io/en/latest/tutorials/Config_System.html) for a more flexible config system, which means you can simply import your own model in your config and train it under LiBai. + +For image classification task, the input data is usually a batch of images and labels. The following code shows how to build a toy model for this task. Import in your code: +```python +# toy_model.py +import oneflow as flow +import oneflow.nn as nn + + +class ToyModel(nn.Module): + def __init__(self, + num_classes=1000, + ): + super().__init__() + self.features = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1) + self.avgpool = nn.AdaptiveAvgPool2d(1) + self.classifier = nn.Linear(64, num_classes) + self.loss_func = nn.CrossEntropyLoss() + + def forward(self, images, labels=None): + x = self.features(images) + x = self.avgpool(x) + x = flow.flatten(x, 1) + x = self.classifier(x) + + if labels is not None and self.training: + losses = self.loss_func(x, labels) + return {"losses": losses} + else: + return {"prediction_scores": x} +``` + +**Note:** +- For classification models, the ``forward`` function must have ``images`` and ``labels`` as arguments, which correspond to the output in ``__getitem__`` of LiBai's built-in datasets. Please refer to [imagenet.py](https://github.com/Oneflow-Inc/libai/blob/main/libai/data/datasets/imagenet.py) for more details about the dataset. +- **This toy model** will return ``losses`` during training and ``prediction_scores`` during inference, and both of them should be the type of ``dict``, which means you should implement the ``loss function`` in your model, like ``self.loss_func=nn.CrossEntropyLoss()`` as the ToyModel shows above. + + +## Import the model in config + +With ``LazyConfig System``, you can simply import the model in your config file. The following code shows how to use ``ToyModel`` in your config file: +```python +# config.py +from libai.config import LazyCall +from toy_model import ToyModel + +model = LazyCall(ToyModel)( + num_classes=1000 +) +``` + + + diff --git a/docs/source/tutorials/basics/index.rst b/docs/source/tutorials/basics/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..430a214c60024448e5c371e39b3ade603ba45a81 --- /dev/null +++ b/docs/source/tutorials/basics/index.rst @@ -0,0 +1,19 @@ +Basics +========= + +.. toctree:: + :glob: + :maxdepth: 2 + + Config_System.md + Features.md + Training.md + Evaluation.md + Train_and_Eval_Command_Line.md + Load_and_Save_Checkpoint.md + Write_Models.md + Write_Dataloaders.md + Build_New_Project_on_LiBai.md + Distributed_Configuration.md + Auto_Parallel.md + Preprocessing_Dataset.md \ No newline at end of file diff --git a/docs/source/tutorials/get_started/Benchmark.md b/docs/source/tutorials/get_started/Benchmark.md new file mode 100644 index 0000000000000000000000000000000000000000..0a05bf68e3c49d997ed4f3040f04db846d3b7689 --- /dev/null +++ b/docs/source/tutorials/get_started/Benchmark.md @@ -0,0 +1,599 @@ +# Benchmarks + +Here we provides our benchmark speed test results of LiBai's models compared with [Megatron-LM](https://github.com/NVIDIA/Megatron-LM) implementations. In LiBai V0.2.0, we only benchmark the speed tests under 32 GPUs in 4 nodes and all of the experiments were conducted under the same settings for a fair comparison. + +## Settings +### Environments + +- The commit of LiBai for comparison: [commit](https://github.com/Oneflow-Inc/libai/commit/9fc504c457da4fd1e92d854c60b7271c89a55222) +- The commit of OneFlow for comparison: [commit](https://github.com/Oneflow-Inc/oneflow/commit/55b822e4d3c88757d11077d7546981309125c73f) +- The commit of Megatron-LM for comparison: [commit](https://github.com/NVIDIA/Megatron-LM/commit/e156d2fea7fc5c98e645f7742eb86b643956d840) + +### Model Hyper-parameters +- **BERT Model** +```python +num_layers = 24/48 +num_attention_heads = 16 +hidden_size = 1024 +seq_length = 512 +``` +- **GPT-2 Model** +```python +num_layers = 24/48 +num_attention_heads = 16 +hidden_size = 1024 +seq_length = 1024 +``` + + +## Main Results +Here we explain the evaluation indicators in the following tables: +- **fp16**: mixed precision training +- **nl**: num layers (When pipeline parallel size = 8, in order to have a relative number of layers per stage for computation, we adjust the num layers from 24 to 48) +- **ac**: enable activation checkpointing +- **mb**: micro-batch size per gpu +- **gb**: global batch size total +- **d x m x p**: + - d: data-parallel-size + - m: tensor-model-parallel-size + - p: pipeline-model-parallel-size +- **1n1g**: 1 node, 1 gpu +- **2n8g**: 2 nodes, 8 gpus per node, 16 gpus in total +- **4n8g**: 4 nodes, 8 gpus per node, 32 gpus in total +- `grad_acc_num_step = global_batch_size / (micro_batch_size * data_parallel_size)` +- **samples/s**: throughput + + +### Data Parallel + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
BERTLiBaiMegatron
nl24_fp16_1x1x1_mb24_gb24_1n1g46.91 + samples/s42.6 + samples/s
nl24_fp16_4x1x1_mb16_gb64_1n4g176.88 + samples/s154.7 + samples/s
nl24_fp16_8x1x1_mb16_gb128_1n8g351.57 + samples/s309.2 + samples/s
nl24_fp16_16x1x1_mb16_gb256_2n8g675.87 + samples/s534.7 + samples/s
nl24_fp16_32x1x1_mb16_gb512_4n8g1207.65 + samples/s950.3 + samples/s
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
GPT-2LiBaiMegatron
nl24_fp16_1x1x1_mb6_gb6_1n1g17.52 + samples/s15.5 + samples/s
nl24_fp16_4x1x1_mb4_gb16_1n4g63.45 + samples/s53.3 + samples/s
nl24_fp16_8x1x1_mb4_gb32_1n8g125.64 + samples/s107.9 + samples/s
nl24_fp16_16x1x1_mb4_gb64_2n8g215.35 + samples/s176.0 + samples/s
nl24_fp16_32x1x1_mb4_gb128_4n8g329.58 + samples/s296.6 + samples/s
+ +### Tensor Model Parallel + + + + + + + + + + + + + + + + + + + + + + + + + + +
BERTLiBaiMegatron
nl24_fp16_1x1x1_ac_mb128_gb1024_1n1g35.74 + samples/s33.6 + samples/s
nl24_fp16_1x4x1_ac_mb128_gb1024_1n4g87.12 + samples/s86.6 + samples/s
nl24_fp16_1x8x1_ac_mb128_gb1024_1n8g131.94 + samples/s128.7 + samples/s
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
GPT-2LiBaiMegatron
nl24_fp16_1x1x1_mb6_gb6_1n1g17.52 + samples/s15.5 + samples/s
nl24_fp16_1x4x1_mb6_gb6_1n4g40.38 + samples/s38.0 + samples/s
nl24_fp16_1x8x1_mb8_gb8_1n8g60.53 + samples/s55.7 + samples/s
+ +### Pipeline Model Parallel + + + + + + + + + + + + + + + + + + + + + + + + + + +
BERTLiBaiMegatron
nl24_fp16_1x1x1_ac_mb128_gb1024_1n1g35.74 + samples/s33.6 + samples/s
nl24_fp16_1x1x4_ac_mb128_gb1024_1n4g103.6 + samples/s88.7 + samples/s
nl48_fp16_1x1x8_ac_mb64_gb1024_1n8g94.4 + samples/s85.5 + samples/s
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
GPT-2LiBaiMegatron
nl24_fp16_1x1x1_ac_mb32_gb256_1n1g14.43 + samples/s13.3 + samples/
nl24_fp16_1x1x4_ac_mb32_gb256_1n4g41.9 + samples/s33.2 + samples/s
nl48_fp16_1x1x8_ac_mb24_gb384_1n8g37.4 + samples/s31.8 + samples/s
+ +### 2-D Parallel + +#### Data Parallel + Tensor Model Parallel + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
BERTLiBaiMegatron
nl24_fp16_2x2x1_ac_mb128_gb2048_1n4g88.47 + samples/s86.6 + samples/s
nl24_fp16_4x2x1_ac_mb128_gb4096_1n8g175.94 + samples/s172.0 + samples/s
nl24_fp16_8x2x1_ac_mb128_gb8192_2n8g348.58 + samples/s343.8 + samples/s
nl24_fp16_2x8x1_ac_mb128_gb2048_2n8g261.78 + samples/s255.8 + samples/s
nl24_fp16_4x4x1_ac_mb128_gb2048_2n8g338.97 + samples/s337.3 + samples/s
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
GPT-2LiBaiMegatron
nl24_fp16_2x2x1_ac_mb32_gb512_1n4g37.63 + samples/s36.9 + samples/s
nl24_fp16_4x2x1_ac_mb32_gb1024_1n8g74.35 + samples/s73.2 + samples/s
nl24_fp16_8x2x1_ac_mb32_gb2048_2n8g148.94 + samples/s146.5 + samples/s
nl24_fp16_2x8x1_ac_mb32_gb512_2n8g116.04 + samples/s109.1 + samples/s
nl24_fp16_4x4x1_ac_mb32_gb512_2n8g141.25 + samples/s138.1 + samples/s
+ +#### Data Parallel + Pipeline Model Parallel + + + + + + + + + + + + + + + + + + + + + + + + + + +
BERTLiBaiMegatron
nl24_fp16_2x1x4_ac_mb128_gb2048_1n8g207.31 + samples/s175.0 + samples/s
nl24_fp16_4x1x4_ac_mb128_gb4096_2n8g406.24 + samples/s342.9 + samples/s
nl24_fp16_8x1x4_ac_mb128_gb8192_4n8g805.04 + samples/s650.7 + samples/s
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
GPT-2LiBaiMegatron
nl24_fp16_2x1x4_ac_mb32_gb512_1n8g83.12 + samples/s65.3 + samples/s
nl24_fp16_4x1x4_ac_mb32_gb1024_2n8g164.23 + samples/s128.4 + samples/s
nl24_fp16_8x1x4_ac_mb32_gb2048_4n8g322.42 + samples/s247.3 + samples/s
+ +### 3-D Parallel + + + + + + + + + + + + + + + + + + + + + + + + + + +
BERTLiBaiMegatron
nl24_fp16_2x2x4_ac_mb128_gb2048_2n8g267.39 + samples/s233.7 + samples/s
nl24_fp16_4x2x4_ac_mb192_gb6144_4n8g503.51 + samples/s439.4 + samples/s
nl24_fp16_2x4x4_ac_mb256_gb4096_4n8g405.75 + samples/s338.7 + samples/s
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
GPT-2LiBaiMegatron
nl24_fp16_2x2x4_ac_mb32_gb1024_2n8g128.77 + samples/s106.3 + samples/s
nl24_fp16_4x2x4_ac_mb48_gb1536_4n8g209.32 + samples/s179.5 + samples/s
nl24_fp16_2x4x4_ac_mb64_gb1024_4n8g186.67 + samples/s178.2 + samples/s
\ No newline at end of file diff --git a/docs/source/tutorials/get_started/Installation.md b/docs/source/tutorials/get_started/Installation.md new file mode 100644 index 0000000000000000000000000000000000000000..da82b89b595c456ef62dcade2b242a8d181d85cb --- /dev/null +++ b/docs/source/tutorials/get_started/Installation.md @@ -0,0 +1,74 @@ +# Installation + +LiBai provides an editable installation way for you to develop your own project based on LiBai's framework. + +## Build LiBai from Source + +- Clone this repo: + +```bash +git clone https://github.com/Oneflow-Inc/libai.git +cd libai +``` + +- Create a conda virtual environment and activate it: + +```bash +conda create -n libai python=3.8 -y +conda activate libai +``` + +- Install the stable release of OneFlow with `CUDA` support. See [OneFlow installation guide](https://github.com/Oneflow-Inc/oneflow#install-with-pip-package). To use **latest** LiBai(branch `main`), we highly recommend you install **Nightly** Oneflow + + - Stable + + ```bash + python3 -m pip install --find-links https://release.oneflow.info oneflow==0.8.0+[PLATFORM] + ``` + + - Nightly + + ``` + python3 -m pip install --pre oneflow -f https://staging.oneflow.info/branch/master/[PLATFORM] + ``` + + - All available `[PLATFORM]`: + + + + + + + + + + + + + + + + + + + + + + + + + + +
PlatformCUDA Driver VersionSupported GPUs
cu112>= 450.80.02GTX 10xx, RTX 20xx, A100, RTX 30xx
cu102>= 440.33GTX 10xx, RTX 20xx
cpuN/AN/A
+ +- Install `pybind11`: + +```bash +pip install pybind11 +``` + +- For an editable installation of LiBai: + +```bash +pip install -e . +``` diff --git a/docs/source/tutorials/get_started/Model_Zoo.md b/docs/source/tutorials/get_started/Model_Zoo.md new file mode 100644 index 0000000000000000000000000000000000000000..9cbb9048012c32d1cd941c2486582cadb5a1f0ad --- /dev/null +++ b/docs/source/tutorials/get_started/Model_Zoo.md @@ -0,0 +1,111 @@ +# LiBai Model Zoo +To date, LiBai has implemented the following models: +- [Vision Transformer](https://arxiv.org/abs/2010.11929) +- [Swin Transformer](https://arxiv.org/abs/2103.14030) +- [ResMLP](https://arxiv.org/abs/2105.03404) +- [BERT](https://arxiv.org/abs/1810.04805) +- [T5](https://arxiv.org/abs/1910.10683) +- [GPT-2](https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf) + + +## Parallelism Mode in LiBai +A collection of parallel training strategies is supported in LiBai: +- **Data Parallel Training** +- **Tensor Parallel Training** +- **Pipeline Parallel Training** + +You can refer to OneFlow official [tutorial](https://docs.oneflow.org/en/master/parallelism/01_introduction.html) to better understand the basic conception of parallelization techniques. + + +## Supported Models in LiBai + +For more details about the supported parallelism training on different models, please refer to the following table: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Model Data ParallelTensor ParallelPipeline Parallel
Vision Transformer ✔✔✔
Swin Transformer ✔--
ResMLP ✔✔✔
BERT ✔✔✔
T5 ✔✔✔
GPT-2 ✔✔✔
+ +**Additions:** +✔ means you can train this model under specific parallelism techniques or combine two or three of them with ✔ for 2D or 3D paralleism training. + +## Baselines +Here is the collection of baselines trained with LiBai. Due to our resource constraints, we will gradually release the training results in the future. + +### Main Results on ImageNet with Pretrained Models + +**ImageNet-1K Pretrained Models** + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Model PretrainResolutionAcc@1Acc@5Download
ViT-Tiny w/o EMA ImageNet-1K 224x224 72.7 91.0 Config | Checkpoint
ViT-Small w/o EMA ImageNet-1K 224x224 79.3 94.5 Config | Checkpoint
+ +**Notes:** `w/o EMA` denotes to models pretrained without **Exponential Moving Average** (EMA). \ No newline at end of file diff --git a/docs/source/tutorials/get_started/index.rst b/docs/source/tutorials/get_started/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..b164f6b887c39ea5a72ea4196a23997b5d1a1ce2 --- /dev/null +++ b/docs/source/tutorials/get_started/index.rst @@ -0,0 +1,12 @@ +Get Started +============= + +.. toctree:: + :glob: + :maxdepth: 2 + + Installation.md + quick_run.md + Model_Zoo.md + Benchmark.md + run_distributed_inference.md \ No newline at end of file diff --git a/docs/source/tutorials/get_started/quick_run.md b/docs/source/tutorials/get_started/quick_run.md new file mode 100644 index 0000000000000000000000000000000000000000..cfe61ce5a33a008c3c96ca3501119897065ba152 --- /dev/null +++ b/docs/source/tutorials/get_started/quick_run.md @@ -0,0 +1,125 @@ +# Quick Run +This is a step-by-step tutorial on how to get started with LiBai: +- [Quick Run](#quick-run) + - [Train Bert-large Model Parallelly](#train-bert-large-model-parallelly) + - [Prepare the Data and the Vocab](#prepare-the-data-and-the-vocab) + - [How to Train Bert_large Model with Parallelism](#how-to-train-bert_large-model-with-parallelism) + - [Train VisionTransformer on ImageNet Dataset](#train-visiontransformer-on-imagenet-dataset) + - [Prepare the Data](#prepare-the-data) + - [Train vit Model from Scratch](#train-vit-model-from-scratch) + + +## Train Bert-large Model Parallelly +### Prepare the Data and the Vocab + +- We have prepared relevant datasets, which can be downloaded from the following links: + +1. [VOCAB_URL](https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/bert-base-chinese-vocab.txt) +2. [BIN_DATA_URL](https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/loss_compara_content_sentence.bin) +3. [IDX_DATA_URL](https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/loss_compara_content_sentence.idx) + +- Download the dataset and move the data file to the folder. The file structure should be like: +```bash +$ tree data +path/to/bert_data +├── bert-base-chinese-vocab.txt +├── loss_compara_content_sentence.bin +└── loss_compara_content_sentence.idx +``` +### How to Train Bert_large Model with Parallelism + +We provide `train.sh` for execute training. Before invoking the script, perform the following steps. + +**Step 1. Set data path and vocab path** + +- Update the data path and vocab path in [bert_large_pretrain](https://github.com/Oneflow-Inc/libai/blob/main/configs/bert_large_pretrain.py) config file: +```python +# Refine data path and vocab path to data folder +vocab_file = "/path/to/bert_data/bert-base-chinese-vocab.txt" +data_prefix = "/path/to/bert_data/loss_compara_content_sentence" +``` + +**Step 2. Configure your parameters** +- In the [`configs/bert_large_pretrain.py`](https://github.com/Oneflow-Inc/libai/blob/main/configs/bert_large_pretrain.py) provided, a set of parameters are defined including training scheme, model, etc. +- You can also modify the parameters setting. For example, if you want to use 8 GPUs for training, you can refer to the file [`configs/common/train.py`](https://github.com/Oneflow-Inc/libai/blob/main/configs/common/train.py). If you want to train model with 2D mesh hybrid parallelism (4 groups for data parallel and 2 groups for tensor parallel), you can set the the parameters as follows: + +```python +train.dist.data_parallel_size=4 +train.dist.tensor_parallel_size=2 +``` + +**Step 3. Invoke parallel training** +- To train `BertForPreTraining` model on a single node with 8 GPUs, run: +```bash +bash tools/train.sh tools/train_net.py configs/bert_large_pretrain.py 8 +``` + +- To train `BertForPreTraining` model on 2 nodes with 16 GPUs, + + in `node0`, run: + ```bash + NODE=2 NODE_RANK=0 ADDR=192.168.0.0 PORT=12345 bash tools/train.sh tools/train_net.py configs/bert_large_pretrain.py 8 + ``` + `NODE=2` means total number of nodes + + `NODE_RANK=0` means current node is node0 + + `ADDR=192.168.0.0` means the ip address of node0 + + `PORT=12345` means the port of node0 + + in `node1`, run: + ```bash + NODE=2 NODE_RANK=1 ADDR=192.168.0.0 PORT=12345 bash tools/train.sh tools/train_net.py configs/bert_large_pretrain.py 8 + ``` + `NODE=2` means total number of nodes + + `NODE_RANK=1` means current node is node1 + + `ADDR=192.168.0.0` means the ip address of node0 + + `PORT=12345` means the port of node0 + +## Train VisionTransformer on ImageNet Dataset +### Prepare the Data +For ImageNet, we use standard ImageNet dataset, which can be downloaded from http://image-net.org/. +- For the standard folder dataset, move validation images to labeled sub-folders. The file structure should be like: +```bash +$ tree data +imagenet +├── train +│ ├── class1 +│ │ ├── img1.jpeg +│ │ ├── img2.jpeg +│ │ └── ... +│ ├── class2 +│ │ ├── img3.jpeg +│ │ └── ... +│ └── ... +└── val + ├── class1 + │ ├── img4.jpeg + │ ├── img5.jpeg + │ └── ... + ├── class2 + │ ├── img6.jpeg + │ └── ... + └── ... + +``` +### Train vit Model from Scratch +- Update the data path in [vit_imagenet](https://github.com/Oneflow-Inc/libai/blob/main/configs/vit_imagenet.py) config file: +```python +# Refine data path to imagenet data folder +dataloader.train.dataset[0].root = "/path/to/imagenet" +dataloader.test[0].dataset.root = "/path/to/imagenet" +``` +- To train `vit_tiny_patch16_224` model on ImageNet on a single node with 8 GPUs for 300 epochs, run: +```bash +bash tools/train.sh tools/train_net.py configs/vit_imagenet.py 8 +``` +- The default vit model in LiBai is set to `vit_tiny_patch16_224`. To train other vit models, update the [vit_imagenet](https://github.com/Oneflow-Inc/libai/blob/main/configs/vit_imagenet.py) config file by importing other vit models in the config file as follows: +```python +# from .common.models.vit.vit_tiny_patch16_224 import model +from .common.models.vit.vit_base_patch16_224 import model +``` diff --git a/docs/source/tutorials/get_started/run_distributed_inference.md b/docs/source/tutorials/get_started/run_distributed_inference.md new file mode 100644 index 0000000000000000000000000000000000000000..c7764210e2305bcefcce170f6aaa0671c479098c --- /dev/null +++ b/docs/source/tutorials/get_started/run_distributed_inference.md @@ -0,0 +1,78 @@ +# Run distributed inference + +This is a tutorial on how to quickly run distributed inference in LiBai from a `huggingface` pretrained model. + +## Download model weights + +run shell +```shell +mkdir -r data_test/t5_inference_model/ +cd data_test/t5_inference_model +wget https://huggingface.co/t5-base/resolve/main/pytorch_model.bin https://huggingface.co/t5-base/resolve/main/config.json https://huggingface.co/t5-base/resolve/main/spiece.model +``` + +the dir will like this: +```shell +data_test/t5_inference_model +├── config.json +├── pytorch_model.bin +├── spiece.model +``` + +## run text_generation.py + +set `vocab_file` path in `projects/MT5/configs/t5_inference.py` + +```python +tokenization.tokenizer = LazyCall(T5Tokenizer)( + vocab_file="data_test/t5_inference_model/spiece.model", + add_bos_token=True, +) +``` + +set your own distributed config in `libai/inference/text_generation.py` + +```python +if __name__ == "__main__": + pipeline = TextGenerationPipeline( + "projects/MT5/configs/t5_inference.py", + data_parallel=1, + tensor_parallel=2, + pipeline_parallel=2, + pipeline_stage_id=[0] * 12 + [1] * 12, + pipeline_num_layers=12 * 2, + model_path="data_test/t5_inference_model", + mode="huggingface", + ) + + text = ["summarize: She is a student, She is tall, She loves study"] + dict1 = pipeline(text) + if dist.is_main_process(): + print(dict1) +``` + +To run distributed inference on 2 nodes with total 4 GPUs, + + in `node0`, run: + ```bash + NODE=2 NODE_RANK=1 ADDR=192.168.0.1 PORT=12345 bash tools/infer.sh libai/inference/text_generation.py 2 + ``` + `NODE=2` means total number of nodes + + `NODE_RANK=0` means current node is node0 + + `ADDR=192.168.0.0` means the ip address of node0 + + `PORT=12345` means the port of node0 + + in `node1`, run: + ```bash + NODE=2 NODE_RANK=1 ADDR=192.168.0.1 PORT=12345 bash tools/infer.sh libai/inference/text_generation.py 2 + ``` + `NODE=2` means total number of nodes + + `NODE_RANK=1` means current node is node1 + + `ADDR=192.168.0.0` means the ip address of node0 + + `PORT=12345` means the port of node0 \ No newline at end of file diff --git a/docs/source/tutorials/index.rst b/docs/source/tutorials/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..9997a1716c5b2f0c3062bb2714f4df4393d90800 --- /dev/null +++ b/docs/source/tutorials/index.rst @@ -0,0 +1,11 @@ +Tutorials +========= + +.. toctree:: + :glob: + :maxdepth: 3 + + get_started/index + basics/index + advanced_tutorials/index + diff --git a/libai/__init__.py b/libai/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b31b3780467be1ba2b45e3af2c13b1487ee14d79 --- /dev/null +++ b/libai/__init__.py @@ -0,0 +1,33 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# This line will be programatically read/write by setup.py. +# Leave them at the bottom of this file and don't touch them. + +from libai import data +from libai import evaluation +from libai import layers +from libai import models +from libai import optim +from libai import scheduler +from libai import tokenizer +from libai import engine +from libai import utils + +try: + from .version import __version__ # noqa: F401 +except ImportError: + pass diff --git a/libai/__pycache__/__init__.cpython-39.pyc b/libai/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..57f1f7c1f51006e56fb6a23444c40b9dd454cd4f Binary files /dev/null and b/libai/__pycache__/__init__.cpython-39.pyc differ diff --git a/libai/__pycache__/version.cpython-39.pyc b/libai/__pycache__/version.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..02ce801a36e63c2cb8b51dd1abe262434feff5df Binary files /dev/null and b/libai/__pycache__/version.cpython-39.pyc differ diff --git a/libai/config/__init__.py b/libai/config/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7d2ebdaf9f1d52bf46bf395484063e5dba2f8fea --- /dev/null +++ b/libai/config/__init__.py @@ -0,0 +1,29 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .arguments import default_argument_parser +from .config import configurable, try_get_key, get_config +from .instantiate import instantiate +from .lazy import LazyCall, LazyConfig + +__all__ = [ + "LazyCall", + "LazyConfig", + "instantiate", + "default_argument_parser", + "configurable", + "try_get_key", + "get_config", +] diff --git a/libai/config/__pycache__/__init__.cpython-39.pyc b/libai/config/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8996ea74ce8abd123a61da2536d9f07bd7bf6687 Binary files /dev/null and b/libai/config/__pycache__/__init__.cpython-39.pyc differ diff --git a/libai/config/__pycache__/arguments.cpython-39.pyc b/libai/config/__pycache__/arguments.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ffc18171d5e5ff529c9b8af4052e3e6b6b1852d8 Binary files /dev/null and b/libai/config/__pycache__/arguments.cpython-39.pyc differ diff --git a/libai/config/__pycache__/config.cpython-39.pyc b/libai/config/__pycache__/config.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8f886826870ff7e806851eeb7a31996d6e86fb4a Binary files /dev/null and b/libai/config/__pycache__/config.cpython-39.pyc differ diff --git a/libai/config/__pycache__/instantiate.cpython-39.pyc b/libai/config/__pycache__/instantiate.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b56f38eba8257c806c3d39c860f2ef2bbc68e9db Binary files /dev/null and b/libai/config/__pycache__/instantiate.cpython-39.pyc differ diff --git a/libai/config/__pycache__/lazy.cpython-39.pyc b/libai/config/__pycache__/lazy.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f12e000fee56e186e5f347865c439b6edb270be4 Binary files /dev/null and b/libai/config/__pycache__/lazy.cpython-39.pyc differ diff --git a/libai/config/arguments.py b/libai/config/arguments.py new file mode 100644 index 0000000000000000000000000000000000000000..533fb1f9ad9e5da642df052dff733fb49ef4dbaf --- /dev/null +++ b/libai/config/arguments.py @@ -0,0 +1,80 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import sys + + +def default_argument_parser(epilog=None): + """Create a parser with some common arguments used by libai users. + + Args: + epilog (str): epilog passed to ArgumentParser describing the usage. + + Returns: + argparse.ArgumentParser. + """ + parser = argparse.ArgumentParser( + epilog=epilog + or f""" +Examples: + +Run on single machine: + $ python3 -m oneflow.distributed.launch \ + --nproc_per_node 8 --nnodes 1 --node_rank 0 --master_addr 127.0.0.1 --master_port 12345 \ + {sys.argv[0]} --config-file cfg.yaml + +Change some config options: + $ python3 -m oneflow.distributed.launch \ + --nproc_per_node 8 --nnodes 1 --node_rank 0 --master_addr 127.0.0.1 --master_port 12345 \ + {sys.argv[0]} --config-file cfg.yaml train.load_weight=/path/to/weight.pth optim.lr=0.001 + +Run on multiple machines: + (machine0)$ python3 -m oneflow.distributed.launch \ + --nproc_per_node 8 --nnodes 2 --node_rank 0 --master_addr --master_port 12345 \ + {sys.argv[0]} --config-file cfg.yaml + + $ python3 -m oneflow.distributed.launch \ + --nproc_per_node 8 --nnodes 2 --node_rank 1 --master_addr --master_port 12345 \ + {sys.argv[0]} --config-file cfg.yaml +""", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--config-file", default="", metavar="FILE", help="path to config file") + parser.add_argument( + "--resume", + action="store_true", + help="Whether to attempt to resume from the checkpoint directory. " + "See documentation of `DefaultTrainer.resume_or_load()` for what it means.", + ) + parser.add_argument("--eval-only", action="store_true", help="Perform evaluation only") + parser.add_argument( + "--fast-dev-run", + action="store_true", + help="Run several batches of train, eval and test to find any bugs, " + "(ie: a sort of unit test)", + ) + parser.add_argument( + "opts", + help=""" +Modify config options at the end of the command. For Yacs configs, use +space-separated "path.key value" pairs. +For python-based LazyConfig, use "path.key=value". + """.strip(), + default=None, + nargs=argparse.REMAINDER, + ) + return parser diff --git a/libai/config/config.py b/libai/config/config.py new file mode 100644 index 0000000000000000000000000000000000000000..3d6671bc4a782e329659ad1a9717441095fd126f --- /dev/null +++ b/libai/config/config.py @@ -0,0 +1,198 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import inspect +import os + +import pkg_resources +from omegaconf import OmegaConf + +from .lazy import LazyConfig + +# -------------------------------------------------------- +# References: +# https://github.com/facebookresearch/detectron2/blob/main/detectron2/config/config.py +# -------------------------------------------------------- + + +def configurable(init_func=None, *, from_config=None): + """ + Decorate a function or a class's __init__ method so that it can be called + with a :class:`CfgNode` object using a :func:`from_config` function that translates + :class:`CfgNode` to arguments. + + Examples: + + .. code-block:: python + + # Usage 1: Decorator on __init__: + class A: + @configurable + def __init__(self, a, b=2, c=3): + pass + + @classmethod + def from_config(cls, cfg): # 'cfg' must be the first argument + # Returns kwargs to be passed to __init__ + return {"a": cfg.A, "b": cfg.B} + + a1 = A(a=1, b=2) # regular construction + a2 = A(cfg) # construct with a cfg + a3 = A(cfg, b=3, c=4) # construct with extra overwrite + + # Usage 2: Decorator on any function. Needs an extra from_config argument: + @configurable(from_config=lambda cfg: {"a: cfg.A, "b": cfg.B}) + def a_func(a, b=2, c=3): + pass + + a1 = a_func(a=1, b=2) # regular call + a2 = a_func(cfg) # call with a cfg + a3 = a_func(cfg, b=3, c=4) # call with extra overwrite + + Args: + init_func (callable): a class's ``__init__`` method in usage 1. The + class must have a ``from_config`` classmethod which takes `cfg` as + the first argument. + from_config (callable): the from_config function in usage 2. It must take `cfg` + as its first argument. + """ + + if init_func is not None: + assert ( + inspect.isfunction(init_func) + and from_config is None + and init_func.__name__ == "__init__" + ), "Incorrect use of @configurable. Check API documentation for examples." + + @functools.wraps(init_func) + def wrapped(self, *args, **kwargs): + try: + from_config_func = type(self).from_config + except AttributeError as e: + raise AttributeError( + "Class with @configurable must have a 'from_config' classmethod." + ) from e + if not inspect.ismethod(from_config_func): + raise TypeError("Class with @configurable must have a 'from_config' classmethod.") + + if _called_with_cfg(*args, **kwargs): + explicit_args = _get_args_from_config(from_config_func, *args, **kwargs) + init_func(self, **explicit_args) + else: + init_func(self, *args, **kwargs) + + return wrapped + + else: + if from_config is None: + return configurable # @configurable() is made equivalent to @configurable + assert inspect.isfunction( + from_config + ), "from_config argument of configurable must be a function!" + + def wrapper(orig_func): + @functools.wraps(orig_func) + def wrapped(*args, **kwargs): + if _called_with_cfg(*args, **kwargs): + explicit_args = _get_args_from_config(from_config, *args, **kwargs) + return orig_func(**explicit_args) + else: + return orig_func(*args, **kwargs) + + wrapped.from_config = from_config + return wrapped + + return wrapper + + +def _get_args_from_config(from_config_func, *args, **kwargs): + """ + Use `from_config` to obtain explicit arguments. + Returns: + dict: arguments to be used for cls.__init__ + """ + signature = inspect.signature(from_config_func) + if list(signature.parameters.keys())[0] != "cfg": + if inspect.isfunction(from_config_func): + name = from_config_func.__name__ + else: + name = f"{from_config_func.__self__}.from_config" + raise TypeError(f"{name} must take 'cfg' as the first argument!") + support_var_arg = any( + param.kind in [param.VAR_POSITIONAL, param.VAR_KEYWORD] + for param in signature.parameters.values() + ) + if support_var_arg: # forward all arguments to from_config, if from_config accepts them + ret = from_config_func(*args, **kwargs) + else: + # forward supported arguments to from_config + supported_arg_names = set(signature.parameters.keys()) + extra_kwargs = {} + for name in list(kwargs.keys()): + if name not in supported_arg_names: + extra_kwargs[name] = kwargs.pop(name) + ret = from_config_func(*args, **kwargs) + # forward the other arguments to __init__ + ret.update(extra_kwargs) + return ret + + +def _called_with_cfg(*args, **kwargs): + """ + Returns: + bool: whether the arguments contain CfgNode and should be considered + forwarded to from_config. + """ + from omegaconf import DictConfig + + if len(args) and isinstance(args[0], DictConfig): + return True + if isinstance(kwargs.pop("cfg", None), DictConfig): + return True + # `from_config`'s first argument is forced to be "cfg". + # So the above check covers all cases. + return False + + +def try_get_key(cfg, *keys, default=None): + """ + Try select keys from cfg until the first key that exists. Otherwise return default. + """ + for k in keys: + none = object() + p = OmegaConf.select(cfg, k, default=none) + if p is not none: + return p + return default + + +def get_config(config_path): + """ + Returns a config object from a config_path. + + Args: + config_path (str): config file name relative to libai's "configs/" + directory, e.g., "common/models/bert.py" + + Returns: + omegaconf.DictConfig: a config object + """ + cfg_file = pkg_resources.resource_filename("libai.config", os.path.join("configs", config_path)) + if not os.path.exists(cfg_file): + raise RuntimeError("{} not available in LiBai configs!".format(config_path)) + cfg = LazyConfig.load(cfg_file) + return cfg diff --git a/libai/config/configs b/libai/config/configs new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/libai/config/instantiate.py b/libai/config/instantiate.py new file mode 100644 index 0000000000000000000000000000000000000000..ff58ad0aaa834cde9f7290efc1b7def4314e753a --- /dev/null +++ b/libai/config/instantiate.py @@ -0,0 +1,201 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dataclasses +import logging +from collections import abc +from enum import Enum +from typing import Any, Callable, Dict, List, Union + +from hydra.errors import InstantiationException +from omegaconf import OmegaConf + +from libai.config.lazy import _convert_target_to_string, locate + +logger = logging.getLogger(__name__) + +__all__ = ["dump_dataclass", "instantiate"] + +# -------------------------------------------------------- +# References: +# https://github.com/facebookresearch/detectron2/blob/main/detectron2/config/instantiate.py +# -------------------------------------------------------- + + +class _Keys(str, Enum): + """Special keys in configs used by instantiate.""" + + TARGET = "_target_" + RECURSIVE = "_recursive_" + + +def _is_target(x: Any) -> bool: + if isinstance(x, dict): + return _Keys.TARGET in x + if OmegaConf.is_dict(x): + return _Keys.TARGET in x + return False + + +def _is_dict(cfg: Any) -> bool: + return OmegaConf.is_dict(cfg) or isinstance(cfg, abc.Mapping) + + +def _is_list(cfg: Any) -> bool: + return OmegaConf.is_list(cfg) or isinstance(cfg, list) + + +def dump_dataclass(obj: Any): + """ + Dump a dataclass recursively into a dict that can be later instantiated. + + Args: + obj: a dataclass object + + Returns: + dict + """ + assert dataclasses.is_dataclass(obj) and not isinstance( + obj, type + ), "dump_dataclass() requires an instance of a dataclass." + ret = {"_target_": _convert_target_to_string(type(obj))} + for f in dataclasses.fields(obj): + v = getattr(obj, f.name) + if dataclasses.is_dataclass(v): + v = dump_dataclass(v) + if isinstance(v, (list, tuple)): + v = [dump_dataclass(x) if dataclasses.is_dataclass(x) else x for x in v] + ret[f.name] = v + return ret + + +def _prepare_input_dict_or_list(d: Union[Dict[Any, Any], List[Any]]) -> Any: + res: Any + if isinstance(d, dict): + res = {} + for k, v in d.items(): + if k == "_target_": + v = _convert_target_to_string(d["_target_"]) + elif isinstance(v, (dict, list)): + v = _prepare_input_dict_or_list(v) + res[k] = v + elif isinstance(d, list): + res = [] + for v in d: + if isinstance(v, (list, dict)): + v = _prepare_input_dict_or_list(v) + res.append(v) + else: + assert False + return res + + +def _resolve_target(target): + if isinstance(target, str): + try: + target = locate(target) + except Exception as e: + msg = f"Error locating target '{target}', see chained exception above." + raise InstantiationException(msg) from e + + if not callable(target): + msg = f"Expected a callable target, got '{target}' of type '{type(target).__name__}'" + raise InstantiationException(msg) + return target + + +def _call_target(_target_: Callable[..., Any], kwargs: Dict[str, Any]): + """Call target (type) with kwargs""" + + try: + return _target_(**kwargs) + except Exception as e: + msg = f"Error in call to target '{_convert_target_to_string(_target_)}':\n{repr(e)}" + raise InstantiationException(msg) from e + + +def instantiate(cfg, **kwargs: Any) -> Any: + """ + Recursively instantiate objects defined in dictionaries by + "_target_" and arguments. + + Args: + cfg: a dict-like object with "_target_" that defines the caller, and + other keys that define the arguments + + Returns: + object instantiated by cfg + """ + if cfg is None: + return None + + if isinstance(cfg, (dict, list)): + cfg = _prepare_input_dict_or_list(cfg) + + kwargs = _prepare_input_dict_or_list(kwargs) + + if _is_dict(cfg): + if kwargs: + cfg = OmegaConf.merge(cfg, kwargs) + + _recursive_ = kwargs.pop(_Keys.RECURSIVE, True) + return instantiate_cfg(cfg, recursive=_recursive_) + + elif _is_list(cfg): + _recursive_ = kwargs.pop(_Keys.RECURSIVE, True) + return instantiate_cfg(cfg, recursive=_recursive_) + else: + return cfg # return as-is if don't know what to do + + +def instantiate_cfg(cfg: Any, recursive: bool = True): + if cfg is None: + return cfg + + if _is_dict(cfg): + recursive = cfg[_Keys.RECURSIVE] if _Keys.RECURSIVE in cfg else recursive + + if not isinstance(recursive, bool): + msg = f"Instantiation: _recursive_ flag must be a bool, got {type(recursive)}" + raise TypeError(msg) + + # If OmegaConf list, create new list of instances if recursive + if OmegaConf.is_list(cfg): + items = [instantiate_cfg(item, recursive=recursive) for item in cfg._iter_ex(resolve=True)] + lst = OmegaConf.create(items, flags={"allow_objects": True}) + return lst + + elif isinstance(cfg, list): + # Specialize for list, because many classes take + # list[objects] as arguments, such as ResNet, DatasetMapper + return [instantiate(item, recursive=recursive) for item in cfg] + + elif _is_dict(cfg): + exclude_keys = set({"_target_", "_recursive_"}) + if _is_target(cfg): + _target_ = instantiate(cfg.get(_Keys.TARGET)) # instantiate lazy target + _target_ = _resolve_target(_target_) + kwargs = {} + for key, value in cfg.items(): + if key not in exclude_keys: + if recursive: + value = instantiate_cfg(value, recursive=recursive) + kwargs[key] = value + return _call_target(_target_, kwargs) + else: + return cfg + else: + return cfg # return as-is if don't know what to do diff --git a/libai/config/lazy.py b/libai/config/lazy.py new file mode 100644 index 0000000000000000000000000000000000000000..bbff11a2d062423cc522d633c878e8d884002afc --- /dev/null +++ b/libai/config/lazy.py @@ -0,0 +1,463 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ast +import builtins +import importlib +import inspect +import logging +import os +import pydoc +import uuid +from collections import abc +from contextlib import contextmanager +from copy import deepcopy +from dataclasses import is_dataclass +from typing import Any, List, Tuple, Union + +import cloudpickle +import yaml +from omegaconf import DictConfig, ListConfig, OmegaConf + +__all__ = ["LazyCall", "LazyConfig"] + +# -------------------------------------------------------- +# References: +# https://github.com/facebookresearch/detectron2/blob/main/detectron2/config/lazy.py +# -------------------------------------------------------- + + +def locate(name: str) -> Any: + """ + Locate and return an object ``x`` using an input string ``{x.__module__}.{x.__qualname__}``, + such as "module.submodule.class_name". + Raise Exception if it cannot be found. + """ + obj = pydoc.locate(name) + + # Some cases (e.g. flow.optim.sgd.SGD) not handled correctly + # by pydoc.locate. Try a private function from hydra. + if obj is None: + try: + # from hydra.utils import get_method - will print many errors + from hydra.utils import _locate + except ImportError as e: + raise ImportError(f"Cannot dynamically locate object {name}!") from e + else: + obj = _locate(name) # it raises if fails + + return obj + + +def _convert_target_to_string(t: Any) -> str: + """ + Inverse of ``locate()``. + Args: + t: any object with ``__module__`` and ``__qualname__`` + """ + module, qualname = t.__module__, t.__qualname__ + + # Compress the path to this object, e.g. ``module.submodule._impl.class`` + # may become ``module.submodule.class``, if the later also resolves to the same + # object. This simplifies the string, and also is less affected by moving the + # class implementation. + module_parts = module.split(".") + for k in range(1, len(module_parts)): + prefix = ".".join(module_parts[:k]) + candidate = f"{prefix}.{qualname}" + try: + if locate(candidate) is t: + return candidate + except ImportError: + pass + return f"{module}.{qualname}" + + +class LazyCall: + """ + Wrap a callable so that when it's called, the call will not be executed, + but returns a dict that describes the call. + + LazyCall object has to be called with only keyword arguments. Positional + arguments are not yet supported. + + Examples: + + .. code-block:: python + + from libai.config import instantiate, LazyCall + layer_cfg = LazyCall(nn.Conv2d)(in_channels=32, out_channels=32) + layer_cfg.out_channels = 64 # can edit it afterwards + layer = instantiate(layer_cfg) + """ + + def __init__(self, target): + if not (callable(target) or isinstance(target, (str, abc.Mapping))): + raise TypeError( + f"target of LazyCall must be a callable or defines a callable! Got {target}" + ) + self._target = target + + def __call__(self, **kwargs): + if is_dataclass(self._target): + # omegaconf object cannot hold dataclass type + # https://github.com/omry/omegaconf/issues/784 + target = _convert_target_to_string(self._target) + else: + target = self._target + kwargs["_target_"] = target + + return DictConfig(content=kwargs, flags={"allow_objects": True}) + + +def _visit_dict_config(cfg, func): + """ + Apply func recursively to all DictConfig in cfg. + """ + if isinstance(cfg, DictConfig): + func(cfg) + for v in cfg.values(): + _visit_dict_config(v, func) + elif isinstance(cfg, ListConfig): + for v in cfg: + _visit_dict_config(v, func) + + +def _validate_py_syntax(filename): + # see also https://github.com/open-mmlab/mmcv/blob/master/mmcv/utils/config.py + with open(filename, "r", encoding="utf-8") as f: + # Setting encoding explicitly to resolve coding issue on windows + content = f.read() + try: + ast.parse(content) + except SyntaxError as e: + raise SyntaxError(f"Config file {filename} has syntax error!") from e + + +def _cast_to_config(obj): + # if given a dict, return DictConfig instead + if isinstance(obj, dict): + return DictConfig(obj, flags={"allow_objects": True}) + return obj + + +_CFG_PACKAGE_NAME = "libai._cfg_loader" +""" +A namespace to put all imported config into. +""" + + +def _random_package_name(filename): + # generate a random package name when loading config files + return _CFG_PACKAGE_NAME + str(uuid.uuid4())[:4] + "." + os.path.basename(filename) + + +@contextmanager +def _patch_import(): + """ + Enhance relative import statements in config files, so that they: + 1. locate files purely based on relative location, regardless of packages. + e.g. you can import file without having __init__ + 2. do not cache modules globally; modifications of module states has no side effect + 3. support other storage system through PathManager + 4. imported dict are turned into omegaconf.DictConfig automatically + """ + old_import = builtins.__import__ + + def find_relative_file(original_file, relative_import_path, level): + cur_file = os.path.dirname(original_file) + for _ in range(level - 1): + cur_file = os.path.dirname(cur_file) + cur_name = relative_import_path.lstrip(".") + for part in cur_name.split("."): + cur_file = os.path.join(cur_file, part) + # NOTE: directory import is not handled. Because then it's unclear + # if such import should produce python module or DictConfig. This can + # be discussed further if needed. + if not cur_file.endswith(".py"): + cur_file += ".py" + if not os.path.isfile(cur_file): + raise ImportError( + f"Cannot import name {relative_import_path} from " + f"{original_file}: {cur_file} has to exist." + ) + return cur_file + + def new_import(name, globals=None, locals=None, fromlist=(), level=0): + if ( + # Only deal with relative imports inside config files + level != 0 + and globals is not None + and (globals.get("__package__", "") or "").startswith(_CFG_PACKAGE_NAME) + ): + cur_file = find_relative_file(globals["__file__"], name, level) + _validate_py_syntax(cur_file) + spec = importlib.machinery.ModuleSpec( + _random_package_name(cur_file), None, origin=cur_file + ) + module = importlib.util.module_from_spec(spec) + module.__file__ = cur_file + with open(cur_file, "r", encoding="utf-8") as f: + content = f.read() + exec(compile(content, cur_file, "exec"), module.__dict__) + for name in fromlist: # turn imported dict into DictConfig automatically + val = _cast_to_config(module.__dict__[name]) + module.__dict__[name] = val + return module + return old_import(name, globals, locals, fromlist=fromlist, level=level) + + builtins.__import__ = new_import + yield new_import + builtins.__import__ = old_import + + +class LazyConfig: + """ + Provide methods to save, load, and overrides an omegaconf config object + which may contain definition of lazily-constructed objects. + """ + + @staticmethod + def load_rel(filename: str, keys: Union[None, str, Tuple[str, ...]] = None): + """ + Similar to :meth:`load()`, but load path relative to the caller's + source file. + This has the same functionality as a relative import, except that this method + accepts filename as a string, so more characters are allowed in the filename. + """ + caller_frame = inspect.stack()[1] + caller_fname = caller_frame[0].f_code.co_filename + assert caller_fname != "", "load_rel Unable to find caller" + caller_dir = os.path.dirname(caller_fname) + filename = os.path.join(caller_dir, filename) + return LazyConfig.load(filename, keys) + + @staticmethod + def load(filename: str, keys: Union[None, str, Tuple[str, ...]] = None): + """ + Load a config file. + + Args: + filename: absolute path or relative path w.r.t. the current working directory + keys: keys to load and return. If not given, return all keys + (whose values are config objects) in a dict. + """ + has_keys = keys is not None + filename = filename.replace("/./", "/") # redundant + if os.path.splitext(filename)[1] not in [".py", ".yaml", ".yml"]: + raise ValueError(f"Config file {filename} has to be a python or yaml file.") + if filename.endswith(".py"): + _validate_py_syntax(filename) + + with _patch_import(): + # Record the filename + module_namespace = { + "__file__": filename, + "__package__": _random_package_name(filename), + } + with open(filename, "r", encoding="utf-8") as f: + content = f.read() + # Compile first with filename to: + # 1. make filename appears in stacktrace + # 2. make load_rel able to find its parent's (possibly remote) location + exec(compile(content, filename, "exec"), module_namespace) + + ret = module_namespace + else: + with open(filename, "r", encoding="utf-8") as f: + obj = yaml.unsafe_load(f) + ret = OmegaConf.create(obj, flags={"allow_objects": True}) + + if has_keys: + if isinstance(keys, str): + return _cast_to_config(ret[keys]) + else: + return tuple(_cast_to_config(ret[a]) for a in keys) + else: + if filename.endswith(".py"): + # when not specified, only load those that are config objects + ret = DictConfig( + { + name: _cast_to_config(value) + for name, value in ret.items() + if isinstance(value, (DictConfig, ListConfig, dict)) + and not name.startswith("_") + }, + flags={"allow_objects": True}, + ) + return ret + + @staticmethod + def save(cfg, filename: str): + """ + Save a config object to a yaml file. + Note that when the config dictionary contains complex objects (e.g. lambda), + it can't be saved to yaml. In that case we will print an error and + attempt to save to a pkl file instead. + + Args: + cfg: an omegaconf config object + filename: yaml file name to save the config file + """ + logger = logging.getLogger(__name__) + try: + cfg = deepcopy(cfg) + except Exception: + pass + else: + # if it's deep-copyable, then... + def _replace_type_by_name(x): + if "_target_" in x and callable(x._target_): + try: + x._target_ = _convert_target_to_string(x._target_) + except AttributeError: + pass + + # not necessary, but makes yaml looks nicer + _visit_dict_config(cfg, _replace_type_by_name) + + save_pkl = False + try: + dict = OmegaConf.to_container(cfg, resolve=False) + dumped = yaml.dump(dict, default_flow_style=None, allow_unicode=True, width=9999) + with open(filename, "w") as f: + f.write(dumped) + + try: + _ = yaml.unsafe_load(dumped) # test that it is loadable + except Exception: + logger.warning( + "The config contains objects that cannot serialize to a valid yaml. " + f"{filename} is human-readable but cannot be loaded." + ) + save_pkl = True + except Exception: + logger.exception("Unable to serialize the config to yaml. Error:") + save_pkl = True + + if save_pkl: + new_filename = filename + ".pkl" + try: + # retry by pickle + with open(new_filename, "wb") as f: + cloudpickle.dump(cfg, f) + logger.warning(f"Config is saved using cloudpickle at {new_filename}.") + except Exception: + pass + + @staticmethod + def apply_overrides(cfg, overrides: List[str]): + """ + In-place override contents of cfg. + + Args: + cfg: an omegaconf config object + overrides: list of strings in the format of "a=b" to override configs. + + See https://hydra.cc/docs/next/advanced/override_grammar/basic/ for syntax. + + Returns: + the cfg object + """ + + def safe_update(cfg, key, value): + parts = key.split(".") + for idx in range(1, len(parts)): + prefix = ".".join(parts[:idx]) + v = OmegaConf.select(cfg, prefix, default=None) + if v is None: + break + if not OmegaConf.is_config(v): + raise KeyError( + f"Trying to update key {key}, but {prefix} " + f"is not a config, but has type {type(v)}." + ) + OmegaConf.update(cfg, key, value, merge=True) + + from hydra.core.override_parser.overrides_parser import OverridesParser + + parser = OverridesParser.create() + overrides = parser.parse_overrides(overrides) + for o in overrides: + key = o.key_or_group + value = o.value() + if o.is_delete(): + # TODO support this + raise NotImplementedError("deletion is not yet a supported override") + safe_update(cfg, key, value) + return cfg + + @staticmethod + def to_py(cfg, prefix: str = "cfg."): + """ + Try to convert a config object into Python-like pseudo code. + Note that perfect conversion is not always possible. So the returned + results are mainly meant to be human-readable, and not meant to be executed. + + Args: + cfg: an omegaconf config object + prefix: root name for the resulting code (default: "cfg.") + + Returns: + str of formatted Python code + """ + import black + + cfg = OmegaConf.to_container(cfg, resolve=True) + + def _to_str(obj, prefix=None, inside_call=False): + if prefix is None: + prefix = [] + if isinstance(obj, abc.Mapping) and "_target_" in obj: + # Dict representing a function call + target = _convert_target_to_string(obj.pop("_target_")) + args = [] + for k, v in sorted(obj.items()): + args.append(f"{k}={_to_str(v, inside_call=True)}") + args = ", ".join(args) + call = f"{target}({args})" + return "".join(prefix) + call + elif isinstance(obj, abc.Mapping) and not inside_call: + # Dict that is not inside a call is a list of top-level config objects that we + # render as one object per line with dot separated prefixes + key_list = [] + for k, v in sorted(obj.items()): + if isinstance(v, abc.Mapping) and "_target_" not in v: + key_list.append(_to_str(v, prefix=prefix + [k + "."])) + else: + key = "".join(prefix) + k + key_list.append(f"{key}={_to_str(v)}") + return "\n".join(key_list) + elif isinstance(obj, abc.Mapping): + # Dict that is inside a call is rendered as a regular dict + return ( + "{" + + ",".join( + f"{repr(k)}: {_to_str(v, inside_call=inside_call)}" + for k, v in sorted(obj.items()) + ) + + "}" + ) + elif isinstance(obj, list): + return "[" + ",".join(_to_str(x, inside_call=inside_call) for x in obj) + "]" + else: + return repr(obj) + + py_str = _to_str(cfg, prefix=[prefix]) + try: + return black.format_str(py_str, mode=black.Mode()) + except black.InvalidInput: + return py_str diff --git a/libai/data/__init__.py b/libai/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f096889cb8de471b62ba8cb53e001f1dfc223356 --- /dev/null +++ b/libai/data/__init__.py @@ -0,0 +1,22 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .structures import DistTensorData, Instance +from .build import ( + build_image_train_loader, + build_image_test_loader, + build_nlp_train_val_test_loader, + build_nlp_test_loader, +) diff --git a/libai/data/__pycache__/__init__.cpython-39.pyc b/libai/data/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..17f0e0995e2ff568d2a6bde5456e8d5caa4c2d66 Binary files /dev/null and b/libai/data/__pycache__/__init__.cpython-39.pyc differ diff --git a/libai/data/__pycache__/build.cpython-39.pyc b/libai/data/__pycache__/build.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..66d5b0a8697362bce3a8bad32c37ce361ee8b125 Binary files /dev/null and b/libai/data/__pycache__/build.cpython-39.pyc differ diff --git a/libai/data/__pycache__/structures.cpython-39.pyc b/libai/data/__pycache__/structures.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eb04e9e5b24a392ae5fec6cf216ca7e92210290c Binary files /dev/null and b/libai/data/__pycache__/structures.cpython-39.pyc differ diff --git a/libai/data/build.py b/libai/data/build.py new file mode 100644 index 0000000000000000000000000000000000000000..656929f8b3ed73b2d824f421ec4d49347bbce3ce --- /dev/null +++ b/libai/data/build.py @@ -0,0 +1,401 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from omegaconf import OmegaConf +from oneflow.utils.data import DataLoader +from oneflow.utils.data.dataset import ConcatDataset + +from libai.config import LazyCall, instantiate +from libai.utils import distributed as dist + +from .data_utils import get_train_valid_test_split_ +from .samplers import CyclicSampler, SingleRoundSampler +from .structures import Instance + + +def build_nlp_train_val_test_loader( + dataset, + splits, + weights, + train_val_test_num_samples, + train_batch_size, + test_batch_size, + train_sampler=LazyCall(CyclicSampler)(shuffle=True), + test_sampler=LazyCall(SingleRoundSampler)(shuffle=False, drop_last=False), + num_workers=4, + consumed_samples=0, + seed=0, + collate_fn=None, + dataset_mixer=ConcatDataset, +): + """ + Build nlp train_val_test dataloader, used for dataset lack of valid/test dataset + + Returns: + It will return train/valid/test dataloader + + * train_loader: dataloader for training + * valid_loader: dataloader for validation + * test_loader: dataloader for testing + + Arguments: + dataset: dataset from which to load the data. e.g.: dataset or [dataset1, dataset2, ...] + splits: ratio config for spliting dataset to train/valid/test. e.g.: [[7, 2, 1], ...] + weights: ratio config for concate dataset list (Not Supported yet). e.g.: [1.0, ...] + train_batch_size: how many samples per batch to load in training (micro-batch-size per GPU). + test_batch_size: how many samples per batch to load in testing (micro-batch-size per GPU). + sampler: defines the strategy to draw + samples from the dataset. Can be any ``Iterable`` with ``__len__`` + implemented. + num_workers: how many subprocesses to use for data + loading. ``0`` means that the data will be loaded in the main process. + (default: ``4``). + consumed_samples: the number of samples that have been trained at the current time, + used for resuming training (default: ``0``). + seed: random seed, used for reproducing experiments (default: ``0``). + collate_fn: merges a list of samples to form a + mini-batch of Tensor(s). Used when using batched loading from a + map-style dataset. + dataset_mixer: function for concating list dataset. + """ + + def build_dataset(index, dataset): + doc_idx_ptr = indexed_dataset.get_doc_idx() + start_index = ds_splits[index] + end_index = ds_splits[index + 1] + 1 + indexed_dataset.set_doc_idx(doc_idx_ptr[start_index:end_index]) + dataset.indexed_dataset = indexed_dataset + dataset.max_num_samples = train_val_test_num_samples[index] + dataset = instantiate(dataset) + + # Set the original pointer so dataset remains the main dataset. + indexed_dataset.set_doc_idx(doc_idx_ptr) + # check + assert indexed_dataset.doc_idx[0] == 0 + assert indexed_dataset.doc_idx.shape[0] == (total_num_of_documents + 1) + return dataset + + if OmegaConf.is_list(dataset): + dataset = list(dataset) + elif not isinstance(dataset, list): + dataset = [dataset] + + assert len(dataset) == len(splits), "datasets length must equal splits length" + assert len(dataset) == len(weights), "datasets length must equal weights length" + + train_datasets, val_datasets, test_datasets = [], [], [] + for dst, split in zip(dataset, splits): + indexed_dataset = instantiate(dst.indexed_dataset) + total_num_of_documents = indexed_dataset.doc_idx.shape[0] - 1 + ds_splits = get_train_valid_test_split_(total_num_of_documents, split) + + train_dataset = build_dataset(0, dst) + val_dataset = build_dataset(1, dst) + test_dataset = build_dataset(2, dst) + + train_datasets.append(train_dataset) + val_datasets.append(val_dataset) + test_datasets.append(test_dataset) + + # [dataset, dataset] -> dataset -> dataloader + train_dataset = dataset_mixer(train_datasets) + val_dataset = dataset_mixer(val_datasets) + test_dataset = dataset_mixer(test_datasets) + + collate_fn = trivial_batch_collator if collate_fn is None else collate_fn + + train_loader, _, _ = build_nlp_train_loader( + dataset=train_dataset, + train_batch_size=train_batch_size, + test_batch_size=None, + sampler=train_sampler, + num_workers=num_workers, + consumed_samples=consumed_samples, + seed=seed, + collate_fn=collate_fn, + ) + + valid_loader = build_nlp_test_loader( + dataset=val_dataset, + test_batch_size=test_batch_size, + sampler=test_sampler, + num_workers=num_workers, + seed=seed, + collate_fn=collate_fn, + ) + + test_loader = build_nlp_test_loader( + dataset=test_dataset, + test_batch_size=test_batch_size, + sampler=test_sampler, + num_workers=num_workers, + seed=seed, + collate_fn=collate_fn, + ) + + return train_loader, valid_loader, test_loader + + +def build_nlp_train_loader( + dataset, + train_batch_size, + test_batch_size=None, + sampler=LazyCall(CyclicSampler)(shuffle=True), + num_workers=4, + consumed_samples=0, + seed=0, + collate_fn=None, + dataset_mixer=ConcatDataset, + **kwargs +): + """ + Build nlp train dataloader, it's used for train dataset + + Returns: + It will return train dataloader, and Nonetype for valid/test dataloader + + * train_loader: dataloader for training + * None: Nonetype + * None: Nonetype + + Arguments: + dataset: dataset from which to load the data. e.g.: dataset or [dataset1, dataset2, ...] + train_batch_size: how many samples per batch to load in training (micro-batch-size per GPU). + test_batch_size: no use, set it to None. + sampler: defines the strategy to draw + samples from the dataset. Can be any ``Iterable`` with ``__len__`` + implemented. + num_workers: how many subprocesses to use for data + loading. ``0`` means that the data will be loaded in the main process. + (default: ``4``). + consumed_samples: the number of samples that have been trained at the current time, + used for resuming training (default: ``0``). + seed: random seed, used for reproducing experiments (default: ``0``). + collate_fn: merges a list of samples to form a + mini-batch of Tensor(s). Used when using batched loading from a + map-style dataset. + dataset_mixer: function for concating list dataset. + """ + dataset = instantiate(dataset) + if OmegaConf.is_list(dataset): + dataset = list(dataset) + elif not isinstance(dataset, list): + dataset = [dataset] + + if len(dataset) > 1: + dataset = dataset_mixer(dataset) + else: + dataset = dataset[0] + + sampler.dataset = dataset + sampler.micro_batch_size = train_batch_size + sampler.consumed_samples = consumed_samples + sampler.data_parallel_rank = dist.get_data_parallel_rank() + sampler.data_parallel_size = dist.get_data_parallel_size() + sampler.seed = seed + sampler = instantiate(sampler) + + dataloader = DataLoader( + dataset, + batch_sampler=sampler, + num_workers=num_workers, + persistent_workers=True if num_workers > 0 else False, + collate_fn=trivial_batch_collator if collate_fn is None else collate_fn, + **kwargs, + ) + + return dataloader, None, None + + +def build_nlp_test_loader( + dataset, + test_batch_size, + sampler=LazyCall(SingleRoundSampler)(shuffle=False, drop_last=False), + num_workers=4, + seed=0, + collate_fn=None, +): + """ + Build nlp test dataloader, it's used for test dataset + + Returns: + It will return test dataloader + + * test_loader: dataloader for testing + + Arguments: + dataset: dataset from which to load the data. e.g.: dataset or [dataset1, dataset2, ...] + test_batch_size: how many samples per batch to load in testing (micro-batch-size per GPU). + sampler: defines the strategy to draw + samples from the dataset. Can be any ``Iterable`` with ``__len__`` + implemented. + num_workers: how many subprocesses to use for data + loading. ``0`` means that the data will be loaded in the main process. + (default: ``4``). + seed: random seed, used for reproducing experiments (default: ``0``). + collate_fn: merges a list of samples to form a + mini-batch of Tensor(s). Used when using batched loading from a + map-style dataset. + """ + dataset = instantiate(dataset) + collate_fn = trivial_batch_collator if collate_fn is None else collate_fn + + sampler.dataset = dataset + sampler.micro_batch_size = test_batch_size + sampler.data_parallel_rank = dist.get_data_parallel_rank() + sampler.data_parallel_size = dist.get_data_parallel_size() + sampler.seed = seed + sampler = instantiate(sampler) + + test_loader = DataLoader( + dataset, + batch_sampler=sampler, + num_workers=num_workers, + persistent_workers=True if num_workers > 0 else False, + collate_fn=collate_fn, + ) + return test_loader + + +def build_image_train_loader( + dataset, + train_batch_size, + test_batch_size=None, + sampler=LazyCall(CyclicSampler)(shuffle=True), + num_workers=4, + consumed_samples=0, + seed=0, + collate_fn=None, + dataset_mixer=ConcatDataset, + mixup_func=None, + **kwargs +): + """ + Build image train dataloader, it's used for train dataset + + Returns: + It will return train dataloader, and Nonetype for valid/test dataloader + + * train_loader: dataloader for training + * None: Nonetype + * None: Nonetype + + Arguments: + dataset: dataset from which to load the data. e.g.: dataset or [dataset1, dataset2, ...] + train_batch_size: how many samples per batch to load in training (micro-batch-size per GPU). + test_batch_size: no use, set it to None. + sampler: defines the strategy to draw + samples from the dataset. Can be any ``Iterable`` with ``__len__`` + implemented. + num_workers: how many subprocesses to use for data + loading. ``0`` means that the data will be loaded in the main process. + (default: ``4``). + consumed_samples: the number of samples that have been trained at the current time, + used for resuming training (default: ``0``). + seed: random seed, used for reproducing experiments (default: ``0``). + collate_fn: merges a list of samples to form a + mini-batch of Tensor(s). Used when using batched loading from a + map-style dataset. + dataset_mixer: function for concating list dataset. + mixup_func: function for data argumentation. + """ + dataset = instantiate(dataset) + + if OmegaConf.is_list(dataset): + dataset = list(dataset) + elif not isinstance(dataset, list): + dataset = [dataset] + + if len(dataset) > 1: + dataset = dataset_mixer(dataset) + else: + dataset = dataset[0] + + sampler.dataset = dataset + sampler.micro_batch_size = train_batch_size + sampler.consumed_samples = consumed_samples + sampler.data_parallel_rank = dist.get_data_parallel_rank() + sampler.data_parallel_size = dist.get_data_parallel_size() + sampler.seed = seed + sampler = instantiate(sampler) + + dataloader = DataLoader( + dataset, + batch_sampler=sampler, + num_workers=num_workers, + persistent_workers=True if num_workers > 0 else False, + collate_fn=trivial_batch_collator if collate_fn is None else collate_fn, + **kwargs, + ) + # Bind up mixup_func to dataloader, and this will be used in Trainer.get_batch + dataloader.mixup_func = instantiate(mixup_func) + + return dataloader, None, None + + +def build_image_test_loader( + dataset, + test_batch_size, + sampler=LazyCall(SingleRoundSampler)(shuffle=True, drop_last=False), + num_workers=4, + seed=0, + collate_fn=None, + **kwargs +): + """ + Build image test dataloader, used for test dataset + + Returns: + It will return test dataloader + + * test_loader: dataloader for testing + + Arguments: + dataset: dataset from which to load the data. e.g.: dataset or [dataset1, dataset2, ...] + test_batch_size: how many samples per batch to load in testing (micro-batch-size per GPU). + sampler: defines the strategy to draw + samples from the dataset. Can be any ``Iterable`` with ``__len__`` + implemented. + num_workers: how many subprocesses to use for data + loading. ``0`` means that the data will be loaded in the main process. + (default: ``4``). + seed: random seed, used for reproducing experiments (default: ``0``). + collate_fn: merges a list of samples to form a + mini-batch of Tensor(s). Used when using batched loading from a + map-style dataset. + """ + dataset = instantiate(dataset) + + sampler.dataset = dataset + sampler.micro_batch_size = test_batch_size + sampler.data_parallel_rank = dist.get_data_parallel_rank() + sampler.data_parallel_size = dist.get_data_parallel_size() + sampler.seed = seed + sampler = instantiate(sampler) + + return DataLoader( + dataset, + batch_sampler=sampler, + num_workers=num_workers, + persistent_workers=True if num_workers > 0 else False, + collate_fn=trivial_batch_collator if collate_fn is None else collate_fn, + **kwargs, + ) + + +def trivial_batch_collator(batch): + assert isinstance(batch[0], Instance), "batch[0] must be `instance` for trivial batch collator" + batch = Instance.stack(batch) + return batch diff --git a/libai/data/data_utils/Makefile b/libai/data/data_utils/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..d42b2bd45939512b219b8a3caac3e36bfd4344fe --- /dev/null +++ b/libai/data/data_utils/Makefile @@ -0,0 +1,9 @@ +CXXFLAGS += -O3 -Wall -shared -std=c++11 -fPIC -fdiagnostics-color +CPPFLAGS += $(shell python3 -m pybind11 --includes) +LIBNAME = helpers +LIBEXT = $(shell python3-config --extension-suffix) + +default: $(LIBNAME)$(LIBEXT) + +%$(LIBEXT): %.cpp + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $< -o $@ \ No newline at end of file diff --git a/libai/data/data_utils/__init__.py b/libai/data/data_utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2f84ba2d76d8280cac2af33daabee53ef47f1a4c --- /dev/null +++ b/libai/data/data_utils/__init__.py @@ -0,0 +1,28 @@ +# coding=utf-8 +# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .dataset_utils import ( + compile_helper, + create_masked_lm_predictions, + get_samples_mapping, + get_train_valid_test_split_, +) + +from .indexed_dataset import ( + IndexedCachedDataset, + IndexedDataset, + MMapIndexedDataset, + get_indexed_dataset, +) diff --git a/libai/data/data_utils/__pycache__/__init__.cpython-39.pyc b/libai/data/data_utils/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f595e961caaa3cbeafecf6ed9bc6a0e641a4fc31 Binary files /dev/null and b/libai/data/data_utils/__pycache__/__init__.cpython-39.pyc differ diff --git a/libai/data/data_utils/__pycache__/dataset_utils.cpython-39.pyc b/libai/data/data_utils/__pycache__/dataset_utils.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f0152c365783cc1c0804c5e2c3905ae26af591bc Binary files /dev/null and b/libai/data/data_utils/__pycache__/dataset_utils.cpython-39.pyc differ diff --git a/libai/data/data_utils/__pycache__/indexed_dataset.cpython-39.pyc b/libai/data/data_utils/__pycache__/indexed_dataset.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d0fbb12de38ff1de865998bb27f954d0bf983abe Binary files /dev/null and b/libai/data/data_utils/__pycache__/indexed_dataset.cpython-39.pyc differ diff --git a/libai/data/data_utils/dataset_utils.py b/libai/data/data_utils/dataset_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f2928860d3a4d79705db3fff6baeaa0b7c9c47d1 --- /dev/null +++ b/libai/data/data_utils/dataset_utils.py @@ -0,0 +1,424 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors, and NVIDIA. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import collections +import logging +import os +import re +import time + +import numpy as np +import oneflow as flow + +from libai.utils import distributed as dist + +logger = logging.getLogger(__name__) + +# Most of the code here has been copied from: +# https://github.com/google-research/albert/blob/master/create_pretraining_data.py +# with some modifications. + + +def compile_helper(): + """Compile helper function at runtime. Make sure this + is invoked on a single process.""" + import os + import subprocess + + path = os.path.abspath(os.path.dirname(__file__)) + ret = subprocess.run(["make", "-C", path]) + if ret.returncode != 0: + logger.info("Making C++ dataset helpers module failed, exiting.") + import sys + + sys.exit(1) + + +MaskedLmInstance = collections.namedtuple("MaskedLmInstance", ["index", "label"]) + + +def is_start_piece(piece): + """Check if the current word piece is the starting piece (BERT).""" + # When a word has been split into + # WordPieces, the first token does not have any marker and any subsequence + # tokens are prefixed with ##. So whenever we see the ## token, we + # append it to the previous set of word indexes. + return not piece.startswith("##") + + +def create_masked_lm_predictions( + tokenizer, + tokens, + vocab_id_list, + vocab_id_to_token_dict, + masked_lm_prob, + cls_id, + sep_id, + mask_id, + max_predictions_per_seq, + np_rng, + max_ngrams=3, + do_whole_word_mask=True, + favor_longer_ngram=False, + do_permutation=False, + geometric_dist=False, + masking_style="bert", +): + """Creates the predictions for the masked LM objective. + Note: Tokens here are vocab ids and not text tokens.""" + + cand_indexes = [] + # Note(mingdachen): We create a list for recording if the piece is + # the starting piece of current token, where 1 means true, so that + # on-the-fly whole word masking is possible. + token_boundary = [0] * len(tokens) + + for (i, token) in enumerate(tokens): + if token == cls_id or token == sep_id: + token_boundary[i] = 1 + continue + # Whole Word Masking means that if we mask all of the wordpieces + # corresponding to an original word. + # + # Note that Whole Word Masking does *not* change the training code + # at all -- we still predict each WordPiece independently, softmaxed + # over the entire vocabulary. + if ( + do_whole_word_mask + and len(cand_indexes) >= 1 + and not is_start_piece(vocab_id_to_token_dict[token]) + ): + cand_indexes[-1].append(i) + else: + cand_indexes.append([i]) + if is_start_piece(vocab_id_to_token_dict[token]): + token_boundary[i] = 1 + + output_tokens = list(tokens) + # add by ganruyi + if masking_style == "bert-cn-wwm": + # if non chinese is False, that means it is chinese, + # then try to remove "##" which is added previously + new_token_ids = [] + for token_id in output_tokens: + token = tokenizer.convert_ids_to_tokens([token_id])[0] + if len(re.findall("##[\u4E00-\u9FA5]", token)) > 0: + token = token[2:] + new_token_id = tokenizer.convert_tokens_to_ids([token])[0] + new_token_ids.append(new_token_id) + output_tokens = new_token_ids + + masked_lm_positions = [] + masked_lm_labels = [] + + if masked_lm_prob == 0: + return (output_tokens, masked_lm_positions, masked_lm_labels, token_boundary) + + num_to_predict = min(max_predictions_per_seq, max(1, int(round(len(tokens) * masked_lm_prob)))) + + ngrams = np.arange(1, max_ngrams + 1, dtype=np.int64) + if not geometric_dist: + # Note(mingdachen): + # By default, we set the probabities to favor shorter ngrams sequences. + pvals = 1.0 / np.arange(1, max_ngrams + 1) + pvals /= pvals.sum(keepdims=True) + if favor_longer_ngram: + pvals = pvals[::-1] + + ngram_indexes = [] + for idx in range(len(cand_indexes)): + ngram_index = [] + for n in ngrams: + ngram_index.append(cand_indexes[idx : idx + n]) + ngram_indexes.append(ngram_index) + + np_rng.shuffle(ngram_indexes) + + (masked_lms, masked_spans) = ([], []) + covered_indexes = set() + for cand_index_set in ngram_indexes: + if len(masked_lms) >= num_to_predict: + break + if not cand_index_set: + continue + # Note(mingdachen): + # Skip current piece if they are covered in lm masking or previous ngrams. + for index_set in cand_index_set[0]: + for index in index_set: + if index in covered_indexes: + continue + + if not geometric_dist: + n = np_rng.choice( + ngrams[: len(cand_index_set)], + p=pvals[: len(cand_index_set)] / pvals[: len(cand_index_set)].sum(keepdims=True), + ) + else: + # Sampling "n" from the geometric distribution and clipping it to + # the max_ngrams. Using p=0.2 default from the SpanBERT paper + # https://arxiv.org/pdf/1907.10529.pdf (Sec 3.1) + n = min(np_rng.geometric(0.2), max_ngrams) + + index_set = sum(cand_index_set[n - 1], []) + n -= 1 + # Note(mingdachen): + # Repeatedly looking for a candidate that does not exceed the + # maximum number of predictions by trying shorter ngrams. + while len(masked_lms) + len(index_set) > num_to_predict: + if n == 0: + break + index_set = sum(cand_index_set[n - 1], []) + n -= 1 + # If adding a whole-word mask would exceed the maximum number of + # predictions, then just skip this candidate. + if len(masked_lms) + len(index_set) > num_to_predict: + continue + is_any_index_covered = False + for index in index_set: + if index in covered_indexes: + is_any_index_covered = True + break + if is_any_index_covered: + continue + for index in index_set: + covered_indexes.add(index) + masked_token = None + if masking_style == "bert": + # 80% of the time, replace with [MASK] + if np_rng.random() < 0.8: + masked_token = mask_id + else: + # 10% of the time, keep original + if np_rng.random() < 0.5: + masked_token = tokens[index] + # 10% of the time, replace with random word + else: + masked_token = vocab_id_list[np_rng.randint(0, len(vocab_id_list))] + elif masking_style == "bert-cn-wwm": + # 80% of the time, replace with [MASK] + if np_rng.random() < 0.8: + masked_token = mask_id + else: + # 10% of the time, keep original + if np_rng.random() < 0.5: + # if it's chinese wwm, remove ## in toknes + token_id = tokens[index] + token = tokenizer.convert_ids_to_tokens([token_id])[0] + if len(re.findall("##[\u4E00-\u9FA5]", token)) > 0: + token = token[2:] + new_token_id = tokenizer.convert_tokens_to_ids([token])[0] + masked_token = new_token_id + # 10% of the time, replace with random word + else: + masked_token = vocab_id_list[np_rng.randint(0, len(vocab_id_list))] + elif masking_style == "t5": + masked_token = mask_id + else: + raise ValueError("invalid value of masking style") + + output_tokens[index] = masked_token + masked_lms.append(MaskedLmInstance(index=index, label=tokens[index])) + + masked_spans.append( + MaskedLmInstance(index=index_set, label=[tokens[index] for index in index_set]) + ) + + assert len(masked_lms) <= num_to_predict + np_rng.shuffle(ngram_indexes) + + select_indexes = set() + if do_permutation: + for cand_index_set in ngram_indexes: + if len(select_indexes) >= num_to_predict: + break + if not cand_index_set: + continue + # Note(mingdachen): + # Skip current piece if they are covered in lm masking or previous ngrams. + for index_set in cand_index_set[0]: + for index in index_set: + if index in covered_indexes or index in select_indexes: + continue + + n = np.random.choice( + ngrams[: len(cand_index_set)], + p=pvals[: len(cand_index_set)] / pvals[: len(cand_index_set)].sum(keepdims=True), + ) + index_set = sum(cand_index_set[n - 1], []) + n -= 1 + + while len(select_indexes) + len(index_set) > num_to_predict: + if n == 0: + break + index_set = sum(cand_index_set[n - 1], []) + n -= 1 + # If adding a whole-word mask would exceed the maximum number of + # predictions, then just skip this candidate. + if len(select_indexes) + len(index_set) > num_to_predict: + continue + is_any_index_covered = False + for index in index_set: + if index in covered_indexes or index in select_indexes: + is_any_index_covered = True + break + if is_any_index_covered: + continue + for index in index_set: + select_indexes.add(index) + assert len(select_indexes) <= num_to_predict + + select_indexes = sorted(select_indexes) + permute_indexes = list(select_indexes) + np_rng.shuffle(permute_indexes) + orig_token = list(output_tokens) + + for src_i, tgt_i in zip(select_indexes, permute_indexes): + output_tokens[src_i] = orig_token[tgt_i] + masked_lms.append(MaskedLmInstance(index=src_i, label=orig_token[src_i])) + + masked_lms = sorted(masked_lms, key=lambda x: x.index) + # Sort the spans by the index of the first span + masked_spans = sorted(masked_spans, key=lambda x: x.index[0]) + + for p in masked_lms: + masked_lm_positions.append(p.index) + masked_lm_labels.append(p.label) + return ( + output_tokens, + masked_lm_positions, + masked_lm_labels, + token_boundary, + masked_spans, + ) + + +def get_samples_mapping( + indexed_dataset, + data_prefix, + num_epochs, + max_num_samples, + max_seq_length, + short_seq_prob, + seed, + name, + binary_head, +): + """Get a list that maps a sample index to a starting sentence index, + end sentence index, and length""" + + if not num_epochs: + if not max_num_samples: + raise ValueError("Need to specify either max_num_samples " "or num_epochs") + num_epochs = np.iinfo(np.int32).max - 1 + if not max_num_samples: + max_num_samples = np.iinfo(np.int64).max - 1 + + # Filename of the index mapping + indexmap_filename = data_prefix + indexmap_filename += "_{}_indexmap".format(name) + if num_epochs != (np.iinfo(np.int32).max - 1): + indexmap_filename += "_{}ep".format(num_epochs) + if max_num_samples != (np.iinfo(np.int64).max - 1): + indexmap_filename += "_{}mns".format(max_num_samples) + indexmap_filename += "_{}msl".format(max_seq_length) + indexmap_filename += "_{:0.2f}ssp".format(short_seq_prob) + indexmap_filename += "_{}s".format(seed) + indexmap_filename += ".npy" + + # Build the indexed mapping if not exist. + # NOTE: use `get_local_rank() == 0` to promise samples will be build in each node. + if flow.env.get_local_rank() == 0 and not os.path.isfile(indexmap_filename): + logger.info( + " > WARNING: could not find index map file {}, building " + "the indices on rank 0 ...".format(indexmap_filename) + ) + + # Make sure the types match the helpers input types. + assert indexed_dataset.doc_idx.dtype == np.int64 + assert indexed_dataset.sizes.dtype == np.int32 + + # Build samples mapping + verbose = flow.env.get_local_rank() == 0 + start_time = time.time() + logger.info(" > building samples index mapping for {} ...".format(name)) + + # First compile and then import. + from libai.data.data_utils import helpers + + samples_mapping = helpers.build_mapping( + indexed_dataset.doc_idx, + indexed_dataset.sizes, + num_epochs, + max_num_samples, + max_seq_length, + short_seq_prob, + seed, + verbose, + 2 if binary_head else 1, + ) + logger.info(" > done building samples index maping") + np.save(indexmap_filename, samples_mapping, allow_pickle=True) + logger.info(" > saved the index mapping in {}".format(indexmap_filename)) + # Make sure all the ranks have built the mapping + logger.info( + " > elapsed time to build and save samples mapping " + "(seconds): {:4f}".format(time.time() - start_time) + ) + + # This should be a barrier but nccl barrier assumes + # device_index=rank which is not the case for model + # parallel case + dist.synchronize() + + # Load indexed dataset. + logger.info(" > loading indexed mapping from {}".format(indexmap_filename)) + start_time = time.time() + samples_mapping = np.load(indexmap_filename, allow_pickle=True, mmap_mode="r") + logger.info(" loaded indexed file in {:3.3f} seconds".format(time.time() - start_time)) + logger.info(" total number of samples: {}".format(samples_mapping.shape[0])) + + return samples_mapping + + +def get_train_valid_test_split_(size, splits=None): + """ + Split a dataset into subsets given proportions of how + much to allocate per split. If a split is 0% returns None for that split. + Purpose: Useful for creating train/val/test splits + + Arguments: + ds (Dataset or array-like): Data to be split. + split (1D array-like): proportions to split `ds`. `sum(splits) != 0` + """ + + if splits is None: + splits = [0.8, 0.2, 0.0] + + while len(splits) < 3: + splits.append(0.0) + splits = splits[:3] + splits_sum = sum(splits) + assert splits_sum > 0.0, "Split sum must be larger than 0." + splits = [split / splits_sum for split in splits] + splits_index = [0] + for index, split in enumerate(splits): + splits_index.append(splits_index[index] + int(round(split * float(size)))) + diff = splits_index[-1] - size + for index in range(1, len(splits_index)): + splits_index[index] -= diff + assert len(splits_index) == 4 + assert splits_index[-1] == size + return splits_index diff --git a/libai/data/data_utils/helpers b/libai/data/data_utils/helpers new file mode 100644 index 0000000000000000000000000000000000000000..70ad4a7a985e48a0d6e92a646086122b123732ee Binary files /dev/null and b/libai/data/data_utils/helpers differ diff --git a/libai/data/data_utils/helpers.cpp b/libai/data/data_utils/helpers.cpp new file mode 100644 index 0000000000000000000000000000000000000000..607e05675f25eb490ab3e6e8be672c289d3c78e6 --- /dev/null +++ b/libai/data/data_utils/helpers.cpp @@ -0,0 +1,607 @@ +/* + coding=utf-8 + Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/* Helper methods for fast index mapping builds */ + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace py = pybind11; +using namespace std; + +const int32_t LONG_SENTENCE_LEN = 512; + +void build_blending_indices(py::array_t& dataset_index, + py::array_t& dataset_sample_index, + const py::array_t& weights, const int32_t num_datasets, + const int64_t size, const bool verbose) { + /* Given multiple datasets and a weighting array, build samples + such that it follows those wieghts.*/ + + if (verbose) { std::cout << "> building indices for blendable datasets ..." << std::endl; } + + // Get the pointer access without the checks. + auto dataset_index_ptr = dataset_index.mutable_unchecked<1>(); + auto dataset_sample_index_ptr = dataset_sample_index.mutable_unchecked<1>(); + auto weights_ptr = weights.unchecked<1>(); + + // Initialize buffer for number of samples used for each dataset. + int64_t current_samples[num_datasets]; + for (int64_t i = 0; i < num_datasets; ++i) { current_samples[i] = 0; } + + // For each sample: + for (int64_t sample_idx = 0; sample_idx < size; ++sample_idx) { + // Determine where the max error in sampling is happening. + auto sample_idx_double = std::max(static_cast(sample_idx), 1.0); + int64_t max_error_index = 0; + double max_error = weights_ptr[0] * sample_idx_double - static_cast(current_samples[0]); + for (int64_t dataset_idx = 1; dataset_idx < num_datasets; ++dataset_idx) { + double error = weights_ptr[dataset_idx] * sample_idx_double + - static_cast(current_samples[dataset_idx]); + if (error > max_error) { + max_error = error; + max_error_index = dataset_idx; + } + } + + // Populate the indices. + dataset_index_ptr[sample_idx] = static_cast(max_error_index); + dataset_sample_index_ptr[sample_idx] = current_samples[max_error_index]; + + // Update the total samples. + current_samples[max_error_index] += 1; + } + + // print info + if (verbose) { + std::cout << " > sample ratios:" << std::endl; + for (int64_t dataset_idx = 0; dataset_idx < num_datasets; ++dataset_idx) { + auto ratio = static_cast(current_samples[dataset_idx]) / static_cast(size); + std::cout << " dataset " << dataset_idx << ", input: " << weights_ptr[dataset_idx] + << ", achieved: " << ratio << std::endl; + } + } +} + +py::array build_sample_idx(const py::array_t& sizes_, const py::array_t& doc_idx_, + const int32_t seq_length, const int32_t num_epochs, + const int64_t tokens_per_epoch) { + /* Sample index (sample_idx) is used for gpt2 like dataset for which + the documents are flattened and the samples are built based on this + 1-D flatten array. It is a 2D array with sizes [number-of-samples + 1, 2] + where [..., 0] contains the index into `doc_idx` and [..., 1] is the + starting offset in that document.*/ + + // Consistency checks. + assert(seq_length > 1); + assert(num_epochs > 0); + assert(tokens_per_epoch > 1); + + // Remove bound checks. + auto sizes = sizes_.unchecked<1>(); + auto doc_idx = doc_idx_.unchecked<1>(); + + // Mapping and it's length (1D). + int64_t num_samples = (num_epochs * tokens_per_epoch - 1) / seq_length; + int32_t* sample_idx = new int32_t[2 * (num_samples + 1)]; + + cout << " using:" << endl << std::flush; + cout << " number of documents: " << doc_idx_.shape(0) / num_epochs << endl + << std::flush; + cout << " number of epochs: " << num_epochs << endl << std::flush; + cout << " sequence length: " << seq_length << endl << std::flush; + cout << " total number of samples: " << num_samples << endl << std::flush; + + // Index into sample_idx. + int64_t sample_index = 0; + // Index into doc_idx. + int64_t doc_idx_index = 0; + // Begining offset for each document. + int32_t doc_offset = 0; + // Start with first document and no offset. + sample_idx[2 * sample_index] = doc_idx_index; + sample_idx[2 * sample_index + 1] = doc_offset; + ++sample_index; + + int count = 0; + while (sample_index <= num_samples) { + count++; + // Start with a fresh sequence. + int32_t remaining_seq_length = seq_length + 1; + while (remaining_seq_length != 0) { + // Get the document length. + auto doc_id = doc_idx[doc_idx_index]; + auto doc_length = sizes[doc_id] - doc_offset; + // And add it to the current sequence. + remaining_seq_length -= doc_length; + // If we have more than a full sequence, adjust offset and set + // remaining length to zero so we return from the while loop. + // Note that -1 here is for the same reason we have -1 in + // `_num_epochs` calculations. + if (remaining_seq_length <= 0) { + doc_offset += (remaining_seq_length + doc_length - 1); + remaining_seq_length = 0; + } else { + // Otherwise, start from the begining of the next document. + ++doc_idx_index; + doc_offset = 0; + } + } + // cout << "count: " << count << endl; + // Record the sequence. + sample_idx[2 * sample_index] = doc_idx_index; + sample_idx[2 * sample_index + 1] = doc_offset; + ++sample_index; + } + + // Method to deallocate memory. + py::capsule free_when_done(sample_idx, [](void* mem_) { + int32_t* mem = reinterpret_cast(mem_); + delete[] mem; + }); + + // Return the numpy array. + const auto byte_size = sizeof(int32_t); + return py::array(std::vector{num_samples + 1, 2}, // shape + {2 * byte_size, byte_size}, // C-style contiguous strides + sample_idx, // the data pointer + free_when_done); // numpy array references +} + +inline int32_t get_target_sample_len(const int32_t short_seq_ratio, const int32_t max_length, + std::mt19937& rand32_gen) { + /* Training sample length. */ + if (short_seq_ratio == 0) { return max_length; } + const auto random_number = rand32_gen(); + if ((random_number % short_seq_ratio) == 0) { return 2 + random_number % (max_length - 1); } + return max_length; +} + +template +py::array build_mapping_impl(const py::array_t& docs_, const py::array_t& sizes_, + const int32_t num_epochs, const uint64_t max_num_samples, + const int32_t max_seq_length, const double short_seq_prob, + const int32_t seed, const bool verbose, const int32_t min_num_sent) { + /* Build a mapping of (start-index, end-index, sequence-length) where + start and end index are the indices of the sentences in the sample + and sequence-length is the target sequence length. + */ + + // Consistency checks. + assert(num_epochs > 0); + assert(max_seq_length > 1); + assert(short_seq_prob >= 0.0); + assert(short_seq_prob <= 1.0); + assert(seed > 0); + + // Remove bound checks. + auto docs = docs_.unchecked<1>(); + auto sizes = sizes_.unchecked<1>(); + + // For efficiency, convert probability to ratio. Note: rand() generates int. + int32_t short_seq_ratio = 0; + if (short_seq_prob > 0) { short_seq_ratio = static_cast(round(1.0 / short_seq_prob)); } + + if (verbose) { + const auto sent_start_index = docs[0]; + const auto sent_end_index = docs[docs_.shape(0) - 1]; + const auto num_sentences = sent_end_index - sent_start_index; + cout << " using:" << endl << std::flush; + cout << " number of documents: " << docs_.shape(0) - 1 << endl << std::flush; + cout << " sentences range: [" << sent_start_index << ", " << sent_end_index + << ")" << endl + << std::flush; + cout << " total number of sentences: " << num_sentences << endl << std::flush; + cout << " number of epochs: " << num_epochs << endl << std::flush; + cout << " maximum number of samples: " << max_num_samples << endl << std::flush; + cout << " maximum sequence length: " << max_seq_length << endl << std::flush; + cout << " short sequence probability: " << short_seq_prob << endl << std::flush; + cout << " short sequence ration (1/prob): " << short_seq_ratio << endl << std::flush; + cout << " seed: " << seed << endl << std::flush; + } + + // Mapping and it's length (1D). + int64_t num_samples = -1; + DocIdx* maps = NULL; + + // Perform two iterations, in the first iteration get the size + // and allocate memory and in the second iteration populate the map. + bool second = false; + for (int32_t iteration = 0; iteration < 2; ++iteration) { + // Set the seed so both iterations produce the same results. + std::mt19937 rand32_gen(seed); + + // Set the flag on second iteration. + second = (iteration == 1); + + // Counters: + uint64_t empty_docs = 0; + uint64_t one_sent_docs = 0; + uint64_t long_sent_docs = 0; + + // Current map index. + uint64_t map_index = 0; + + // For each epoch: + for (int32_t epoch = 0; epoch < num_epochs; ++epoch) { + if (map_index >= max_num_samples) { + if (verbose && (!second)) { + cout << " reached " << max_num_samples << " samples after " << epoch << " epochs ..." + << endl + << std::flush; + } + break; + } + // For each document: + for (int32_t doc = 0; doc < (docs.shape(0) - 1); ++doc) { + // Document sentences are in [sent_index_first, sent_index_last) + const auto sent_index_first = docs[doc]; + const auto sent_index_last = docs[doc + 1]; + + // At the begining of the document previous index is the + // start index. + auto prev_start_index = sent_index_first; + + // Remaining documents. + auto num_remain_sent = sent_index_last - sent_index_first; + + // Some bookkeeping + if ((epoch == 0) && (!second)) { + if (num_remain_sent == 0) { ++empty_docs; } + if (num_remain_sent == 1) { ++one_sent_docs; } + } + + // Detect documents with long sentences. + bool contains_long_sentence = false; + if (num_remain_sent > 1) { + for (auto sent_index = sent_index_first; sent_index < sent_index_last; ++sent_index) { + if (sizes[sent_index] > LONG_SENTENCE_LEN) { + if ((epoch == 0) && (!second)) { ++long_sent_docs; } + contains_long_sentence = true; + break; + } + } + } + + // If we have more than two sentences. + if ((num_remain_sent >= min_num_sent) && (!contains_long_sentence)) { + // Set values. + auto seq_len = int32_t{0}; + auto num_sent = int32_t{0}; + auto target_seq_len = get_target_sample_len(short_seq_ratio, max_seq_length, rand32_gen); + + // Loop through sentences. + for (auto sent_index = sent_index_first; sent_index < sent_index_last; ++sent_index) { + // Add the size and number of sentences. + seq_len += sizes[sent_index]; + ++num_sent; + --num_remain_sent; + + // If we have reached the target length. + // and if not only one sentence is left in the document. + // and if we have at least two sentneces. + // and if we have reached end of the document. + if (((seq_len >= target_seq_len) && (num_remain_sent > 1) && (num_sent >= min_num_sent)) + || (num_remain_sent == 0)) { + // Check for overflow. + if ((3 * map_index + 2) > std::numeric_limits::max()) { + cout << "number of samples exceeded maximum " + << "allowed by type int64: " << std::numeric_limits::max() << endl; + throw std::overflow_error("Number of samples"); + } + + // Populate the map. + if (second) { + const auto map_index_0 = 3 * map_index; + maps[map_index_0] = static_cast(prev_start_index); + maps[map_index_0 + 1] = static_cast(sent_index + 1); + maps[map_index_0 + 2] = static_cast(target_seq_len); + } + + // Update indices / counters. + ++map_index; + prev_start_index = sent_index + 1; + target_seq_len = get_target_sample_len(short_seq_ratio, max_seq_length, rand32_gen); + seq_len = 0; + num_sent = 0; + } + + } // for (auto sent_index=sent_index_first; ... + } // if (num_remain_sent > 1) { + } // for (int doc=0; doc < num_docs; ++doc) { + } // for (int epoch=0; epoch < num_epochs; ++epoch) { + + if (!second) { + if (verbose) { + cout << " number of empty documents: " << empty_docs << endl << std::flush; + cout << " number of documents with one sentence: " << one_sent_docs << endl << std::flush; + cout << " number of documents with long sentences: " << long_sent_docs << endl + << std::flush; + cout << " will create mapping for " << map_index << " samples" << endl << std::flush; + } + assert(maps == NULL); + assert(num_samples < 0); + maps = new DocIdx[3 * map_index]; + num_samples = static_cast(map_index); + } + + } // for (int iteration=0; iteration < 2; ++iteration) { + + // Shuffle. + // We need a 64 bit random number generator as we might have more + // than 2 billion samples. + std::mt19937_64 rand64_gen(seed + 1); + for (auto i = (num_samples - 1); i > 0; --i) { + const auto j = static_cast(rand64_gen() % (i + 1)); + const auto i0 = 3 * i; + const auto j0 = 3 * j; + // Swap values. + swap(maps[i0], maps[j0]); + swap(maps[i0 + 1], maps[j0 + 1]); + swap(maps[i0 + 2], maps[j0 + 2]); + } + + // Method to deallocate memory. + py::capsule free_when_done(maps, [](void* mem_) { + DocIdx* mem = reinterpret_cast(mem_); + delete[] mem; + }); + + // Return the numpy array. + const auto byte_size = sizeof(DocIdx); + return py::array(std::vector{num_samples, 3}, // shape + {3 * byte_size, byte_size}, // C-style contiguous strides + maps, // the data pointer + free_when_done); // numpy array references +} + +py::array build_mapping(const py::array_t& docs_, const py::array_t& sizes_, + const int num_epochs, const uint64_t max_num_samples, + const int max_seq_length, const double short_seq_prob, const int seed, + const bool verbose, const int32_t min_num_sent) { + if (sizes_.size() > std::numeric_limits::max()) { + if (verbose) { cout << " using uint64 for data mapping..." << endl << std::flush; } + return build_mapping_impl(docs_, sizes_, num_epochs, max_num_samples, max_seq_length, + short_seq_prob, seed, verbose, min_num_sent); + } else { + if (verbose) { cout << " using uint32 for data mapping..." << endl << std::flush; } + return build_mapping_impl(docs_, sizes_, num_epochs, max_num_samples, max_seq_length, + short_seq_prob, seed, verbose, min_num_sent); + } +} + +template +py::array build_blocks_mapping_impl(const py::array_t& docs_, + const py::array_t& sizes_, + const py::array_t& titles_sizes_, + const int32_t num_epochs, const uint64_t max_num_samples, + const int32_t max_seq_length, const int32_t seed, + const bool verbose, const bool use_one_sent_blocks) { + /* Build a mapping of (start-index, end-index, sequence-length) where + start and end index are the indices of the sentences in the sample + and sequence-length is the target sequence length. + */ + + // Consistency checks. + assert(num_epochs > 0); + assert(max_seq_length > 1); + assert(seed > 0); + + // Remove bound checks. + auto docs = docs_.unchecked<1>(); + auto sizes = sizes_.unchecked<1>(); + auto titles_sizes = titles_sizes_.unchecked<1>(); + + if (verbose) { + const auto sent_start_index = docs[0]; + const auto sent_end_index = docs[docs_.shape(0) - 1]; + const auto num_sentences = sent_end_index - sent_start_index; + cout << " using:" << endl << std::flush; + cout << " number of documents: " << docs_.shape(0) - 1 << endl << std::flush; + cout << " sentences range: [" << sent_start_index << ", " << sent_end_index + << ")" << endl + << std::flush; + cout << " total number of sentences: " << num_sentences << endl << std::flush; + cout << " number of epochs: " << num_epochs << endl << std::flush; + cout << " maximum number of samples: " << max_num_samples << endl << std::flush; + cout << " maximum sequence length: " << max_seq_length << endl << std::flush; + cout << " seed: " << seed << endl << std::flush; + } + + // Mapping and its length (1D). + int64_t num_samples = -1; + DocIdx* maps = NULL; + + // Acceptable number of sentences per block. + int min_num_sent = 2; + if (use_one_sent_blocks) { min_num_sent = 1; } + + // Perform two iterations, in the first iteration get the size + // and allocate memory and in the second iteration populate the map. + bool second = false; + for (int32_t iteration = 0; iteration < 2; ++iteration) { + // Set the flag on second iteration. + second = (iteration == 1); + + // Current map index. + uint64_t map_index = 0; + + uint64_t empty_docs = 0; + uint64_t one_sent_docs = 0; + uint64_t long_sent_docs = 0; + // For each epoch: + for (int32_t epoch = 0; epoch < num_epochs; ++epoch) { + // assign every block a unique id + int32_t block_id = 0; + + if (map_index >= max_num_samples) { + if (verbose && (!second)) { + cout << " reached " << max_num_samples << " samples after " << epoch << " epochs ..." + << endl + << std::flush; + } + break; + } + // For each document: + for (int32_t doc = 0; doc < (docs.shape(0) - 1); ++doc) { + // Document sentences are in [sent_index_first, sent_index_last) + const auto sent_index_first = docs[doc]; + const auto sent_index_last = docs[doc + 1]; + const auto target_seq_len = max_seq_length - titles_sizes[doc]; + + // At the begining of the document previous index is the + // start index. + auto prev_start_index = sent_index_first; + + // Remaining documents. + auto num_remain_sent = sent_index_last - sent_index_first; + + // Some bookkeeping + if ((epoch == 0) && (!second)) { + if (num_remain_sent == 0) { ++empty_docs; } + if (num_remain_sent == 1) { ++one_sent_docs; } + } + // Detect documents with long sentences. + bool contains_long_sentence = false; + if (num_remain_sent >= min_num_sent) { + for (auto sent_index = sent_index_first; sent_index < sent_index_last; ++sent_index) { + if (sizes[sent_index] > LONG_SENTENCE_LEN) { + if ((epoch == 0) && (!second)) { ++long_sent_docs; } + contains_long_sentence = true; + break; + } + } + } + // If we have enough sentences and no long sentences. + if ((num_remain_sent >= min_num_sent) && (!contains_long_sentence)) { + // Set values. + auto seq_len = int32_t{0}; + auto num_sent = int32_t{0}; + + // Loop through sentences. + for (auto sent_index = sent_index_first; sent_index < sent_index_last; ++sent_index) { + // Add the size and number of sentences. + seq_len += sizes[sent_index]; + ++num_sent; + --num_remain_sent; + + // If we have reached the target length. + // and there are an acceptable number of sentences left + // and if we have at least the minimum number of sentences. + // or if we have reached end of the document. + if (((seq_len >= target_seq_len) && (num_remain_sent >= min_num_sent) + && (num_sent >= min_num_sent)) + || (num_remain_sent == 0)) { + // Populate the map. + if (second) { + const auto map_index_0 = 4 * map_index; + // Each sample has 4 items: the starting sentence index, ending sentence index, + // the index of the document from which the block comes (used for fetching titles) + // and the unique id of the block (used for creating block indexes) + + maps[map_index_0] = static_cast(prev_start_index); + maps[map_index_0 + 1] = static_cast(sent_index + 1); + maps[map_index_0 + 2] = static_cast(doc); + maps[map_index_0 + 3] = static_cast(block_id); + } + + // Update indices / counters. + ++map_index; + ++block_id; + prev_start_index = sent_index + 1; + seq_len = 0; + num_sent = 0; + } + } // for (auto sent_index=sent_index_first; ... + } // if (num_remain_sent > 1) { + } // for (int doc=0; doc < num_docs; ++doc) { + } // for (int epoch=0; epoch < num_epochs; ++epoch) { + + if (!second) { + if (verbose) { + cout << " number of empty documents: " << empty_docs << endl << std::flush; + cout << " number of documents with one sentence: " << one_sent_docs << endl << std::flush; + cout << " number of documents with long sentences: " << long_sent_docs << endl + << std::flush; + cout << " will create mapping for " << map_index << " samples" << endl << std::flush; + } + assert(maps == NULL); + assert(num_samples < 0); + maps = new DocIdx[4 * map_index]; + num_samples = static_cast(map_index); + } + + } // for (int iteration=0; iteration < 2; ++iteration) { + + // Shuffle. + // We need a 64 bit random number generator as we might have more + // than 2 billion samples. + std::mt19937_64 rand64_gen(seed + 1); + for (auto i = (num_samples - 1); i > 0; --i) { + const auto j = static_cast(rand64_gen() % (i + 1)); + const auto i0 = 4 * i; + const auto j0 = 4 * j; + // Swap values. + swap(maps[i0], maps[j0]); + swap(maps[i0 + 1], maps[j0 + 1]); + swap(maps[i0 + 2], maps[j0 + 2]); + swap(maps[i0 + 3], maps[j0 + 3]); + } + + // Method to deallocate memory. + py::capsule free_when_done(maps, [](void* mem_) { + DocIdx* mem = reinterpret_cast(mem_); + delete[] mem; + }); + + // Return the numpy array. + const auto byte_size = sizeof(DocIdx); + return py::array(std::vector{num_samples, 4}, // shape + {4 * byte_size, byte_size}, // C-style contiguous strides + maps, // the data pointer + free_when_done); // numpy array references +} + +py::array build_blocks_mapping(const py::array_t& docs_, const py::array_t& sizes_, + const py::array_t& titles_sizes_, const int num_epochs, + const uint64_t max_num_samples, const int max_seq_length, + const int seed, const bool verbose, const bool use_one_sent_blocks) { + if (sizes_.size() > std::numeric_limits::max()) { + if (verbose) { cout << " using uint64 for data mapping..." << endl << std::flush; } + return build_blocks_mapping_impl(docs_, sizes_, titles_sizes_, num_epochs, + max_num_samples, max_seq_length, seed, verbose, + use_one_sent_blocks); + } else { + if (verbose) { cout << " using uint32 for data mapping..." << endl << std::flush; } + return build_blocks_mapping_impl(docs_, sizes_, titles_sizes_, num_epochs, + max_num_samples, max_seq_length, seed, verbose, + use_one_sent_blocks); + } +} + +PYBIND11_MODULE(helpers, m) { + m.def("build_mapping", &build_mapping); + m.def("build_blocks_mapping", &build_blocks_mapping); + m.def("build_sample_idx", &build_sample_idx); + m.def("build_blending_indices", &build_blending_indices); +} diff --git a/libai/data/data_utils/helpers.cpython-39-x86_64-linux-gnu.so b/libai/data/data_utils/helpers.cpython-39-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..b130ef93e2ef10ea552ef2ed48588a07e25dd4dc Binary files /dev/null and b/libai/data/data_utils/helpers.cpython-39-x86_64-linux-gnu.so differ diff --git a/libai/data/data_utils/indexed_dataset.py b/libai/data/data_utils/indexed_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..7bbb9409fb2880f72496cc1a10f4f2c4a9654a86 --- /dev/null +++ b/libai/data/data_utils/indexed_dataset.py @@ -0,0 +1,603 @@ +# coding=utf-8 +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +# copied from fairseq/fairseq/data/indexed_dataset.py +# Removed IndexedRawTextDataset since it relied on Fairseq dictionary +# other slight modifications to remove fairseq dependencies +# Added document index to index file and made it accessible. +# An empty sentence no longer separates documents. + +import logging +import os +import shutil +import struct +import time +from functools import lru_cache +from itertools import accumulate + +import numpy as np +import oneflow as flow + +logger = logging.getLogger(__name__) + + +def __best_fitting_dtype(vocab_size=None): + if vocab_size is not None and vocab_size < 65500: + return np.uint16 + else: + return np.int32 + + +def get_available_dataset_impl(): + return ["lazy", "cached", "mmap"] + + +def infer_dataset_impl(path): + if IndexedDataset.exists(path): + with open(index_file_path(path), "rb") as f: + magic = f.read(8) + if magic == IndexedDataset._HDR_MAGIC: + return "cached" + elif magic == MMapIndexedDataset.Index._HDR_MAGIC[:8]: + return "mmap" + else: + return None + else: + logger.info(f"Dataset does not exist: {path}") + logger.info( + "Path should be a basename that both .idx and .bin can be " + "appended to get full filenames." + ) + return None + + +def make_builder(out_file, impl, vocab_size=None): + if impl == "mmap": + return MMapIndexedDatasetBuilder(out_file, dtype=__best_fitting_dtype(vocab_size)) + else: + return IndexedDatasetBuilder(out_file) + + +def make_dataset(path, impl, skip_warmup=False): + if not IndexedDataset.exists(path): + logger.info(f"Dataset does not exist: {path}") + logger.info( + "Path should be a basename that both .idx and .bin can be " + "appended to get full filenames." + ) + raise ValueError(f"Dataset does not exist: {path}") + if impl == "infer": + impl = infer_dataset_impl(path) + if impl == "lazy" and IndexedDataset.exists(path): + return IndexedDataset(path) + elif impl == "cached" and IndexedDataset.exists(path): + return IndexedCachedDataset(path) + elif impl == "mmap" and MMapIndexedDataset.exists(path): + return MMapIndexedDataset(path, skip_warmup) + logger.info(f"Unknown dataset implementation: {impl}") + return None + + +def dataset_exists(path, impl): + if impl == "mmap": + return MMapIndexedDataset.exists(path) + else: + return IndexedDataset.exists(path) + + +def read_longs(f, n): + a = np.empty(n, dtype=np.int64) + f.readinto(a) + return a + + +def write_longs(f, a): + f.write(np.array(a, dtype=np.int64)) + + +dtypes = { + 1: np.uint8, + 2: np.int8, + 3: np.int16, + 4: np.int32, + 5: np.int64, + 6: np.float32, + 7: np.double, + 8: np.uint16, +} + + +def code(dtype): + for k in dtypes.keys(): + if dtypes[k] == dtype: + return k + raise ValueError(dtype) + + +def index_file_path(prefix_path): + return prefix_path + ".idx" + + +def data_file_path(prefix_path): + return prefix_path + ".bin" + + +def create_doc_idx(sizes): + doc_idx = [0] + for i, s in enumerate(sizes): + if s == 0: + doc_idx.append(i + 1) + return doc_idx + + +class IndexedDataset(flow.utils.data.Dataset): + """Loader for IndexedDataset""" + + _HDR_MAGIC = b"TNTIDX\x00\x00" + + def __init__(self, path): + super().__init__() + self.path = path + self.data_file = None + self.read_index(path) + + def read_index(self, path): + with open(index_file_path(path), "rb") as f: + magic = f.read(8) + assert magic == self._HDR_MAGIC, ( + "Index file doesn't match expected format. " + "Make sure that --dataset-impl is configured properly." + ) + version = f.read(8) + assert struct.unpack("= self._len: + raise IndexError("index out of range") + + def __del__(self): + if self.data_file: + self.data_file.close() + + # @lru_cache(maxsize=8) + def __getitem__(self, idx): + if not self.data_file: + self.read_data(self.path) + if isinstance(idx, int): + i = idx + self.check_index(i) + tensor_size = self.sizes[self.dim_offsets[i] : self.dim_offsets[i + 1]] + a = np.empty(tensor_size, dtype=self.dtype) + self.data_file.seek(self.data_offsets[i] * self.element_size) + self.data_file.readinto(a) + return a + elif isinstance(idx, slice): + start, stop, step = idx.indices(len(self)) + if step != 1: + raise ValueError("Slices into indexed_dataset must be contiguous") + sizes = self.sizes[self.dim_offsets[start] : self.dim_offsets[stop]] + size = sum(sizes) + a = np.empty(size, dtype=self.dtype) + self.data_file.seek(self.data_offsets[start] * self.element_size) + self.data_file.readinto(a) + offsets = list(accumulate(sizes)) + sents = np.split(a, offsets[:-1]) + return sents + + def __len__(self): + return self._len + + def num_tokens(self, index): + return self.sizes[index] + + def size(self, index): + return self.sizes[index] + + @staticmethod + def exists(path): + return os.path.exists(index_file_path(path)) and os.path.exists(data_file_path(path)) + + @property + def supports_prefetch(self): + return False # avoid prefetching to save memory + + +class IndexedCachedDataset(IndexedDataset): + def __init__(self, path): + super().__init__(path) + self.cache = None + self.cache_index = {} + + @property + def supports_prefetch(self): + return True + + def prefetch(self, indices): + if all(i in self.cache_index for i in indices): + return + if not self.data_file: + self.read_data(self.path) + indices = sorted(set(indices)) + total_size = 0 + for i in indices: + total_size += self.data_offsets[i + 1] - self.data_offsets[i] + self.cache = np.empty(total_size, dtype=self.dtype) + ptx = 0 + self.cache_index.clear() + for i in indices: + self.cache_index[i] = ptx + size = self.data_offsets[i + 1] - self.data_offsets[i] + a = self.cache[ptx : ptx + size] + self.data_file.seek(self.data_offsets[i] * self.element_size) + self.data_file.readinto(a) + ptx += size + if self.data_file: + # close and delete data file after prefetch so we can pickle + self.data_file.close() + self.data_file = None + + # @lru_cache(maxsize=8) + def __getitem__(self, idx): + if isinstance(idx, int): + i = idx + self.check_index(i) + tensor_size = self.sizes[self.dim_offsets[i] : self.dim_offsets[i + 1]] + a = np.empty(tensor_size, dtype=self.dtype) + ptx = self.cache_index[i] + np.copyto(a, self.cache[ptx : ptx + a.size]) + return a + elif isinstance(idx, slice): + # Hack just to make this work, can optimizer later if necessary + sents = [] + for i in range(*idx.indices(len(self))): + sents.append(self[i]) + return sents + + +class IndexedDatasetBuilder(object): + element_sizes = { + np.uint8: 1, + np.int8: 1, + np.int16: 2, + np.int32: 4, + np.int64: 8, + np.float32: 4, + np.double: 8, + } + + def __init__(self, out_file, dtype=np.int32): + self.out_file = open(out_file, "wb") + self.dtype = dtype + self.data_offsets = [0] + self.dim_offsets = [0] + self.sizes = [] + self.element_size = self.element_sizes[self.dtype] + self.doc_idx = [0] + + def add_item(self, tensor): + bytes = self.out_file.write(np.array(tensor.numpy(), dtype=self.dtype)) + self.data_offsets.append(self.data_offsets[-1] + bytes / self.element_size) + for s in tensor.size(): + self.sizes.append(s) + self.dim_offsets.append(self.dim_offsets[-1] + len(tensor.size())) + + def end_document(self): + self.doc_idx.append(len(self.sizes)) + + def merge_file_(self, another_file): + index = IndexedDataset(another_file) + assert index.dtype == self.dtype + + begin = self.data_offsets[-1] + for offset in index.data_offsets[1:]: + self.data_offsets.append(begin + offset) + self.sizes.extend(index.sizes) + begin = self.dim_offsets[-1] + for dim_offset in index.dim_offsets[1:]: + self.dim_offsets.append(begin + dim_offset) + + with open(data_file_path(another_file), "rb") as f: + while True: + data = f.read(1024) + if data: + self.out_file.write(data) + else: + break + + def finalize(self, index_file): + self.out_file.close() + index = open(index_file, "wb") + index.write(b"TNTIDX\x00\x00") + index.write(struct.pack(" 1 + assert target_seq_length <= max_seq_length + + # Divide sample into two segments (A and B). + if binary_head: + tokens_a, tokens_b, is_next_random = get_a_and_b_segments(sample, np_rng) + else: + tokens_a = [] + for j in range(len(sample)): + tokens_a.extend(sample[j]) + tokens_b = [] + is_next_random = False + + # Truncate to `target_sequence_length`. + max_num_tokens = target_seq_length + truncate_segments(tokens_a, tokens_b, len(tokens_a), len(tokens_b), max_num_tokens, np_rng) + + # Build tokens and toketypes. + tokens, tokentypes = create_tokens_and_tokentypes(tokens_a, tokens_b, cls_id, sep_id) + + # Masking. + max_predictions_per_seq = masked_lm_prob * max_num_tokens + (tokens, masked_positions, masked_labels, _, _) = create_masked_lm_predictions( + tokenizer, + tokens, + vocab_id_list, + vocab_id_to_token_dict, + masked_lm_prob, + cls_id, + sep_id, + mask_id, + max_predictions_per_seq, + np_rng, + masking_style=masking_style, + ) + + # Padding. + tokens_np, tokentypes_np, labels_np, padding_mask_np, loss_mask_np = pad_and_convert_to_numpy( + tokens, tokentypes, masked_positions, masked_labels, pad_id, max_seq_length + ) + + train_sample = Instance( + input_ids=DistTensorData(flow.tensor(tokens_np)), + attention_mask=DistTensorData(flow.tensor(padding_mask_np)), + tokentype_ids=DistTensorData(flow.tensor(tokentypes_np)), + ns_labels=DistTensorData( + flow.tensor(int(is_next_random), dtype=flow.long), placement_idx=-1 + ), + lm_labels=DistTensorData(flow.tensor(labels_np), placement_idx=-1), + loss_mask=DistTensorData(flow.tensor(loss_mask_np), placement_idx=-1), + ) + + return train_sample + + +def pad_and_convert_to_numpy( + tokens, tokentypes, masked_positions, masked_labels, pad_id, max_seq_length +): + """Pad sequences and convert them to numpy.""" + + # Some checks. + num_tokens = len(tokens) + padding_length = max_seq_length - num_tokens + assert padding_length >= 0 + assert len(tokentypes) == num_tokens + assert len(masked_positions) == len(masked_labels) + + # Tokens and token types. + filler = [pad_id] * padding_length + tokens_np = np.array(tokens + filler, dtype=np.int64) + tokentypes_np = np.array(tokentypes + filler, dtype=np.int64) + + # Padding mask. + padding_mask_np = np.array([1] * num_tokens + [0] * padding_length, dtype=np.bool) + + # Lables and loss mask. + labels = [-1] * max_seq_length + loss_mask = [0] * max_seq_length + for i in range(len(masked_positions)): + assert masked_positions[i] < num_tokens + labels[masked_positions[i]] = masked_labels[i] + loss_mask[masked_positions[i]] = 1 + labels_np = np.array(labels, dtype=np.int64) + loss_mask_np = np.array(loss_mask, dtype=np.bool) + + return tokens_np, tokentypes_np, labels_np, padding_mask_np, loss_mask_np + + +def get_a_and_b_segments(sample, np_rng): + """Divide sample into a and b segments.""" + + # Number of sentences in the sample. + n_sentences = len(sample) + # Make sure we always have two sentences. + assert n_sentences > 1, "make sure each sample has at least two sentences." + + # First part: + # `a_end` is how many sentences go into the `A`. + a_end = 1 + if n_sentences >= 3: + # Note that randin in numpy is exclusive. + a_end = np_rng.randint(1, n_sentences) + tokens_a = [] + for j in range(a_end): + tokens_a.extend(sample[j]) + + # Second part: + tokens_b = [] + for j in range(a_end, n_sentences): + tokens_b.extend(sample[j]) + + # Random next: + is_next_random = False + if np_rng.random() < 0.5: + is_next_random = True + tokens_a, tokens_b = tokens_b, tokens_a + + return tokens_a, tokens_b, is_next_random + + +def truncate_segments(tokens_a, tokens_b, len_a, len_b, max_num_tokens, np_rng): + """Truncates a pair of sequences to a maximum sequence length.""" + assert len_a > 0 + if len_a + len_b <= max_num_tokens: + return False + while len_a + len_b > max_num_tokens: + if len_a > len_b: + len_a -= 1 + tokens = tokens_a + else: + len_b -= 1 + tokens = tokens_b + if np_rng.random() < 0.5: + del tokens[0] + else: + tokens.pop() + return True + + +def create_tokens_and_tokentypes(tokens_a, tokens_b, cls_id, sep_id): + """Merge segments A and B, add [CLS] and [SEP] and build tokentypes.""" + + tokens = [] + tokentypes = [] + # [CLS]. + tokens.append(cls_id) + tokentypes.append(0) + # Segment A. + for token in tokens_a: + tokens.append(token) + tokentypes.append(0) + # [SEP]. + tokens.append(sep_id) + tokentypes.append(0) + # Segment B. + for token in tokens_b: + tokens.append(token) + tokentypes.append(1) + if tokens_b: + # [SEP]. + tokens.append(sep_id) + tokentypes.append(1) + + return tokens, tokentypes diff --git a/libai/data/datasets/cifar.py b/libai/data/datasets/cifar.py new file mode 100644 index 0000000000000000000000000000000000000000..db249759ce51709fde93324c2ff41d78368a002e --- /dev/null +++ b/libai/data/datasets/cifar.py @@ -0,0 +1,96 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Optional + +import oneflow as flow +from flowvision import datasets + +from libai.data.structures import DistTensorData, Instance + + +class CIFAR10Dataset(datasets.CIFAR10): + r"""`CIFAR10 `_ Dataset in LiBai. + + Args: + + root (string): Root directory of dataset where directory + ``cifar-10-batches-py`` exists or will be saved to if download is set to True. + train (bool, optional): If True, creates dataset from training set, otherwise + creates from test set. + transform (callable, optional): A function/transform that takes in a PIL image + and returns a transformed version. E.g, ``transforms.RandomCrop`` + download (bool, optional): If true, downloads the dataset from the internet and + puts it in root directory. If the dataset is already downloaded, it will not be + downloaded again. + """ + + def __init__( + self, + root: str, + train: bool = True, + transform: Optional[Callable] = None, + download: bool = False, + **kwargs + ): + super(CIFAR10Dataset, self).__init__( + root=root, train=train, transform=transform, download=download, **kwargs + ) + + def __getitem__(self, index: int): + img, target = super().__getitem__(index) + data_sample = Instance( + images=DistTensorData(img, placement_idx=0), + labels=DistTensorData(flow.tensor(target, dtype=flow.long), placement_idx=-1), + ) + return data_sample + + +class CIFAR100Dataset(datasets.CIFAR100): + r"""`CIFAR100 `_ Dataset in LiBai. + + Args: + + root (string): Root directory of dataset where directory + ``cifar-10-batches-py`` exists or will be saved to if download is set to True. + train (bool, optional): If True, creates dataset from training set, otherwise + creates from test set. + transform (callable, optional): A function/transform that takes in a PIL image + and returns a transformed version. E.g, ``transforms.RandomCrop`` + download (bool, optional): If true, downloads the dataset from the internet and + puts it in root directory. If the dataset is already downloaded, it will not be + downloaded again. + dataset_name (str, optional): Name for the dataset as an identifier. E.g, ``cifar100`` + """ + + def __init__( + self, + root: str, + train: bool = True, + transform: Optional[Callable] = None, + download: bool = False, + **kwargs + ): + super(CIFAR100Dataset, self).__init__( + root=root, train=train, transform=transform, download=download, **kwargs + ) + + def __getitem__(self, index: int): + img, target = super().__getitem__(index) + data_sample = Instance( + images=DistTensorData(img, placement_idx=0), + labels=DistTensorData(flow.tensor(target, dtype=flow.long), placement_idx=-1), + ) + return data_sample diff --git a/libai/data/datasets/gpt_dataset.py b/libai/data/datasets/gpt_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..53244a252a31b9ac27264ac055f217c1b6e8e201 --- /dev/null +++ b/libai/data/datasets/gpt_dataset.py @@ -0,0 +1,296 @@ +# coding=utf-8 +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""GPT style dataset.""" + +import logging +import os +import time + +import numpy as np +import oneflow as flow + +from libai.data.structures import DistTensorData, Instance +from libai.utils import distributed as dist + +logger = logging.getLogger(__name__) + + +class GPT2Dataset(flow.utils.data.Dataset): + def __init__( + self, + name, + tokenizer, + data_prefix, + indexed_dataset, + max_num_samples, + max_seq_length, + seed=1234, + ): + + self.name = name + self.tokenizer = tokenizer + self.indexed_dataset = indexed_dataset + + documents = np.arange(start=0, stop=indexed_dataset.sizes.shape[0], step=1, dtype=np.int32) + + # Build index mappings. + self.doc_idx, self.sample_idx, self.shuffle_idx = _build_index_mappings( + self.name, + data_prefix, + documents, + self.indexed_dataset.sizes, + max_num_samples, + max_seq_length, + seed, + ) + + def __len__(self): + # -1 is due to data structure used to retrieve the index: + # sample i --> [sample_idx[i], sample_idx[i+1]) + return self.sample_idx.shape[0] - 1 + + def __getitem__(self, idx): + # Get the shuffled index. + idx = self.shuffle_idx[idx] + # Start and end documents and offsets. + doc_index_f = self.sample_idx[idx][0] + doc_index_l = self.sample_idx[idx + 1][0] + offset_f = self.sample_idx[idx][1] + offset_l = self.sample_idx[idx + 1][1] + # If we are within the same document, just extract the chunk. + if doc_index_f == doc_index_l: + sample = self.indexed_dataset.get( + self.doc_idx[doc_index_f], offset=offset_f, length=offset_l - offset_f + 1 + ) + else: + # Otherwise, get the rest of the initial document. + sample_list = [self.indexed_dataset.get(self.doc_idx[doc_index_f], offset=offset_f)] + # Loop over all in between documents and add the entire document. + for i in range(doc_index_f + 1, doc_index_l): + sample_list.append(self.indexed_dataset.get(self.doc_idx[i])) + # And finally add the relevant portion of last document. + sample_list.append( + self.indexed_dataset.get(self.doc_idx[doc_index_l], length=offset_l + 1) + ) + sample = np.concatenate(sample_list) + + input_ids = flow.tensor(np.array(sample[:-1], dtype=np.int64)) + lm_labels = flow.tensor(np.array(sample[1:], dtype=np.int64)) + sample = Instance( + input_ids=DistTensorData(input_ids), + labels=DistTensorData(lm_labels, placement_idx=-1), + ) + return sample + + +def _build_index_mappings(name, data_prefix, documents, sizes, num_samples, seq_length, seed): + """Build doc-idx, sample-idx, and shuffle-idx. + doc-idx: is an array (ordered) of documents to be used in training. + sample-idx: is the start document index and document offset for each + training sample. + shuffle-idx: maps the sample index into a random index into sample-idx. + """ + # Number of tokens in each epoch and number of required epochs. + tokens_per_epoch = _num_tokens(documents, sizes) + num_epochs = _num_epochs(tokens_per_epoch, seq_length, num_samples) + # rng state + np_rng = np.random.RandomState(seed=seed) + + # Filename of the index mappings. + _filename = data_prefix + _filename += "_{}_indexmap".format(name) + _filename += "_{}ns".format(num_samples) + _filename += "_{}sl".format(seq_length) + _filename += "_{}s".format(seed) + doc_idx_filename = _filename + "_doc_idx.npy" + sample_idx_filename = _filename + "_sample_idx.npy" + shuffle_idx_filename = _filename + "_shuffle_idx.npy" + + # Build the indexed mapping if not exist. + # NOTE: use `get_local_rank() == 0` to promise samples will be build in each node. + if flow.env.get_local_rank() == 0: + if ( + (not os.path.isfile(doc_idx_filename)) + or (not os.path.isfile(sample_idx_filename)) + or (not os.path.isfile(shuffle_idx_filename)) + ): + + logger.info( + " > WARNING: could not find index map files, building " "the indices on rank 0 ..." + ) + + # For the last epoch, decide whether include the entire epoch + # in the global shuffle or not. + + # If we need only one epoch, then separating last epoch does + # not mean anything. + if num_epochs == 1: + separate_last_epoch = False + logger.info(" > only one epoch required, setting " "separate_last_epoch to False") + + else: + # Get the number of samples for the last epoch + num_samples_from_epochs_minus_one = ( + (num_epochs - 1) * tokens_per_epoch - 1 + ) // seq_length + last_epoch_num_samples = num_samples - num_samples_from_epochs_minus_one + assert ( + last_epoch_num_samples >= 0 + ), "last epoch number of samples should be non-negative." + num_samples_per_epoch = (tokens_per_epoch - 1) // seq_length + assert last_epoch_num_samples < ( + num_samples_per_epoch + 1 + ), "last epoch number of samples exceeded max value." + # If we have less than 80% of the samples for the last epoch, + # separate out the epoch and treat it differently. + # Note: the 80% number is just based on common sense and can + # be adjusted if needed. + separate_last_epoch = last_epoch_num_samples < int(0.80 * num_samples_per_epoch) + if separate_last_epoch: + string = ( + " > last epoch number of samples ({}) is smaller " + "than 80% of number of samples per epoch ({}), " + "setting separate_last_epoch to True" + ) + else: + string = ( + " > last epoch number of samples ({}) is larger " + "than 80% of number of samples per epoch ({}), " + "setting separate_last_epoch to False" + ) + logger.info(string.format(last_epoch_num_samples, num_samples_per_epoch)) + + # doc-idx. + logger.info("start to build and save doc-idx mapping ...") + start_time = time.time() + doc_idx = _build_doc_idx(documents, num_epochs, np_rng, separate_last_epoch) + np.save(doc_idx_filename, doc_idx, allow_pickle=True) + logger.info( + " > elapsed time to build and save doc-idx mapping " + "(seconds): {:4f}".format(time.time() - start_time) + ) + # sample-idx. + + logger.info("start to build and save sample-idx mapping ...") + start_time = time.time() + + # Use C++ implementation for speed. + # First compile and then import. + from libai.data.data_utils import helpers + + assert doc_idx.dtype == np.int32 + assert sizes.dtype == np.int32 + sample_idx = helpers.build_sample_idx( + sizes, doc_idx, seq_length, num_epochs, tokens_per_epoch + ) + # sample_idx = _build_sample_idx(sizes, doc_idx, seq_length, + # num_epochs, tokens_per_epoch) + np.save(sample_idx_filename, sample_idx, allow_pickle=True) + logger.info( + " > elapsed time to build and save sample-idx mapping " + "(seconds): {:4f}".format(time.time() - start_time) + ) + # shuffle-idx. + start_time = time.time() + # -1 is due to data structure used to retrieve the index: + # sample i --> [sample_idx[i], sample_idx[i+1]) + if separate_last_epoch: + num_samples_ = num_samples_from_epochs_minus_one + else: + num_samples_ = sample_idx.shape[0] - 1 + shuffle_idx = _build_shuffle_idx(num_samples_, sample_idx.shape[0] - 1, np_rng) + np.save(shuffle_idx_filename, shuffle_idx, allow_pickle=True) + logger.info( + " > elapsed time to build and save shuffle-idx mapping" + " (seconds): {:4f}".format(time.time() - start_time) + ) + + # This should be a barrier but nccl barrier assumes + # device_index=rank which is not the case for model + # parallel case + dist.synchronize() + + # Load mappings. + start_time = time.time() + logger.info(" > loading doc-idx mapping from {}".format(doc_idx_filename)) + doc_idx = np.load(doc_idx_filename, allow_pickle=True, mmap_mode="r") + logger.info(" > loading sample-idx mapping from {}".format(sample_idx_filename)) + sample_idx = np.load(sample_idx_filename, allow_pickle=True, mmap_mode="r") + logger.info(" > loading shuffle-idx mapping from {}".format(shuffle_idx_filename)) + shuffle_idx = np.load(shuffle_idx_filename, allow_pickle=True, mmap_mode="r") + logger.info(" loaded indexed file in {:3.3f} seconds".format(time.time() - start_time)) + logger.info(" total number of samples: {}".format(sample_idx.shape[0])) + logger.info(" total number of epochs: {}".format(num_epochs)) + + return doc_idx, sample_idx, shuffle_idx + + +def _num_tokens(documents, sizes): + """Total number of tokens in the dataset.""" + return np.sum(sizes[documents]) + + +def _num_epochs(tokens_per_epoch, seq_length, num_samples): + """Based on number of samples and sequence length, calculate how many + epochs will be needed.""" + num_epochs = 0 + total_tokens = 0 + while True: + num_epochs += 1 + total_tokens += tokens_per_epoch + # -1 is because we need to retrieve seq_length + 1 token each time + # but the last token will overlap with the first token of the next + # sample except for the last sample. + if ((total_tokens - 1) // seq_length) >= num_samples: + return num_epochs + + +def _build_doc_idx(documents, num_epochs, np_rng, separate_last_epoch): + """Build an array with length = number-of-epochs * number-of-documents. + Each index is mapped to a corresponding document.""" + if not separate_last_epoch or num_epochs == 1: + doc_idx = np.mgrid[0:num_epochs, 0 : len(documents)][1] + doc_idx[:] = documents + doc_idx = doc_idx.reshape(-1) + doc_idx = doc_idx.astype(np.int32) + np_rng.shuffle(doc_idx) + return doc_idx + + doc_idx_first = _build_doc_idx(documents, num_epochs - 1, np_rng, False) + doc_idx_last = _build_doc_idx(documents, 1, np_rng, False) + return np.concatenate((doc_idx_first, doc_idx_last)) + + +def _build_shuffle_idx(num_samples, total_size, np_rng): + """Build the range [0, size) and shuffle.""" + logger.info( + " > building shuffle index with split [0, {}) and [{}, {}) " + "...".format(num_samples, num_samples, total_size) + ) + + dtype_ = np.uint32 + if total_size >= (np.iinfo(np.uint32).max - 1): + dtype_ = np.int64 + + shuffle_idx_first = np.arange(start=0, stop=num_samples, step=1, dtype=dtype_) + np_rng.shuffle(shuffle_idx_first) + if num_samples == total_size: + return shuffle_idx_first + + shuffle_idx_last = np.arange(start=num_samples, stop=total_size, step=1, dtype=dtype_) + np_rng.shuffle(shuffle_idx_last) + + return np.concatenate((shuffle_idx_first, shuffle_idx_last)) diff --git a/libai/data/datasets/imagenet.py b/libai/data/datasets/imagenet.py new file mode 100644 index 0000000000000000000000000000000000000000..d62294c33ce015110326d3a41f6e999c9d150446 --- /dev/null +++ b/libai/data/datasets/imagenet.py @@ -0,0 +1,50 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import Callable, Optional + +import oneflow as flow +from flowvision import datasets + +from libai.data.structures import DistTensorData, Instance + + +class ImageNetDataset(datasets.ImageFolder): + r"""`ImageNet `_ 2012 Classification Dataset in LiBai. + + Args: + + root (string): Root directory of the ImageNet Dataset. + train (bool, optional): If True, creates dataset from training set, otherwise + creates from test set. + transform (callable, optional): A function/transform that takes in a PIL image + and returns a transformed version. E.g, ``transforms.RandomCrop`` + """ + + def __init__( + self, root: str, train: bool = True, transform: Optional[Callable] = None, **kwargs + ): + prefix = "train" if train else "val" + root = os.path.join(root, prefix) + super(ImageNetDataset, self).__init__(root=root, transform=transform, **kwargs) + + def __getitem__(self, index: int): + sample, target = super().__getitem__(index) + data_sample = Instance( + images=DistTensorData(sample, placement_idx=0), + labels=DistTensorData(flow.tensor(target, dtype=flow.long), placement_idx=-1), + ) + return data_sample diff --git a/libai/data/datasets/mnist.py b/libai/data/datasets/mnist.py new file mode 100644 index 0000000000000000000000000000000000000000..3584430c2d04521e896ab71f1e2117c9ac9f7ab7 --- /dev/null +++ b/libai/data/datasets/mnist.py @@ -0,0 +1,58 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Optional + +import oneflow as flow +from flowvision import datasets + +from libai.data.structures import DistTensorData, Instance + + +class MNISTDataset(datasets.MNIST): + r"""`MNIST `_ Dataset in LiBai. + + Args: + root (string): Root directory of dataset where ``MNIST/processed/training.pt`` + and ``MNIST/processed/test.pt`` exist. + train (bool, optional): If True, creates dataset from ``training.pt``, + otherwise from ``test.pt``. + download (bool, optional): If true, downloads the dataset from the internet and + puts it in root directory. If the dataset is already downloaded, it will not be + downloaded again. + transform (callable, optional): A function/transform that takes in a PIL image + and returns a transformed version. E.g, ``transforms.RandomCrop`` + dataset_name (str, optional): Name for the dataset as an identifier. E.g, ``mnist`` + """ + + def __init__( + self, + root: str, + train: bool = True, + transform: Optional[Callable] = None, + download: bool = False, + **kwargs + ): + super(MNISTDataset, self).__init__( + root=root, train=train, transform=transform, download=download, **kwargs + ) + + def __getitem__(self, index: int): + img, target = super().__getitem__(index) + data_sample = Instance( + images=DistTensorData(img, placement_idx=0), + labels=DistTensorData(flow.tensor(target, dtype=flow.long), placement_idx=-1), + ) + return data_sample diff --git a/libai/data/datasets/roberta_dataset.py b/libai/data/datasets/roberta_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..eae432d60abd096b6d13d6e9dbe0f7d5fd8fd5ba --- /dev/null +++ b/libai/data/datasets/roberta_dataset.py @@ -0,0 +1,219 @@ +# coding=utf-8 +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Roberta Style dataset.""" + +import numpy as np +import oneflow as flow + +from libai.data.structures import DistTensorData, Instance + +from ..data_utils import create_masked_lm_predictions, get_samples_mapping +from .bert_dataset import pad_and_convert_to_numpy + + +class RobertaDataset(flow.utils.data.Dataset): + """Dataset containing sentence for RoBERTa training. + Each index corresponds to a randomly selected sentence. + + Args: + name: Name of dataset for clarification. + tokenizer: Tokenizer to use. + data_prefix: Path to the training dataset. + indexed_dataset: Indexed dataset to use. + max_seq_length: Maximum length of the sequence. All values are padded to + this length. Defaults to 512. + mask_lm_prob: Probability to mask tokens. Defaults to 0.15. + short_seq_prob: Probability of producing a short sequence. Defaults to 0.0. + max_predictions_per_seq: Maximum number of mask tokens in each sentence. Defaults to None. + seed: Seed for random number generator for reproducibility. Defaults to 1234. + """ + + def __init__( + self, + name, + tokenizer, + indexed_dataset, + data_prefix, + max_num_samples, + mask_lm_prob, + max_seq_length, + short_seq_prob=0.0, + seed=1234, + masking_style="bert", + ): + super().__init__() + + # Params to store. + self.name = name + self.seed = seed + self.masked_lm_prob = mask_lm_prob + self.max_seq_length = max_seq_length + self.masking_style = masking_style + + # Dataset. + self.indexed_dataset = indexed_dataset + + # Build the samples mapping. + self.samples_mapping = get_samples_mapping( + self.indexed_dataset, + data_prefix, + None, + max_num_samples, + self.max_seq_length - 2, # account for added tokens + short_seq_prob, + self.seed, + self.name, + binary_head=False, + ) + + # Vocab stuff. + self.tokenizer = tokenizer + self.vocab_id_list = list(tokenizer.get_vocab().values()) + self.vocab_id_to_token_dict = {v: k for k, v in tokenizer.get_vocab().items()} + + self.cls_id = tokenizer.cls_token_id + self.sep_id = tokenizer.sep_token_id + self.mask_id = tokenizer.mask_token_id + self.pad_id = tokenizer.pad_token_id + + def __len__(self): + return self.samples_mapping.shape[0] + + def __getitem__(self, idx): + start_idx, end_idx, seq_length = self.samples_mapping[idx] + sample = [self.indexed_dataset[i] for i in range(start_idx, end_idx)] + # Note that this rng state should be numpy and not python since + # python randint is inclusive whereas the numpy one is exclusive. + # We % 2**32 since numpy requires the seed to be between 0 and 2**32 - 1 + + np_rng = np.random.RandomState(seed=((self.seed + idx) % 2 ** 32)) + return build_training_sample( + self.tokenizer, + sample, + seq_length, + self.max_seq_length, # needed for padding + self.vocab_id_list, + self.vocab_id_to_token_dict, + self.cls_id, + self.sep_id, + self.mask_id, + self.pad_id, + self.masked_lm_prob, + np_rng, + masking_style=self.masking_style, + ) + + +def build_training_sample( + tokenizer, + sample, + target_seq_length, + max_seq_length, + vocab_id_list, + vocab_id_to_token_dict, + cls_id, + sep_id, + mask_id, + pad_id, + masked_lm_prob, + np_rng, + masking_style="bert", +): + """Build training sample. + + Arguments: + sample: A list of sentences in which each sentence is a list token ids. + target_seq_length: Desired sequence length. + max_seq_length: Maximum length of the sequence. All values are padded to + this length. + vocab_id_list: List of vocabulary ids. Used to pick a random id. + vocab_id_to_token_dict: A dictionary from vocab ids to text tokens. + cls_id: Start of example id. + sep_id: Separator id. + mask_id: Mask token id. + pad_id: Padding token id. + masked_lm_prob: Probability to mask tokens. + np_rng: Random number genenrator. Note that this rng state should be + numpy and not python since python randint is inclusive for + the upper bound whereas the numpy one is exclusive. + """ + assert target_seq_length <= max_seq_length + + tokens = [] + for j in range(len(sample)): + tokens.extend(sample[j]) + + max_num_tokens = target_seq_length + truncate_segments(tokens, len(tokens), max_num_tokens, np_rng) + + # create tokens and tokentypes + tokens, tokentypes = create_tokens_and_tokentypes(tokens, cls_id, sep_id) + + # Masking + max_predictions_per_seq = masked_lm_prob * max_num_tokens + (tokens, masked_positions, masked_labels, _, _) = create_masked_lm_predictions( + tokenizer, + tokens, + vocab_id_list, + vocab_id_to_token_dict, + masked_lm_prob, + cls_id, + sep_id, + mask_id, + max_predictions_per_seq, + np_rng, + masking_style=masking_style, + ) + + # Padding. + tokens_np, tokentypes_np, labels_np, padding_mask_np, loss_mask_np = pad_and_convert_to_numpy( + tokens, tokentypes, masked_positions, masked_labels, pad_id, max_seq_length + ) + + train_sample = Instance( + input_ids=DistTensorData(flow.tensor(tokens_np)), + attention_mask=DistTensorData(flow.tensor(padding_mask_np)), + tokentype_ids=DistTensorData(flow.tensor(tokentypes_np)), + lm_labels=DistTensorData(flow.tensor(labels_np), placement_idx=-1), + loss_mask=DistTensorData(flow.tensor(loss_mask_np), placement_idx=-1), + ) + + return train_sample + + +def truncate_segments(tokens, len_tokens, max_num_tokens, np_rng): + """Truncates a sequences to a maximum sequence length.""" + assert len_tokens > 0 + if len_tokens <= max_num_tokens: + return False + while len_tokens > max_num_tokens: + if np_rng.random() < 0.5: + del tokens[0] + else: + tokens.pop() + len_tokens -= 1 + return True + + +def create_tokens_and_tokentypes(tokens, cls_id, sep_id): + """Add [CLS] and [SEP] and build tokentypes.""" + # [CLS]. + tokens.insert(0, cls_id) + # [SPE]. + tokens.append(sep_id) + tokentypes = [0] * len(tokens) + + return tokens, tokentypes diff --git a/libai/data/datasets/t5_dataset.py b/libai/data/datasets/t5_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..5b9aaf1f73d98a7d902e1aee2c9f4c97ac555fcf --- /dev/null +++ b/libai/data/datasets/t5_dataset.py @@ -0,0 +1,347 @@ +# coding=utf-8 +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""T5 Style dataset.""" + +import collections + +import numpy as np +import oneflow as flow + +from libai.data.structures import DistTensorData, Instance + +from ..data_utils import create_masked_lm_predictions, get_samples_mapping + + +class T5Dataset(flow.utils.data.Dataset): + """ + Dataset containing sentences for T5 training. + + Args: + name: Name of dataset. + tokenizer: Tokenizer to use. + data_prefix (str): Path to the training dataset. + indexed_dataset: Indexed dataset to use. + max_seq_length (int, optional): Maximum length of the sequence passing into encoder. + All values are padded to this length. Defaults to 512. + max_seq_length_dec (int, optional): Maximum length of the sequence passing into decoder. + All values are padded to this length. Defaults to 128. + mask_lm_prob (float, optional): Probability to mask tokens. Defaults to 0.15. + max_preds_per_seq (int, optional): Maximum number of masked tokens in each sentence. + Defaults to None. + short_seq_prob (float, optional): + Probability of producing a short sequence. Defaults to 0.0. + seed (int, optional): + Seed for random number generator for reproducibility. Defaults to 1234. + """ + + def __init__( + self, + name, + tokenizer, + indexed_dataset, + data_prefix, + max_num_samples, + masked_lm_prob, + max_seq_length, + max_seq_length_dec, + short_seq_prob, + seed, + ): + # Params to store. + self.name = name + self.seed = seed + self.masked_lm_prob = masked_lm_prob + self.max_seq_length = max_seq_length + self.max_seq_length_dec = max_seq_length_dec + + # Dataset. + self.indexed_dataset = indexed_dataset + + # Build the samples mapping. + self.samples_mapping = get_samples_mapping( + self.indexed_dataset, + data_prefix, + None, + max_num_samples, + self.max_seq_length - 2, # account for added tokens + short_seq_prob, + self.seed, + self.name, + False, + ) + + # Vocab stuff. + self.tokenizer = tokenizer + tokenizer.add_tokens( + [tokenizer._bos_token, tokenizer._eos_token, *tokenizer._additional_special_tokens] + ) + vocab = tokenizer.get_vocab() + inv_vocab = {v: k for k, v in vocab.items()} + self.vocab_id_list = list(inv_vocab.keys()) + self.vocab_id_to_token_dict = inv_vocab + self.cls_id = vocab[tokenizer._cls_token] + self.sep_id = vocab[tokenizer._sep_token] + self.mask_id = vocab[tokenizer._mask_token] + self.pad_id = vocab[tokenizer._pad_token] + self.bos_id = vocab[tokenizer._bos_token] + self.eos_id = vocab[tokenizer._eos_token] + self.sentinel_tokens = [vocab[x] for x in tokenizer._additional_special_tokens] + assert len(self.sentinel_tokens) > 0 + + def __len__(self): + return self.samples_mapping.shape[0] + + def __getitem__(self, idx): + + start_index, end_index, seq_length = self.samples_mapping[idx] + sample = [] + for index in range(start_index, end_index): + sample.append(self.indexed_dataset[index]) + # Note that this rng state should be numpy and not python since + # python randint is inclusive whereas the numpy one is exclusive. + np_rng = np.random.RandomState(seed=(self.seed + idx)) + return build_training_sample( + self.tokenizer, + sample, + seq_length, + self.max_seq_length, # needed for padding + self.max_seq_length_dec, + self.vocab_id_list, + self.vocab_id_to_token_dict, + self.cls_id, + self.sep_id, + self.mask_id, + self.pad_id, + self.masked_lm_prob, + np_rng, + self.bos_id, + self.eos_id, + self.sentinel_tokens, + ) + + +def build_training_sample( + tokenizer, + sample, + target_seq_length, + max_seq_length, + max_seq_length_dec, + vocab_id_list, + vocab_id_to_token_dict, + cls_id, + sep_id, + mask_id, + pad_id, + masked_lm_prob, + np_rng, + bos_id=None, + eos_id=None, + sentinel_tokens=None, +): + """Build training sample. + + Arguments: + sample: A list of sentences in which each sentence is a list token ids. + target_seq_length: Desired sequence length. + max_seq_length: Maximum length of the sequence. All values are padded to + this length. + vocab_id_list: List of vocabulary ids. Used to pick a random id. + vocab_id_to_token_dict: A dictionary from vocab ids to text tokens. + cls_id: Start of example id. + sep_id: Separator id. + mask_id: Mask token id. + pad_id: Padding token id. + masked_lm_prob: Probability to mask tokens. + np_rng: Random number genenrator. Note that this rng state should be + numpy and not python since python randint is inclusive for + the opper bound whereas the numpy one is exclusive. + bos_id: start of decoder example id + eos_id: end of generation id + sentinel_tokens: unique value to be substituted for every replaced span + """ + + assert target_seq_length <= max_seq_length + + # flatten sentences into one list + tokens = [token for sentence in sample for token in sentence] + + # Truncate to `target_sequence_length`. + max_num_tokens = target_seq_length + len(tokens) > max_num_tokens + tokens = tokens[:max_num_tokens] + + # Masking. + max_predictions_per_seq = masked_lm_prob * max_num_tokens + (tokens, masked_positions, masked_labels, _, masked_spans) = create_masked_lm_predictions( + tokenizer, + tokens, + vocab_id_list, + vocab_id_to_token_dict, + masked_lm_prob, + cls_id, + sep_id, + mask_id, + max_predictions_per_seq, + np_rng, + max_ngrams=10, + geometric_dist=True, + masking_style="t5", + ) + + # Padding. + ( + tokens_enc, + tokens_dec_in, + labels, + enc_mask, + dec_mask, + enc_dec_mask, + loss_mask, + ) = pad_and_convert_to_numpy( + tokens, + masked_positions, + masked_labels, + pad_id, + max_seq_length, + max_seq_length_dec, + masked_spans, + bos_id, + eos_id, + sentinel_tokens, + ) + + sample = Instance( + encoder_input_ids=DistTensorData(tokens_enc), + decoder_input_ids=DistTensorData(tokens_dec_in), + encoder_attn_mask=DistTensorData(enc_mask), + decoder_attn_mask=DistTensorData(dec_mask), + encoder_decoder_attn_mask=DistTensorData(enc_dec_mask), + lm_labels=DistTensorData(labels, placement_idx=-1), + loss_mask=DistTensorData(loss_mask, placement_idx=-1), + ) + return sample + + +def pad_and_convert_to_numpy( + tokens, + masked_positions, + masked_labels, + pad_id, + max_seq_length, + max_seq_length_dec, + masked_spans=None, + bos_id=None, + eos_id=None, + sentinel_tokens=None, +): + """Pad sequences and convert them to numpy.""" + + sentinel_tokens = collections.deque(sentinel_tokens) + t5_input = [] + (t5_decoder_in, t5_decoder_out) = ([bos_id], []) + (start_index, end_index) = (0, None) + for span in masked_spans: + flag = sentinel_tokens.popleft() + + # Append the same tokens in decoder input and output + t5_decoder_in.append(flag) + t5_decoder_in.extend(span.label) + t5_decoder_out.append(flag) + t5_decoder_out.extend(span.label) + + end_index = span.index[0] + t5_input.extend(tokens[start_index:end_index]) + t5_input.append(flag) + + # the next start index is the token after the last span token + start_index = span.index[-1] + 1 + + # Add token to the t5_decoder_out + t5_decoder_out.append(eos_id) + + # Add the remaining tokens to the t5 input + t5_input.extend(tokens[start_index:]) + + # assert (len(t5_input) - len(masked_spans)) + \ + # (len(t5_decoder_in) - (len(masked_spans) + 1)) == len(tokens) + + # Some checks. + + # Encoder-side padding mask. + num_tokens = len(t5_input) + padding_length = max_seq_length - num_tokens + assert padding_length >= 0 + assert len(masked_positions) == len(masked_labels) + + # Tokens.. + filler = [pad_id] * padding_length + tokens_enc = np.array(t5_input + filler, dtype=np.int64) + + # Decoder-side padding mask. + num_tokens_dec = len(t5_decoder_in) + padding_length_dec = max_seq_length_dec - num_tokens_dec + assert padding_length_dec >= 0 + filler_dec = [pad_id] * padding_length_dec + tokens_dec_in = np.array(t5_decoder_in + filler_dec, dtype=np.int64) + + # Create attention masks + enc_mask = make_attention_mask(tokens_enc, tokens_enc) + enc_dec_mask = make_attention_mask(tokens_dec_in, tokens_enc) + dec_mask = make_attention_mask(tokens_dec_in, tokens_dec_in) + dec_mask = dec_mask * make_history_mask(tokens_dec_in) + + # Labels mask. + labels = t5_decoder_out + ([-1] * padding_length_dec) + labels = np.array(labels, dtype=np.int64) + + # Loss mask + loss_mask = ([1] * num_tokens_dec) + ([0] * padding_length_dec) + loss_mask = np.array(loss_mask, dtype=np.bool) + + tokens_enc = flow.tensor(tokens_enc, dtype=flow.long) + tokens_dec_in = flow.tensor(tokens_dec_in, dtype=flow.long) + labels = flow.tensor(labels, dtype=flow.long) + enc_mask = flow.tensor(enc_mask, dtype=flow.bool) + dec_mask = flow.tensor(dec_mask, dtype=flow.bool) + enc_dec_mask = flow.tensor(enc_dec_mask, dtype=flow.bool) + loss_mask = flow.tensor(loss_mask, dtype=flow.bool) + + return tokens_enc, tokens_dec_in, labels, enc_mask, dec_mask, enc_dec_mask, loss_mask + + +def make_attention_mask(source_block, target_block): + """ + Returns a 2-dimensional (2-D) attention mask + :param source_block: 1-D array + :param target_block: 1-D array + """ + mask = (target_block[None, :] >= 1) * (source_block[:, None] >= 1) + mask = mask.astype(np.int64) + # (source_length, target_length) + return mask + + +def make_history_mask(block): + length = block.shape[0] + arange = np.arange(length) + history_mask = ( + arange[ + None, + ] + <= arange[:, None] + ) + history_mask = history_mask.astype(np.int64) + return history_mask diff --git a/libai/data/samplers/__init__.py b/libai/data/samplers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8a749f08a1c0b39b008400bab5925d428c086eb9 --- /dev/null +++ b/libai/data/samplers/__init__.py @@ -0,0 +1,16 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .samplers import CyclicSampler, SingleRoundSampler diff --git a/libai/data/samplers/__pycache__/__init__.cpython-39.pyc b/libai/data/samplers/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0c860060aa15f2fb0daaf5efdd625f760d379383 Binary files /dev/null and b/libai/data/samplers/__pycache__/__init__.cpython-39.pyc differ diff --git a/libai/data/samplers/__pycache__/samplers.cpython-39.pyc b/libai/data/samplers/__pycache__/samplers.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..440dc59fbd8fbb3570190e7d794f7378204b4a7a Binary files /dev/null and b/libai/data/samplers/__pycache__/samplers.cpython-39.pyc differ diff --git a/libai/data/samplers/samplers.py b/libai/data/samplers/samplers.py new file mode 100644 index 0000000000000000000000000000000000000000..aac7fccb1c9445a4ec71826fa59e6dfe79f78dc7 --- /dev/null +++ b/libai/data/samplers/samplers.py @@ -0,0 +1,185 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +from oneflow.utils.data import Sampler + + +class CyclicSampler(Sampler): + """ + This sampler supports cyclic sampling, and it is also compatible with + non-data parallelism and data parallelism. + + Arguments: + dataset: dataset to be sampled. + micro_batch_size: batch size for per model instance. + global_batch_size is micro_batch_size times data_parallel_size. + shuffle: whether to shuffle the dataset. + consumed_samples: the number of samples that have been trained at the current time, + used for resuming training (default: ``0``). + data_parallel_rank: local rank for data parallelism. + data_parallel_size: the size of data parallelism. + seed: random seed, used for reproducing experiments (default: ``0``). + """ + + def __init__( + self, + dataset, + micro_batch_size, + shuffle=False, + consumed_samples=0, + data_parallel_rank=0, + data_parallel_size=1, + seed=0, + ): + self.dataset = dataset + self.data_size = len(self.dataset) + self.shuffle = shuffle + + self.data_parallel_rank = data_parallel_rank + self.data_parallel_size = data_parallel_size + self.micro_batch_size = micro_batch_size + self.actual_batch_size = self.micro_batch_size * self.data_parallel_size + self.data_size_per_epoch = self.data_size // self.actual_batch_size * self.micro_batch_size + self.consumed_samples = consumed_samples + + self.seed = seed + + def __iter__(self): + """divide the data into data_parallel_size buckets, + and shuffle it if `shuffle` is set to `True`. + Each processor samples from its own buckets and data_loader + will load the corresponding data. + """ + epoch = self.consumed_samples // self.data_size_per_epoch + current_epoch_samples = self.consumed_samples % self.data_size_per_epoch + batch = [] + + while True: + bucket_offset = current_epoch_samples // self.data_parallel_size + start_idx = self.data_parallel_rank * self.data_size_per_epoch + + if self.shuffle: + generator = flow.Generator() + generator.manual_seed(self.seed + epoch) + random_idx = flow.randperm(self.data_size_per_epoch, generator=generator).tolist() + indices = [start_idx + x for x in random_idx[bucket_offset:]] + else: + seq_idx = flow.arange(self.data_size_per_epoch).tolist() + indices = [start_idx + x for x in seq_idx[bucket_offset:]] + + epoch += 1 + + if hasattr(self.dataset, "supports_prefetch") and self.dataset.supports_prefetch: + self.dataset.prefetch(indices) + + for idx in indices: + batch.append(idx) + if len(batch) == self.micro_batch_size: + self.consumed_samples += self.actual_batch_size + yield batch + batch = [] + + current_epoch_samples = 0 + + def __len__(self): + return self.data_size + + def set_consumed_samples(self, consumed_samples): + """You can recover the training iteration by setting `consumed_samples`.""" + self.consumed_samples = consumed_samples + + def set_epoch(self, epoch): + """Used for restoring training status.""" + self.epoch = epoch + + +class SingleRoundSampler(Sampler): + """ + This sampler supports single round sampling, and it is also compatible with + non data parallelism and data parallelism. + + Arguments: + dataset: dataset to be sampled. + micro_batch_size: batch size for per model instance, global_batch_size + is micro_batch_size times data_parallel_size. + shuffle: whether to shuffle the dataset. + data_parallel_rank: local rank for data parallelism. + data_parallel_size: the size of data parallelism. + seed: random seed, used for reproducing experiments (default: ``0``). + drop_last: whether to drop the remaining data (default: ``False``). + """ + + def __init__( + self, + dataset, + micro_batch_size, + shuffle=False, + data_parallel_rank=0, + data_parallel_size=1, + seed=0, + drop_last=False, + ): + self.dataset = dataset + self.data_size = len(self.dataset) + self.shuffle = shuffle + + self.data_parallel_rank = data_parallel_rank + self.data_parallel_size = data_parallel_size + self.micro_batch_size = micro_batch_size + + self.seed = seed + self.drop_last = drop_last + + def __iter__(self): + bucket_size = self.data_size // self.data_parallel_size + remain = self.data_size % self.data_parallel_size + start_idx = self.data_parallel_rank * bucket_size + + if self.data_parallel_rank < remain: + bucket_size += 1 + start_idx += min(self.data_parallel_rank, remain) + + if self.shuffle: + generator = flow.Generator() + generator.manual_seed(self.seed) + random_idx = flow.randperm(bucket_size, generator=generator).tolist() + indices = [start_idx + x for x in random_idx] + else: + seq_idx = flow.arange(bucket_size).tolist() + indices = [start_idx + x for x in seq_idx] + + if hasattr(self.dataset, "supports_prefetch") and self.dataset.supports_prefetch: + self.dataset.prefetch(indices) + + batch = [] + for idx in indices: + batch.append(idx) + if len(batch) == self.micro_batch_size: + yield batch + batch = [] + + if not self.drop_last: + if self.data_parallel_rank >= remain and remain > 0: + batch.append(0) + if len(batch) > 0: + yield batch + + def __len__(self): + global_batch_size = self.micro_batch_size * self.data_parallel_size + if self.drop_last: + return self.data_size // global_batch_size + else: + return (self.data_size + global_batch_size - 1) // global_batch_size diff --git a/libai/data/structures.py b/libai/data/structures.py new file mode 100644 index 0000000000000000000000000000000000000000..380a8a1cebf0dc6f3c0b1e242f84f24f1f048af2 --- /dev/null +++ b/libai/data/structures.py @@ -0,0 +1,196 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import OrderedDict +from dataclasses import dataclass, field +from typing import Any, List + +import oneflow as flow + +from libai.utils import distributed as dist + + +@dataclass +class DistTensorData: + tensor: flow.Tensor + sbp_list: list = field(default_factory=lambda: ["split_0", "broadcast"]) + placement_idx: int = 0 + + # Tensor-like methods + def to_global(self, sbp=None, placement=None, device_type="cuda"): + if sbp is not None: + self.sbp = sbp + else: + sbp_list = [] + for sbp in self.sbp_list: + sbp = sbp.split("_") + if len(sbp) > 1: + # split dim + assert sbp[0] == "split" + split_dim = int(sbp[1]) + sbp_list.append(flow.sbp.split(split_dim)) + else: + sbp_sign = sbp[0] + sbp_list.append(getattr(flow.sbp, sbp_sign)) + self.sbp = dist.get_nd_sbp(sbp_list) + + if placement is not None: + self.tensor = self.tensor.to_global(sbp=self.sbp, placement=placement) + else: + # Convert local tensor to global tensor with default setting, + # if the placement parameter is not provided. + # When enable pipeline parallel training, + # all the devices will be grouped into several device groups + # and the model will be split into several stages. + # Each stage will be placed on the corresponding device group. + # For those tensors to be used in the last stage, + # we first convert them to global tensor by only retain those on the device group 0, + # then transfer the result to the last stage. + # We do that to make sure that all the tensors used by the model are all generated + # by the fist device group, in case that each device group containg + # some random augmentations to the tensors without setting the same global seed. + main_placement = dist.get_layer_placement(0, device_type) + self.tensor = self.tensor.to_global(sbp=self.sbp, placement=main_placement) + if self.placement_idx != 0: + self.tensor = self.tensor.to_global( + placement=dist.get_layer_placement(self.placement_idx, device_type) + ) + + @staticmethod + def stack(distTensor_lists: List["DistTensorData"]) -> "DistTensorData": + if not isinstance(distTensor_lists[0].tensor, flow.Tensor): + raise TypeError( + "DistTensorData.tensor must be a flow.Tensor, but got {}. " + "Please check the return values of `__getitem__` in dataset.".format( + type(distTensor_lists[0].tensor) + ) + ) + + assert len(distTensor_lists) > 0 + if len(distTensor_lists) == 1: + # TODO(l1aoxingyu): add inplace unsqueeze + # distTensor_lists[0].tensor.unsqueeze_(0) # add batch dim + distTensor_lists[0].tensor = distTensor_lists[0].tensor.unsqueeze(0) # add batch dim + return distTensor_lists[0] + + tensor_size = distTensor_lists[0].tensor.size() + sbp_list = distTensor_lists[0].sbp_list + placement_idx = distTensor_lists[0].placement_idx + tensors = [] + for data in distTensor_lists: + assert ( + data.tensor.size() == tensor_size + ), f"tensor shape is not equal, {data.tensor.size()} != {tensor_size}" + assert ( + data.sbp_list == sbp_list + ), f"sbp_list is not equal, {data.sbp_list} != {sbp_list}!" + assert ( + data.placement_idx == placement_idx + ), f"placement_idx is not equal, {data.placement_idx} != {placement_idx}" + tensors.append(data.tensor) + tensors = flow.stack(tensors, dim=0) + ret = DistTensorData(tensors, sbp_list=sbp_list, placement_idx=placement_idx) + return ret + + +class Instance: + """ + This class represents a instance with metadata as attributes. + It stores the attributes of an instance (e.g., image, tokens) as "fields". + + all other (non-filed) attributes of this class are considered private: + they must start with '_' and are not modifiable by a user. + + Some basic usage: + + 1. Set/get/check a field: + + .. code-block:: python + + instance.tokens = Metadata(...) + instance.mask = Metadata(...) + print(instance.tokens) + print(instance.has("mask")) # True + + 2. ``len(instance)`` returns the number of instance + """ + + def __init__(self, **kwargs): + + self._fields = OrderedDict() + for k, v in kwargs.items(): + self.set(k, v) + + def __setattr__(self, name: str, val: Any) -> None: + if name.startswith("_"): + super().__setattr__(name, val) + else: + self.set(name, val) + + def __getattr__(self, name: str): + if name == "_fields" or name not in self._fields: + raise AttributeError(f"Cannot find field '{name}' in the given Instance!") + return self._fields[name] + + def set(self, name: str, value: Any): + """ + Set the field named `name` to `value`. + """ + self._fields[name] = value + + def has(self, name: str): + return name in self._fields + + def remove(self, name: str): + del self._fields[name] + + def get(self, name: str): + return self._fields[name] + + def get_fields(self): + return self._fields + + def __len__(self): + return len(self._fields.keys()) + + def __iter__(self): + raise NotImplementedError("`Instances` object is not iterable!") + + @staticmethod + def stack(instance_lists: List["Instance"]) -> "Instance": + assert all(isinstance(i, Instance) for i in instance_lists) + assert len(instance_lists) > 0 + + ret = Instance() + for k in instance_lists[0]._fields.keys(): + values = [i.get(k) for i in instance_lists] + v0 = values[0] + if isinstance(v0, flow.Tensor): + values = flow.stack(values, dim=0) + elif isinstance(v0, list): + pass + elif hasattr(type(v0), "stack"): + values = type(v0).stack(values) + else: + raise ValueError("Unsupported type {} for stack.".format(type(v0))) + ret.set(k, values) + return ret + + def __str__(self): + s = self.__class__.__name__ + "(" + s += "fields=[{}]".format(", ".join((f"{k}: {v}" for k, v in self._fields.items()))) + return s + + __repr__ = __str__ diff --git a/libai/engine/__init__.py b/libai/engine/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6bbcf184ad8b707f039704ed84fcefd870fbfceb --- /dev/null +++ b/libai/engine/__init__.py @@ -0,0 +1,16 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .default import DefaultTrainer, default_setup diff --git a/libai/engine/__pycache__/__init__.cpython-39.pyc b/libai/engine/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..57e1c6bde418723cdb978cf8940bb4f55e2ef1b3 Binary files /dev/null and b/libai/engine/__pycache__/__init__.cpython-39.pyc differ diff --git a/libai/engine/__pycache__/default.cpython-39.pyc b/libai/engine/__pycache__/default.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f7d52828565e72bb77e2e4cb689e0c6be1a984b Binary files /dev/null and b/libai/engine/__pycache__/default.cpython-39.pyc differ diff --git a/libai/engine/__pycache__/hooks.cpython-39.pyc b/libai/engine/__pycache__/hooks.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c7b93254cdbb6cb912942158f6a1b4535671bcec Binary files /dev/null and b/libai/engine/__pycache__/hooks.cpython-39.pyc differ diff --git a/libai/engine/__pycache__/trainer.cpython-39.pyc b/libai/engine/__pycache__/trainer.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab10e5b666e6a73191a70e0eee170372461487f6 Binary files /dev/null and b/libai/engine/__pycache__/trainer.cpython-39.pyc differ diff --git a/libai/engine/default.py b/libai/engine/default.py new file mode 100644 index 0000000000000000000000000000000000000000..c5c8cf21fa9a7c83fe48aa7a369c6e039c7667a0 --- /dev/null +++ b/libai/engine/default.py @@ -0,0 +1,843 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import math +import os +import time +from collections import OrderedDict +from typing import Callable, Optional + +import oneflow as flow +from omegaconf import OmegaConf +from termcolor import colored + +from libai.config import LazyConfig, instantiate, try_get_key +from libai.data import Instance +from libai.engine import hooks +from libai.engine.trainer import EagerTrainer, GraphTrainer, TrainerBase +from libai.evaluation import inference_on_dataset, print_csv_format +from libai.models import build_graph, build_model +from libai.optim import build_optimizer +from libai.scheduler import build_lr_scheduler +from libai.tokenizer import build_tokenizer +from libai.utils import distributed as dist +from libai.utils.checkpoint import Checkpointer +from libai.utils.events import CommonMetricPrinter, JSONWriter, TensorboardXWriter +from libai.utils.logger import setup_logger + +# -------------------------------------------------------- +# References: +# https://github.com/facebookresearch/detectron2/blob/main/detectron2/engine/defaults.py +# -------------------------------------------------------- + + +def _highlight(code, filename): + try: + import pygments + except ImportError: + return code + + from pygments.formatters import Terminal256Formatter + from pygments.lexers import Python3Lexer, YamlLexer + + lexer = Python3Lexer() if filename.endswith(".py") else YamlLexer() + code = pygments.highlight(code, lexer, Terminal256Formatter(style="monokai")) + return code + + +def _check_batch_size(cfg): + train_micro_batch_size = try_get_key(cfg, "train.train_micro_batch_size", default=None) + global_batch_size = try_get_key(cfg, "train.global_batch_size", default=None) + num_accumulation_steps = try_get_key(cfg, "train.num_accumulation_steps", default=None) + + if train_micro_batch_size is not None and global_batch_size is not None: + if num_accumulation_steps is None: + if global_batch_size % (train_micro_batch_size * dist.get_data_parallel_size()) != 0: + raise ValueError( + f"global_batch_size {global_batch_size} must be divisible by " + "train_micro_batch_size * data_parallel_size " + f"({train_micro_batch_size} * {dist.get_data_parallel_size()})" + ) + + cfg.train.num_accumulation_steps = global_batch_size // ( + train_micro_batch_size * dist.get_data_parallel_size() + ) + + else: + if ( + global_batch_size + != train_micro_batch_size * dist.get_data_parallel_size() * num_accumulation_steps + ): + raise ValueError( + f"global_batch_size {global_batch_size} must equal to " + "train_micro_batch_size * data_parallel_size * num_accumulation_steps " + f"({train_micro_batch_size} * {dist.get_data_parallel_size()} * {num_accumulation_steps})" # noqa + ) + elif train_micro_batch_size is not None and global_batch_size is None: + if num_accumulation_steps is None: + cfg.train.num_accumulation_steps = 1 + + cfg.train.global_batch_size = ( + train_micro_batch_size + * dist.get_data_parallel_size() + * cfg.train.num_accumulation_steps + ) + elif train_micro_batch_size is None and global_batch_size is not None: + if num_accumulation_steps is None: + cfg.train.num_accumulation_steps = 1 + + if ( + global_batch_size % (dist.get_data_parallel_size() * cfg.train.num_accumulation_steps) + != 0 + ): + raise ValueError( + f"global_batch_size {global_batch_size} must be divisible by " + "data_parallel_size * num_accumulation_steps " + f"({dist.get_data_parallel_size()} * {cfg.train.num_accumulation_steps})" + ) + + cfg.train.train_micro_batch_size = global_batch_size // ( + dist.get_data_parallel_size() * cfg.train.num_accumulation_steps + ) + else: + raise ValueError("train_micro_batch_size and global_batch_size must be set either") + # Set total training samples. + cfg.train.samples = cfg.train.train_iter * cfg.train.global_batch_size + + +def _compile_dependencies(): + logger = logging.getLogger(__name__) + # ========================= + # Compile dataset C++ code. + # ========================= + # TODO: move this to ninja + if dist.get_local_rank() == 0: + start_time = time.time() + logger.info("> compiling dataset index builder ...") + from libai.data.data_utils import compile_helper + + compile_helper() + logger.info( + ">>> done with dataset index builder. Compilation time: {:.3f} " + "seconds".format(time.time() - start_time) + ) + + dist.synchronize() + if dist.get_local_rank() == 0: + logger.info( + ">>> done with compiling. " + "Compilation time: {:.3f} seconds".format(time.time() - start_time) + ) + + +def default_setup(cfg, args): + """ + Perform some basic common setups at the beginning of a job, including: + + 1. Set up the libai logger + 2. Log basic information about environment, cmdline arguments, and config + 3. Setup the distributed environment + 4. Setup tokenizer if it's an NLP related task + 5. Check batch_size + 6. Backup the config to the output directory + 7. Compile dependencies + + Args: + args (argparse.NameSpace): the command line arguments to be logged + """ + + output_dir = try_get_key(cfg, "train.output_dir") + if dist.is_main_process() and output_dir: + os.makedirs(output_dir, exist_ok=True) + + cfg.train.resume = args.resume + + rank = dist.get_rank() + logger = setup_logger(output_dir, distributed_rank=rank) + + logger.info("Rank of current process: {}. World size: {}".format(rank, dist.get_world_size())) + logger.info("Command line arguments: " + str(args)) + + if hasattr(args, "config_file") and args.config_file != "": + logger.info( + "Contents of args.config_file={}:\n{}".format( + args.config_file, + _highlight(open(args.config_file, "r").read(), args.config_file), + ) + ) + + dist.setup_dist_util(cfg.train.dist) + + _check_batch_size(cfg) + + if dist.is_main_process() and output_dir: + # Note: some of our scripts may expect the existence of + # config.yaml in output directory + path = os.path.join(output_dir, "config.yaml") + LazyConfig.save(cfg, path) + logger.info("Full config saved to {}".format(path)) + + flow.boxing.nccl.set_fusion_threshold_mbytes( + try_get_key(cfg, "train.nccl_fusion_threshold_mb", default=16) + ) + flow.boxing.nccl.set_fusion_max_ops_num( + try_get_key(cfg, "train.nccl_fusion_max_ops", default=24) + ) + + _compile_dependencies() + + +class DefaultTrainer(TrainerBase): + """ + A trainer with default training logic. Compared to `TrainerBase`, it + also contains the following logic: + + 1. Create model, optimizer, scheduler, dataloader from the given config. + 2. Load a checkpoint or `cfg.MODEL.WEIGHTS`, if exists. + 3. Register a few common hooks defined by the config. + + With standard features, it is created to simplify the **standard model training workflow** and + reduce code boilerplate for users who only need the standard training workflow. + + It means this class makes **many assumptions** about your training logic that + may easily become invalid in a new research. In fact, any assumptions beyond those made in the + :class:`TrainerBase` are too much for research. + + The code of this class has been annotated about restrictive assumptions it made. + When they do not work for you, you're encouraged to: + + 1. Overwrite methods of this class, OR: + 2. Use :class:`TrainerBase`, which only does minimal SGD training and + nothing else. You can then add your own hooks if needed. OR: + 3. Write your own training loop similar to ``tools/train_net.py``. + + Also note that the behavior of this class, like other functions/classes in + this file, is not stable, since it is meant to represent the "common default behavior". + It is only guaranteed to work well with the standard models and training workflow in libai. + To obtain more stable behavior, write your own training logic with other public APIs. + + + Examples: + + .. code-block:: python + + trainer = DefaultTrainer(cfg) + trainer.resume_or_load() # load last checkpoint or MODEL.WEIGHTS + trainer.train() + + Attributes: + scheduler: + checkpointer (Checkpointer): + cfg (omegaconf.dictconfig.DictConfig): + """ + + def __init__(self, cfg): + """ + Args: + cfg (omegaconf.dictconfig.DictConfig): + """ + super().__init__() + self.cfg = cfg + logger = logging.getLogger("libai") + + # setup_logger is not called for LiBai + if not logger.isEnabledFor(logging.INFO): + setup_logger() + + # Initialize tokenizer + self.tokenizer = self.build_tokenizer(cfg) + + self.start_iter = 0 + if cfg.train.resume: + save_file = os.path.join(cfg.train.output_dir, "last_checkpoint") + try: + with open(save_file, "r") as f: + last_saved = f.read().strip() + assert ( + last_saved != "model_final" + ), "model training has finished, check your model in train.output_dir" + self.start_iter = int(last_saved.split("_")[-1]) + 1 + except IOError: + # If file doesn't exist, maybe because it has just been deleted. + # We just set start_iter to 0. + self.start_iter = 0 + if cfg.graph.enabled: + cfg.dataloader.consumed_samples = self.start_iter * cfg.train.global_batch_size + else: + cfg.dataloader.consumed_samples = ( + self.start_iter * cfg.train.global_batch_size // cfg.train.num_accumulation_steps + ) + + self.train_loader = None + self.test_loader = [] + + train_loader, val_loader, test_loader = self.build_train_loader(cfg, self.tokenizer) + self.train_loader = train_loader + + if val_loader is not None: + self.test_loader.append(val_loader) + if test_loader is not None: + self.test_loader.append(test_loader) + + self.test_loader.extend(self.build_test_loader(cfg, self.tokenizer)) + + if cfg.train.rdma_enabled: + # set rdma + flow.env.init_rdma() + + # Automatically scale the hyperparams + self.auto_scale_hyperparams(cfg, self.train_loader) + + # Assume these objects must be constructed in this order. + dist.synchronize() + start_time = time.time() + logger.info("> Start building model...") + self.model = self.build_model(cfg) + + dist.synchronize() + logger.info( + ">>> done with building model. " + "Building time: {:.3f} seconds".format(time.time() - start_time) + ) + + self.optimizer = self.build_optimizer(cfg, self.model) + self.lr_scheduler = self.build_lr_scheduler(cfg, self.optimizer) + + if cfg.graph.enabled: + self.graph_train = self.build_graph( + cfg, self.model, self.optimizer, self.lr_scheduler, is_train=True + ) + self.graph_eval = self.build_graph(cfg, self.model, is_train=False) + self._trainer = GraphTrainer( + self.graph_train, self.train_loader, cfg.train.num_accumulation_steps + ) + else: + self._trainer = EagerTrainer( + self.model, self.train_loader, self.optimizer, cfg.train.num_accumulation_steps + ) + + # Assume no other objects need to be checkpointed. + # We can later make it checkpoint the stateful hooks + if cfg.graph.enabled: + self.checkpointer = Checkpointer( + # Assume you want to save checkpoints together with logs/statistics + self.model, + cfg.train.output_dir, + # In static graph mode, optimizer and scheduler state_dict will + # be saved with graph.state_dict(). + graph=self.graph_train, + # We print lr by `LRScheduler` hook, so we need to save/load eager lr_scheduler, + # otherwise, lr will be reset to initial state when resuming training. + lr_scheduler=self.lr_scheduler, + ) + else: + self.checkpointer = Checkpointer( + # Assume you want to save checkpoints together with logs/statistics + self.model, + cfg.train.output_dir, + optimizer=self.optimizer, + lr_scheduler=self.lr_scheduler, + ) + + # Loading checkpoint before dataloader construction, because + # dataloader needs to know the consumed iterations from + # the last breakpoint. + self.resume_or_load(cfg.train.resume) + cfg.train.start_iter = self.start_iter + + # global_batch_size = micro_batch_size * num_gpus * num_accumulation_steps + # When using gradient accumulation in graph mode, each run_step + # handle `global_batch_size` samples. + # When using gradient accumulation in eager mode, each run_step just handle + # `micro_batch_size * num_gpus` samples, so we need to divide `num_accumulation_steps` + # to get the actual `batch_size` for computing `throughput` and `consumed_samples` + self.global_batch_size = ( + cfg.train.global_batch_size + if cfg.graph.enabled + else cfg.train.global_batch_size // cfg.train.num_accumulation_steps + ) + self.max_iter = cfg.train.train_iter + + self.register_hooks(self.build_hooks()) + + def resume_or_load(self, resume=True): + """ + If `resume==True` and `cfg.train.output_dir` contains the last checkpoint (defined by + a `last_checkpoint` file), resume from the file. Resuming means loading all + available states (eg. optimizer and scheduler) and update iteration counter + from the checkpoint. ``cfg.train.load_weight`` will not be used. + Otherwise, this is considered as an independent training. The method will load model + weights from the file ``cfg.train.load_weight`` (but will not load other states) and start + from iteration 0. + + Args: + resume (bool): whether to do resume or not + """ + weight_path = self.cfg.train.load_weight + assert isinstance( + weight_path, str + ), f"cfg.train.load_weight:{self.cfg.train.load_weight} must be string" + if resume: + assert self.checkpointer.has_checkpoint() + # The checkpoint stores the training iteration that just finished, thus we start + # at the next iteration (or iter zero if there's no checkpoint). + assert self.start_iter == ( + self.checkpointer.resume_or_load(None, resume=True).get("iter", -1) + 1 + ) + elif len(weight_path) != 0: + assert os.path.isdir( + weight_path + ), f"cfg.train.load_weight:{self.cfg.train.load_weight} must be directory" + self.checkpointer.load(weight_path, checkpointables=[]) + + def build_hooks(self): + """ + Build a list of default hooks, including timing, evaluation, + checkpointing, lr scheduling, precise BN, writing events. + + Returns: + list[HookBase]: + """ + + ret = [ + hooks.IterationTimer(), + hooks.LRScheduler(), # for beauty lr scheduler printer in `nn.Graph` mode + hooks.PeriodicCheckpointer(self.checkpointer, self.cfg.train.checkpointer.period), + ] + + if self.cfg.train.evaluation.enabled: + assert self.cfg.train.evaluation.eval_iter > 0, "run_iter must be positive number" + + def test_and_save_results(): + model = self.graph_eval if self.cfg.graph.enabled else self.model + self._last_eval_results = self.test(self.cfg, self.test_loader, model) + return self._last_eval_results + + ret.append(hooks.EvalHook(self.cfg.train.evaluation.eval_period, test_and_save_results)) + ret.append( + hooks.BestCheckpointer( + self.cfg.train.evaluation.eval_period, + self.checkpointer, + val_metric=try_get_key( + self.cfg, "train.evaluation.eval_metric", default="Acc@1" + ), + mode=try_get_key(self.cfg, "train.evaluation.eval_mode", default="max"), + ) + ) + + if dist.is_main_process(): + # run writers in the end, so that evaluation metrics are written + ret.append(hooks.PeriodicWriter(self.build_writers(), self.cfg.train.log_period)) + return ret + + def build_writers(self): + """ + Build a list of writers to be used. By default it contains + writers that write metrics to the screen, + a json file, and a tensorboard event file respectively. + If you'd like a different list of writers, you can overwrite it in + your trainer. + + Returns: + list[EventWriter]: a list of :class:`EventWriter` objects. + + It is now implemented by: + + .. code-block:: python + + return [ + CommonMetricPrinter(self.global_batch_size, self.max_iter), + JSONWriter(os.path.join(self.cfg.train.output_dir, "metrics.json")), + TensorboardXWriter(self.cfg.train.output_dir), + ] + """ + # Assume the default print/log frequency. + return [ + # It may not always print what you want to see, since it prints "common" metrics only. + CommonMetricPrinter(self.global_batch_size, self.max_iter), + JSONWriter(os.path.join(self.cfg.train.output_dir, "metrics.json")), + TensorboardXWriter(self.cfg.train.output_dir), + ] + + def train(self): + """ + Run training. + + Returns: + OrderedDict of results, if evaluation is enabled. Otherwise None. + """ + super().train(self.start_iter, self.max_iter) + + def run_step(self): + self._trainer.iter = self.iter + self._trainer.run_step(self.get_batch, self.cfg.train.input_placement_device) + + @classmethod + def get_batch( + cls, + data: Instance, + input_placement_device: str = "cuda", + mixup_func: Optional[Callable] = None, + ): + """ + Convert batched local tensor to distributed tensor for model step running. + + If you want to do something with batched data before model, (e.g. mixup), + you can rewrite this function. + """ + if isinstance(data, flow.utils.data._utils.worker.ExceptionWrapper): + data.reraise() + + if mixup_func is not None: + images, labels = mixup_func( + data.get("images").tensor.cuda(), + data.get("labels").tensor.cuda(), + ) + data.get("images").tensor = images + data.get("labels").tensor = labels + + ret_dict = {} + for key, value in data.get_fields().items(): + value.to_global(device_type=input_placement_device) + ret_dict[key] = value.tensor + return ret_dict + + @classmethod + def build_tokenizer(cls, cfg): + """ + Returns: + libai.tokenizer.PreTrainedTokenizer: + + It now calls :func:`libai.tokenizer.build_tokenizer`. + """ + tokenizer = None + if try_get_key(cfg, "tokenization") is not None: + tokenizer = build_tokenizer(cfg.tokenization) + # FIXME(lxy): In case model is not defined with cfg, the `vocab_size` can be + # accessed by `model.vocab_size`. + if try_get_key(cfg, "model.cfg.vocab_size", default=None) is not None: + # In case the model does not need vocab_size as argument + multiple = ( + cfg.tokenization.make_vocab_size_divisible_by + * cfg.train.dist.tensor_parallel_size + ) + cfg.model.cfg.vocab_size = tokenizer.padded_vocab_size(multiple) + return tokenizer + + @classmethod + def build_model(cls, cfg): + """ + Returns: + flow.nn.Module: + + It now calls :func:`libai.models.build_model`. + Overwrite it if you'd like a different model. + """ + assert try_get_key(cfg, "model") is not None, "cfg must contain `model` namespace" + # Set model fp16 option because of embedding layer `white_identity` manual + # insert for amp training if provided. + if try_get_key(cfg.model, "cfg.amp_enabled") is not None: + cfg.model.cfg.amp_enabled = cfg.train.amp.enabled and cfg.graph.enabled + # In case some model define without cfg keyword. + elif try_get_key(cfg.model, "amp_enabled") is not None: + cfg.model.amp_enabled = cfg.train.amp.enabled and cfg.graph.enabled + model = build_model(cfg.model) + logger = logging.getLogger(__name__) + logger.info("Model:\n{}".format(model)) + model._apply(dist.convert_to_distributed_default_setting) + return model + + @classmethod + def build_graph(cls, cfg, model, optimizer=None, lr_scheduler=None, is_train=True): + assert try_get_key(cfg, "graph") is not None, "cfg must contain `graph` namespace" + graph = build_graph(cfg, model, optimizer, lr_scheduler, is_train) + debug_graph = try_get_key(cfg, "graph.debug", default=-1) + if debug_graph >= 0: + logger = logging.getLogger(__name__) + logger.info("Graph debug mode on, automatically output debug info.") + graph.debug(cfg.graph.debug) + return graph + + @classmethod + def build_optimizer(cls, cfg, model): + """ + Returns: + flow.optim.Optimizer: + + It now calls :func:`libai.optim.build_optimizer`. + Overwrite it if you'd like a different optimizer. + """ + assert try_get_key(cfg, "optim") is not None, "cfg must contain `optim` namespace" + return build_optimizer(cfg.optim, model) + + @classmethod + def build_lr_scheduler(cls, cfg, optimizer): + """ + It now calls :func:`libai.scheduler.build_lr_scheduler`. + Overwrite it if you'd like a different scheduler. + """ + assert ( + try_get_key(cfg, "train.scheduler") is not None + ), "cfg.train must contain `scheduler` namespace" + return build_lr_scheduler(cfg.train.scheduler, optimizer) + + @classmethod + def build_train_loader(cls, cfg, tokenizer=None): + """ + Returns: + iterable + + It now calls :func:`libai.data.build_train_valid_test_loader`. + Overwrite it if you'd like a different data loader. + """ + assert ( + try_get_key(cfg, "dataloader.train") is not None + ), "cfg must contain `dataloader.train` namespace" + logger = logging.getLogger(__name__) + logger.info("Prepare training, validating, testing set") + if cfg.graph.enabled: + # In static graph mode, data will be sliced in nn.Graph automatically, + # dataloader will get micro-batch-size and data will be concated + # in graph_trainer.run_step to get mini-batch-size. + cfg.dataloader.train.train_batch_size = cfg.train.train_micro_batch_size + else: + # In eager mode, gradient accumulation will act like PyTorch, so dataloader + # will get micro-batch-size + cfg.dataloader.train.train_batch_size = cfg.train.train_micro_batch_size + cfg.dataloader.train.test_batch_size = cfg.train.test_micro_batch_size + cfg.dataloader.train.seed = cfg.train.seed + + # used by nlp dataloader + if hasattr(cfg.dataloader.train, "train_val_test_num_samples"): + eval_iter = ( + (cfg.train.train_iter // cfg.train.evaluation.eval_period + 1) + * cfg.train.evaluation.eval_iter + if cfg.train.evaluation.enabled + # samples for test_dataset must be larger than 0 even if there is no evaluation + else 1 + ) + test_iter = cfg.train.evaluation.eval_iter if cfg.train.evaluation.enabled else 1 + + cfg.dataloader.train.train_val_test_num_samples = [ + int(cfg.train.samples), + int(eval_iter * cfg.train.test_micro_batch_size * dist.get_data_parallel_size()), + int(test_iter * cfg.train.test_micro_batch_size * dist.get_data_parallel_size()), + ] + + if OmegaConf.is_list(cfg.dataloader.train.dataset): + for dataset in cfg.dataloader.train.dataset: + if hasattr(dataset, "seed"): + dataset.seed = cfg.train.seed + else: + dataset = cfg.dataloader.train.dataset + if hasattr(dataset, "seed"): + dataset.seed = cfg.train.seed + + # Set tokenizer for each dataset + if tokenizer: + if OmegaConf.is_list(cfg.dataloader.train.dataset): + for dataset in cfg.dataloader.train.dataset: + dataset.tokenizer = tokenizer + else: + cfg.dataloader.train.dataset.tokenizer = tokenizer + + train_loader, valid_loader, test_loader = instantiate( + cfg.dataloader.train, _recursive_=False + ) + return train_loader, valid_loader, test_loader + + @classmethod + def build_test_loader(cls, cfg, tokenizer=None): + """ + Returns: + iterable + + It now calls :func:`libai.data.build_image_test_loader` for CV tasks + or :func:`libai.data.build_nlp_test_loader` for NLP tasks. + Overwrite it if you'd like a different data loader. + """ + # If there is no test_loader, just return [] + if not try_get_key(cfg, "dataloader.test", default=False): + return [] + logger = logging.getLogger(__name__) + logger.info("Prepare testing set") + assert OmegaConf.is_list( + cfg.dataloader.test + ), f"dataloader.test must be list but got type of {type(cfg.dataloader.test)}" + for i in range(len(cfg.dataloader.test)): + cfg.dataloader.test[i].test_batch_size = cfg.train.test_micro_batch_size + cfg.dataloader.test[i].seed = cfg.train.seed # set seed + if tokenizer: + cfg.dataloader.test[i].dataset.tokenizer = tokenizer + # list[dataloader1, dataloader2, ...] + test_loader = instantiate(cfg.dataloader.test, _recursive_=False) + return test_loader + + @classmethod + def auto_scale_hyperparams(cls, cfg, data_loader): + logger = logging.getLogger(__name__) + log_info = "" + + # Get or set default iteration cfg + train_iter = try_get_key(cfg, "train.train_iter", default=0) + train_epoch = try_get_key(cfg, "train.train_epoch", default=0) + warmup_ratio = try_get_key(cfg, "train.warmup_ratio", default=0) + assert ( + warmup_ratio < 1 and warmup_ratio >= 0 + ), "warmup_ratio must be in [0, 1) that presents the ratio of warmup iter to the train iter" + + # Automatically scale iteration num depend on the settings + # The total iters in one epoch is `len(dataset) / global_batch_size` + cfg.train.train_iter = max( + math.ceil(len(data_loader.dataset) * train_epoch / cfg.train.global_batch_size), + train_iter, + ) + cfg.train.warmup_iter = math.ceil(cfg.train.train_iter * cfg.train.warmup_ratio) + if not cfg.graph.enabled: + # In eager mode, dataloader only get micro-batch-size each iter, + # which is mini-batch-size // num_accumulation, so scale `train_iter` + # and `warmup_iter` to be consistent with static graph mode. + cfg.train.train_iter *= cfg.train.num_accumulation_steps + cfg.train.warmup_iter *= cfg.train.num_accumulation_steps + log_info += "Auto-scaling the config to train.train_iter={}, train.warmup_iter={}".format( + cfg.train.train_iter, cfg.train.warmup_iter + ) + + # Automatically scale the milestones + if try_get_key(cfg, "train.scheduler.milestones"): + if len( + [ + milestone + for milestone in cfg.train.scheduler.milestones + if milestone < 0 or milestone >= 1 + ] + ): + raise ValueError( + "milestones should be a list of increasing ratio in [0, 1), but got {}".format( + cfg.train.scheduler.milestones + ) + ) + cfg.train.scheduler.milestones = [ + int(milestone * cfg.train.train_iter) + for milestone in cfg.train.scheduler.milestones + ] + log_info += f", scheduler milestones={cfg.train.scheduler.milestones}" + logger.info(log_info) + + # Global scheduler cfg + cfg.train.scheduler.warmup_iter = cfg.train.warmup_iter + cfg.train.scheduler.max_iter = cfg.train.train_iter + + # train iter per epoch + iter_per_epoch = len(data_loader.dataset) // cfg.train.global_batch_size + + # rescale eval period + if try_get_key(cfg, "train.evaluation.eval_after_n_epoch"): + cfg.train.evaluation.eval_period = ( + iter_per_epoch * cfg.train.evaluation.eval_after_n_epoch + ) + logger.info( + f"Auto-scaling the config " + f"train.evaluation.eval_after_n_epoch={cfg.train.evaluation.eval_after_n_epoch} " + f"to train.evaluation.eval_period={cfg.train.evaluation.eval_period}" + ) + + # rescale save model period + if try_get_key(cfg, "train.checkpointer.save_model_after_n_epoch"): + cfg.train.checkpointer.period = ( + iter_per_epoch * cfg.train.checkpointer.save_model_after_n_epoch + ) + logger.info( + f"Auto-scaling the config " + f"train.checkpointer.save_model_after_n_epoch=" + f"{cfg.train.checkpointer.save_model_after_n_epoch} " + f"to train.checkpointer.period={cfg.train.checkpointer.period}" + ) + + @classmethod + def build_evaluator(cls, cfg): + evaluator = instantiate(cfg.train.evaluation.evaluator) + return evaluator + + @classmethod + def test(cls, cfg, test_loaders, model, evaluator=None): + """ + Evaluate the given model. The given model is expected to already contain + weights to evaluate. + + Args: + cfg (CfgNode): + test_loaders: list [dataloader1, dataloader2, ...] + model (nn.Graph): + evaluators (list[DatasetEvaluator] or None): if None, will call + :meth:`build_evaluator`. Otherwise, must have the same length as + ``cfg.DATASETS.TEST``. + + Returns: + dict: a dict of result metrics + """ + logger = logging.getLogger(__name__) + # TODO: support multi evaluator + # if isinstance(evaluators, DatasetEvaluator): + # evaluators = [evaluators] + test_batch_size = cfg.train.test_micro_batch_size * dist.get_data_parallel_size() + evaluator = cls.build_evaluator(cfg) if not evaluator else evaluator + + results = OrderedDict() + for idx, data_loader in enumerate(test_loaders): + # When evaluators are passed in as arguments, + # implicitly assume that evaluators can be created before data_loader. + dataset_name = type(data_loader.dataset).__name__ + # TODO: support multi evaluator + # if evaluators is not None: + # evaluator = evaluators[idx] + # else: + # try: + # evaluator = cls.build_evaluator(cfg) + # except NotImplementedError: + # logger.warn( + # "No evaluator found. Use `DefaultTrainer.test(evaluators=)`, " + # "or implement its `build_evaluator` method." + # ) + # results[dataset_name] = {} + # continue + results_i = inference_on_dataset( + model, + data_loader, + test_batch_size, + cfg.train.evaluation.eval_iter, + cls.get_batch, + cfg.train.input_placement_device, + evaluator, + ) + results[dataset_name] = results_i + if dist.is_main_process(): + assert isinstance( + results_i, dict + ), "Evaluator must return a dict on the main process. Got {} instead.".format( + results_i + ) + logger.info( + "Evaluation results for {} in csv format:".format( + colored(dataset_name, "green") + ) + ) + print_csv_format(results_i) + + if len(results) == 1: + results = list(results.values())[0] + return results diff --git a/libai/engine/hooks.py b/libai/engine/hooks.py new file mode 100644 index 0000000000000000000000000000000000000000..a66cf4155376b5390045205f1244bb2bbbff9c9c --- /dev/null +++ b/libai/engine/hooks.py @@ -0,0 +1,417 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import logging +import math +import operator +import time +from collections import Counter + +import oneflow as flow + +from libai.evaluation import flatten_results_dict +from libai.utils import distributed as dist +from libai.utils.checkpoint import Checkpointer +from libai.utils.checkpoint import PeriodicCheckpointer as _PeriodicCheckpointer +from libai.utils.events import EventWriter +from libai.utils.timer import Timer + +from .trainer import HookBase + +# -------------------------------------------------------- +# References: +# https://github.com/facebookresearch/detectron2/blob/main/detectron2/engine/hooks.py +# -------------------------------------------------------- + +""" +Implement some common hooks. +""" +logger = logging.getLogger(__name__) + + +class CallbackHook(HookBase): + """ + Create a hook using callback functions provided by the user. + """ + + def __init__(self, *, before_train=None, after_train=None, before_step=None, after_step=None): + """ + Each argument is a function that takes one argument: the trainer. + """ + self._before_train = before_train + self._before_step = before_step + self._after_step = after_step + self._after_train = after_train + + def before_train(self): + if self._before_train: + self._before_train(self.trainer) + + def after_train(self): + if self._after_train: + self._after_train(self.trainer) + # The functions may be closures that hold reference to the trainer + # Therefore, delete them to avoid circular reference. + del self._before_train, self._after_train + del self._before_step, self._after_step + + def before_step(self): + if self._before_step: + self._before_step(self.trainer) + + def after_step(self): + if self._after_step: + self._after_step(self.trainer) + + +class IterationTimer(HookBase): + """ + Track the time spent for each iteration (each run_step call in the trainer). + Print a summary in the end of training. + This hook uses the time between the call to its :meth:`before_step` + and :meth:`after_step` methods. + Under the convention that :meth:`before_step` of all hooks should only + take negligible amount of time, the :class:`IterationTimer` hook should be + placed at the beginning of the list of hooks to obtain accurate timing. + """ + + def __init__(self, warmup_iter=3): + """ + Args: + warmup_iter (int): the number of iterations at the beginning to exclude + from timing. + """ + self._warmup_iter = warmup_iter + self._step_timer = Timer() + + def before_train(self): + self._start_time = time.perf_counter() + self._total_timer = Timer() + self._total_timer.pause() + + def after_train(self): + total_time = time.perf_counter() - self._start_time + total_time_minus_hooks = self._total_timer.seconds() + hook_time = total_time - total_time_minus_hooks + + num_iter = self.trainer.iter + 1 - self.trainer.start_iter - self._warmup_iter + + if num_iter > 0 and total_time_minus_hooks > 0: + # Speed is meaningful only after warmup + # NOTE this format is parsed by grep in some scripts + logger.info( + "Overall training speed: {} iterations in {} ({:.4f} s / it)".format( + num_iter, + str(datetime.timedelta(seconds=int(total_time_minus_hooks))), + total_time_minus_hooks / num_iter, + ) + ) + + logger.info( + "Total training time: {} ({} on hooks)".format( + str(datetime.timedelta(seconds=int(total_time))), + str(datetime.timedelta(seconds=int(hook_time))), + ) + ) + + def before_step(self): + self._step_timer.reset() + self._total_timer.resume() + + def after_step(self): + # +1 because we're in after_step + iter_done = self.trainer.iter - self.trainer.start_iter + 1 + if iter_done >= self._warmup_iter: + sec = self._step_timer.seconds() + self.trainer.storage.put_scalars(time=sec) + else: + self._start_time = time.perf_counter() + self._total_timer.reset() + + self._total_timer.pause() + + +class PeriodicWriter(HookBase): + """ + Write events to EventStorage periodically. + It is executed every ``period`` iterations and after the last iteration. + """ + + def __init__(self, writers, period=20): + """ + Args: + writers (list[EventWriter]): a list of EventWriter objects + period (int): + """ + self._writers = writers + for w in writers: + assert isinstance(w, EventWriter), w + self._period = period + + def after_step(self): + if (self.trainer.iter + 1) % self._period == 0 or ( + self.trainer.iter == self.trainer.max_iter - 1 + ): + for writer in self._writers: + writer.write() + + def after_train(self): + for writer in self._writers: + writer.close() + + +class PeriodicCheckpointer(_PeriodicCheckpointer, HookBase): + """ + Same as :class:`libai.utils.checkpoint.PeriodicCheckpointer`, but as a hook. + Note that when used as a hook, + it is unable to save additional data other than what's defined + by the given `checkpointer`. + It is executed every ``period`` iterations and after the last iteration. + """ + + def before_train(self): + self.max_iter = self.trainer.max_iter + + def after_step(self): + self.step(self.trainer.iter) + + +class BestCheckpointer(HookBase): + """ + Checkpoints best weights based off given metric. + This hook should be used in conjunction to and executed after the hook + that produces the metric, e.g. `EvalHook`. + """ + + def __init__( + self, + eval_period: int, + checkpointer: Checkpointer, + val_metric: str, + mode: str = "max", + file_prefix: str = "model_best", + ) -> None: + """ + Args: + eval_period (int): the period `EvalHook` is set to run. + checkpointer: the checkpointer object used to save checkpoints. + val_metric (str): validation metric to track for best checkpoint, e.g. "acc@1" + mode (str): one of {'max', 'min'}. controls whether the chosen val metric should be + maximized or minimized, e.g. for "acc@1" it should be "max" + file_prefix (str): the prefix of checkpoint's filename, defaults to "model_best" + """ + self._period = eval_period + self._val_metric = val_metric + assert mode in [ + "max", + "min", + ], f'Mode "{mode}" to `BestCheckpointer` is unknown. It should be one of {"max", "min"}.' + if mode == "max": + self._compare = operator.gt + else: + self._compare = operator.lt + self._checkpointer = checkpointer + self._file_prefix = file_prefix + self.best_metric = None + self.best_iter = None + + def _update_best(self, val, iteration): + if math.isnan(val) or math.isinf(val): + return False + self.best_metric = val + self.best_iter = iteration + return True + + def _best_checking(self): + metric_tuple = self.trainer.storage.latest().get(self._val_metric) + flag = flow.zeros(1) + if dist.is_main_process(): + if metric_tuple is None: + logger.warning( + f"Given val metric {self._val_metric} does not seem to be computed/stored. " + "Will not be checkpointed based on that." + ) + else: + latest_metric, metric_iter = metric_tuple + + if self.best_metric is None: + if self._update_best(latest_metric, metric_iter): + flag = flag + 1 + logger.info( + f"Saved first model at {self.best_metric:0.5f} @ {self.best_iter} steps" + ) + elif self._compare(latest_metric, self.best_metric): + flag = flag + 1 + logger.info( + f"Saved best model as latest eval score for {self._val_metric} is " + f"{latest_metric:0.5f}, better than last best score " + f"{self.best_metric:0.5f} @ iteration {self.best_iter}." + ) + self._update_best(latest_metric, metric_iter) + else: + logger.info( + f"Not saving as latest eval score for " + f"{self._val_metric} is {latest_metric:0.5f}, " + f"not better than best score {self.best_metric:0.5f} " + f"@ iteration {self.best_iter}." + ) + + dist.synchronize() + flag = flag.to_global( + sbp=flow.sbp.broadcast, placement=flow.env.all_device_placement("cpu") + ) + if flag.to_local().item() == 1: + self._checkpointer.save(f"{self._file_prefix}") + + def after_step(self): + # same conditions as `EvalHook` + next_iter = self.trainer.iter + 1 + if ( + self._period > 0 + and next_iter % self._period == 0 + and next_iter != self.trainer.max_iter + ): + self._best_checking() + + def after_train(self): + # same conditions as `EvalHook` + if self.trainer.iter + 1 >= self.trainer.max_iter: + self._best_checking() + + +class EvalHook(HookBase): + """ + Run an evaluation function periodically, and at the end of training. + It is executed every ``eval_period`` iterations and after the last iteration. + """ + + def __init__(self, eval_period, eval_function): + """ + Args: + eval_period (int): the period to run `eval_function`. + eval_function (callable): a function which takes no arguments, and + returns a nested dict of evaluation metrics. + Note: + This hook must be enabled in all or none workers. + If you would like only certain workers to perform evaluation, + give other workers a no-op function (`eval_function=lambda: None`). + """ + self._period = eval_period + self._func = eval_function + + def _do_eval(self): + + results = self._func() + + if results: + assert isinstance( + results, dict + ), "Eval function must return a dict. Got {} instead.".format(results) + + flattened_results = flatten_results_dict(results) + # fixme: flatten_results_dict is not defined + for k, v in flattened_results.items(): + try: + v = float(v) + except Exception: + raise ValueError( + "[EvalHook] eval_function should return a nested dict of float. " + "Got '{}: {}' instead.".format(k, v) + ) + self.trainer.storage.put_scalars(**flattened_results, smoothing_hint=False) + + # Evaluation may take different time among workers. + # A barrier make them start the next iteration together. + dist.synchronize() + + def after_step(self): + next_iter = self.trainer.iter + 1 + if self._period > 0 and next_iter % self._period == 0: + # do the last eval in after_train + if next_iter != self.trainer.max_iter: + self._do_eval() + + def after_train(self): + # This condition is to prevent the eval from running after a failed training + if self.trainer.iter + 1 >= self.trainer.max_iter: + self._do_eval() + # func is likely a closure that holds reference to the trainer + # therefore we clean it to avoid circular reference in the end + del self._func + + +class LRScheduler(HookBase): + """ + A hook which executes a oneflow builtin LR scheduler and summarizes the LR. + It is executed after every iteration. + """ + + def __init__(self, optimizer=None, scheduler=None): + """ + Args: + optimizer (flow.optim.Optimizer): + scheduler (flow.optim.LRScheduler): + if a :class:`ParamScheduler` object, it defines the multiplier over the base LR + in the optimizer. + If any argument is not given, will try to obtain it from the trainer. + """ + self._optimizer = optimizer + self._scheduler = scheduler + + def before_train(self): + self._optimizer = self._optimizer or self.trainer.optimizer + self._best_param_group_id = LRScheduler.get_best_param_group_id(self._optimizer) + + @staticmethod + def get_best_param_group_id(optimizer): + # NOTE: some heuristics on what LR to summarize + # summarize the param group with most parameters + largest_group = max(len(g["params"]) for g in optimizer.state_dict()["param_groups"]) + + if largest_group == 1: + # If all groups have one parameter, + # then find the most common initial LR, and use it for summary + lr_count = Counter( + [g["_options"]["lr"] for g in optimizer.state_dict()["param_groups"]] + ) + lr = lr_count.most_common()[0][0] + for i, g in enumerate(optimizer.state_dict()["param_groups"]): + if g["_options"]["lr"] == lr: + return i + else: + for i, g in enumerate(optimizer.state_dict()["param_groups"]): + if len(g["params"]) == largest_group: + return i + + def after_step(self): + lr = self.scheduler.get_last_lr()[self._best_param_group_id] + self.trainer.storage.put_scalar("lr", lr, smoothing_hint=False) + self.scheduler.step() + + @property + def scheduler(self): + return self._scheduler or self.trainer.lr_scheduler + + def state_dict(self): + if isinstance(self.scheduler, flow.optim.lr_scheduler._LRScheduler): + return self.scheduler.state_dict() + return {} + + def load_state_dict(self, state_dict): + if isinstance(self.scheduler, flow.optim.lr_scheduler._LRScheduler): + logger.info("Loading scheduler from state_dict ...") + self.scheduler.load_state_dict(state_dict) diff --git a/libai/engine/trainer.py b/libai/engine/trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..ffd64ebaa97afe8723d64d597e705773b846cf9c --- /dev/null +++ b/libai/engine/trainer.py @@ -0,0 +1,349 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import time +import weakref +from typing import Callable, List, Mapping + +import oneflow as flow + +from libai.utils import distributed as dist +from libai.utils.events import EventStorage, get_event_storage + +# -------------------------------------------------------- +# References: +# https://github.com/facebookresearch/detectron2/blob/main/detectron2/engine/train_loop.py +# -------------------------------------------------------- + + +class HookBase: + """ + Base class for hooks that can be registered with :class:`TrainerBase`. + + Each hook can implement 4 methods. The way they are called is demonstrated + in the following snippet: + :: + + hook.before_train() + for iter in range(start_iter, max_iter): + hook.before_step() + trainer.run_step() + hook.after_step() + iter += 1 + hook.after_train() + + Notes: + 1. In the hook method, users can access ``self.trainer`` to access more + properties about the context (e.g., model, current iteration, or config + if using :class:`DefaultTrainer`). + + 2. A hook that does something in :meth:`before_step` can often be + implemented equivalently in :meth:`after_step`. + If the hook takes non-trivial time, it is strongly recommended to + implement the hook in :meth:`after_step` instead of :meth:`before_step`. + The convention is that :meth:`before_step` should only take negligible time. + + Following this convention will allow hooks that do care about the difference + between :meth:`before_step` and :meth:`after_step` (e.g., timer) to + function properly. + """ + + trainer: "TrainerBase" = None + """ + A weak reference to the trainer object. Set by the trainer when the hook is registered. + """ + + def before_train(self): + """ + Called before the first iteration. + """ + + def after_train(self): + """ + Called after the last iteration. + """ + + def before_step(self): + """ + Called before each iteration. + """ + + def after_step(self): + """ + Called after each iteration. + """ + + +class TrainerBase: + """ + Base class for iterative trainer with hooks. + The only assumption we made here is: the training runs in a loop. + A subclass can implement what the loop is. + We made no assumptions about the existence of dataloader, optimizer, model, etc. + + Attributes: + iter(int): The current iteration. + start_iter(int): The iteration to start with. + By convention the minimum possible value is 0. + max_iter(int): The iteration to end training. + storage(EventStorage): An EventStorage that's opened during the course of training. + """ + + def __init__(self): + self._hooks: List[HookBase] = [] + self.iter: int = 0 + self.start_iter: int = 0 + self.max_iter: int + self.storage: EventStorage + + def register_hooks(self, hooks): + """ + Register hooks to the trainer. The hooks are executed in the order + they are registered. + + Args: + hooks (list[Optional[HookBase]]): list of hooks + """ + hooks = [h for h in hooks if h is not None] + for h in hooks: + assert isinstance(h, HookBase) + # To avoid circular reference, hooks and trainer cannot own each other. + # This normally does not matter, but will cause memory leak if the + # involved objects contain __del__: + # See http://engineering.hearsaysocial.com/2013/06/16/circular-references-in-python/ + h.trainer = weakref.proxy(self) + self._hooks.extend(hooks) + + def train(self, start_iter: int, max_iter: int): + """ + Args: + start_iter, max_iter (int): See docs above + """ + logger = logging.getLogger(__name__) + logger.info("Starting training from iteration {}".format(start_iter)) + + self.iter = self.start_iter = start_iter + self.max_iter = max_iter + + with EventStorage(self.start_iter) as self.storage: + try: + self.before_train() + for self.iter in range(start_iter, max_iter): + self.before_step() + self.run_step() + self.after_step() + # self.iter == max_iter can be used by `after_train` to + # tell whether the training successfully finished or failed + # due to exceptions. + self.iter += 1 + except Exception: + logger.exception("Exception during training:") + raise + finally: + self.after_train() + + def before_train(self): + for h in self._hooks: + h.before_train() + + def after_train(self): + for h in self._hooks: + h.after_train() + + def before_step(self): + self.storage.iter = self.iter + for h in self._hooks: + h.before_step() + + def after_step(self): + self.storage.samples = (self.iter + 1) * self.cfg.train.global_batch_size + for h in self._hooks: + h.after_step() + + def run_step(self): + raise NotImplementedError + + @staticmethod + def write_metrics( + loss_dict: Mapping[str, flow.Tensor], + data_time: float, + prefix: str = "", + ) -> None: + """ + Args: + loss_dict (dict): dict of scalar losses + data_time (float): time taken by the dataloader iteration + prefix (str): prefix for logging keys + """ + # get metric value, remove it to rank0 cause logger.info only work in rank0 + metrics_dict = { + k: dist.tensor_to_rank0(v, device="cpu", to_local=True) for k, v in loss_dict.items() + } + metrics_dict["data_time"] = data_time + + # TODO: Gather metrics among all workers for logging + # all_metrics_dict = dist.gather(metrics_dict) + all_metrics_dict = metrics_dict + + if dist.is_main_process(): + storage = get_event_storage() + + # data_time among workers can have high variance. The actual latency + # caused by data_time is the maximum among workers. + # data_time = np.max([x.pop("data_time") for x in all_metrics_dict]) + data_time = all_metrics_dict.pop("data_time") + storage.put_scalar("data_time", data_time) + + # average the rest metrics + # metrics_dict = { + # k: np.mean([x[k] for x in all_metrics_dict]) for k in all_metrics_dict[0].keys() + # } + metrics_dict = all_metrics_dict + total_losses_reduced = sum(v for k, v in metrics_dict.items() if "loss" in k) + + storage.put_scalar("{}total_loss".format(prefix), total_losses_reduced) + if len(metrics_dict) > 1: + storage.put_scalars(**metrics_dict) + + +class EagerTrainer(TrainerBase): + """ + A simple eager trainer for the most common type of task: + single-cost single-optimizer single-data-source iterative optimization, + optionally using data-parallelism. + It assumes that in every step, you: + + 1. Compute the loss with a data from the data_loader. + 2. Compute the gradients with the above loss. + 3. Update the model with the optimizer. + + All other tasks during training (checkpointing, logging, evaluation, LR schedule) + are maintained by hooks, which can be registered by :meth:`TrainerBase.register_hooks`. + If you want to do anything fancier than this, + either subclass TrainerBase and implement your own `run_step`, + or write your own training loop. + """ + + def __init__(self, model, data_loader, optimizer, grad_acc_steps=1): + """ + Args: + model: a flow.nn.Module. Takes a data from data_loader and returns a + dict of losses. + data_loader: an iterable. Contains data to be used to call model. + optimizer: a flow optimizer. + """ + super().__init__() + + # We set the model to training mode in the trainer. + # However it's valid to train a model that's in eval mode. + # If you want your model (or a submodule of it) to behave + # like evaluation during training, you can overwrite its train() method. + + model.train() + + self.model = model + self.data_loader = data_loader + self._data_loader_iter = iter(data_loader) + self.optimizer = optimizer + self.grad_acc_steps = grad_acc_steps + + def run_step(self, get_batch: Callable, input_placement_device: str = "cuda"): + """ + Implement the standard training logic described above. + """ + assert self.model.training, "[SimpleTrainer] model was changed to eval mode!" + start = time.perf_counter() + + # If you want to do something with the data, you can wrap the dataloader. + data = next(self._data_loader_iter) + data = get_batch( + data, input_placement_device, getattr(self.data_loader, "mixup_func", None) + ) + data_time = time.perf_counter() - start + + loss_dict = self.model(**data) + losses = sum(v for k, v in loss_dict.items() if "loss" in k) / self.grad_acc_steps + + losses.backward() + self.write_metrics(loss_dict, data_time) + + if (self.iter + 1) % self.grad_acc_steps == 0: + self.optimizer.clip_grad() + self.optimizer.step() + self.optimizer.zero_grad() + + +class GraphTrainer(TrainerBase): + """ + A simple graph trainer for training and evaluating models in a static graph mode. + """ + + def __init__(self, graph, data_loader, grad_acc_steps=1): + super().__init__() + + graph.model.train() + self.data_loader = data_loader + self._data_loader_iter = iter(data_loader) + self.graph = graph + self.grad_acc_steps = grad_acc_steps + self._temp_data = None + self._temp_count = 0 + + def run_step(self, get_batch: Callable, input_placement_device: str = "cuda"): + """ + Implement the standard training logic described above. + """ + assert self.graph.model.training, "[SimpleTrainer] model was changed to eval mode!" + start = time.perf_counter() + + while self._temp_count != self.grad_acc_steps: + # If you want to do something with the data, you can wrap the dataloader. + data = next(self._data_loader_iter) + + self._temp_count += 1 + if self._temp_data is None: + self._temp_data = data + else: + # In static graph mode, data will be sliced in nn.Graph automatically, + # for geting mini-batch_size, we concat local_tensor first. + for key, value in data.get_fields().items(): + temp_value = self._temp_data.get(key) + self._temp_data.get(key).tensor = flow.cat( + (temp_value.tensor, value.tensor), dim=0 + ) + + data = self._temp_data + self._temp_count = 0 + self._temp_data = None + + data = get_batch( + data, input_placement_device, getattr(self.data_loader, "mixup_func", None) + ) + + data_time = time.perf_counter() - start + + # If you want to do something with the losses, you can wrap the model. + loss_dict = self.graph(**data) + # Add this because when set up gradient accumulations, graph will return + # an unpacked n-d tensor whose size is accumulation step + for key, value in loss_dict.items(): + if "loss" in key: + loss_dict[key] = value.mean() + else: + # NOTE: only support scalar tensor currently + loss_dict[key] = value.sum() + + self.write_metrics(loss_dict, data_time) diff --git a/libai/evaluation/__init__.py b/libai/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a118bc9704f4def2629392ada4f4e8b34cd47600 --- /dev/null +++ b/libai/evaluation/__init__.py @@ -0,0 +1,21 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .evaluator import DatasetEvaluator, inference_on_dataset +from .utils import print_csv_format, flatten_results_dict +from .cls_evaluator import ClsEvaluator +from .ppl_evaluator import PPLEvaluator +from .reg_evaluator import RegEvaluator +from .bleu_evaluator import BLEUEvaluator diff --git a/libai/evaluation/__pycache__/__init__.cpython-39.pyc b/libai/evaluation/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f9825b18ad2fc45284a3d990a71bcbc5b1fdd17b Binary files /dev/null and b/libai/evaluation/__pycache__/__init__.cpython-39.pyc differ diff --git a/libai/evaluation/__pycache__/bleu_evaluator.cpython-39.pyc b/libai/evaluation/__pycache__/bleu_evaluator.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d0dbb2baf282eb594d84326a12b3564d0ef5f980 Binary files /dev/null and b/libai/evaluation/__pycache__/bleu_evaluator.cpython-39.pyc differ diff --git a/libai/evaluation/__pycache__/cls_evaluator.cpython-39.pyc b/libai/evaluation/__pycache__/cls_evaluator.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c53a53c8670d69e356a2eca262751d6e38ff7787 Binary files /dev/null and b/libai/evaluation/__pycache__/cls_evaluator.cpython-39.pyc differ diff --git a/libai/evaluation/__pycache__/evaluator.cpython-39.pyc b/libai/evaluation/__pycache__/evaluator.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..467baab0c744a914cf00b44c179fa2930cfa7aff Binary files /dev/null and b/libai/evaluation/__pycache__/evaluator.cpython-39.pyc differ diff --git a/libai/evaluation/__pycache__/ppl_evaluator.cpython-39.pyc b/libai/evaluation/__pycache__/ppl_evaluator.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0ba75b498dab2bfee7d14d00a12b1820f04fc1b8 Binary files /dev/null and b/libai/evaluation/__pycache__/ppl_evaluator.cpython-39.pyc differ diff --git a/libai/evaluation/__pycache__/reg_evaluator.cpython-39.pyc b/libai/evaluation/__pycache__/reg_evaluator.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..115cae2979b874e633b03d7104b687221dad2824 Binary files /dev/null and b/libai/evaluation/__pycache__/reg_evaluator.cpython-39.pyc differ diff --git a/libai/evaluation/__pycache__/utils.cpython-39.pyc b/libai/evaluation/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f8cdd8887e2c51ce298f32ecf4ce1bdc9d6e99c3 Binary files /dev/null and b/libai/evaluation/__pycache__/utils.cpython-39.pyc differ diff --git a/libai/evaluation/bleu_evaluator.py b/libai/evaluation/bleu_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..86a5a4f6b8df8f2c60af1dc44ef019cbdd9f6e83 --- /dev/null +++ b/libai/evaluation/bleu_evaluator.py @@ -0,0 +1,64 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +from collections import OrderedDict + +from nltk.translate.bleu_score import corpus_bleu + +from libai.utils import distributed as dist + +from .evaluator import DatasetEvaluator + + +class BLEUEvaluator(DatasetEvaluator): + """ + Evaluate BLEU(Bilingual Evaluation Understudy) score. + + BLEU is a score for comparing a candidate translation + of text to one or more reference translations. + """ + + def __init__(self): + super().__init__() + self._predictions = [] + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + candidate = outputs["candidate"] + reference = inputs["reference"] + + self._predictions.append({"candidate": candidate, "reference": reference}) + + def evaluate(self): + if not dist.is_main_process(): + return {} + else: + predictions = self._predictions + + candidates = [] + references = [] + for pred in predictions: + candidates.append(pred["candidate"]) + references.append(pred["reference"]) + + bleu_score = corpus_bleu(references, candidates) + + self._results = OrderedDict() + self._results["bleu_score"] = bleu_score + + return copy.deepcopy(self._results) diff --git a/libai/evaluation/cls_evaluator.py b/libai/evaluation/cls_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..e46f9cfd11093119486f95acc90396243d6c5aac --- /dev/null +++ b/libai/evaluation/cls_evaluator.py @@ -0,0 +1,84 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +from collections import OrderedDict + +from libai.utils import distributed as dist + +from .evaluator import DatasetEvaluator + + +def accuracy(output, target, topk=(1,)): + maxk = min(max(topk), output.size()[1]) + batch_size = target.size(0) + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.reshape(1, -1).expand_as(pred)) + return [ + (correct[: min(k, maxk)].reshape(-1).float().sum(0) * 100.0 / batch_size).item() + for k in topk + ] + + +class ClsEvaluator(DatasetEvaluator): + """ + Evaluate accuracy for classification. + The metrics range from 0 to 100 (instead of 0 to 1). + We support evaluate different topk accuracy. + You can reset `cfg.train.topk=(1, 5, N)` according to your needs. + """ + + def __init__(self, topk=(1, 5)): + self.topk = topk + self._predictions = [] + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + pred_logits = outputs["prediction_scores"] + labels = inputs["labels"] + + # measure accuracy + topk_acc = accuracy(pred_logits, labels, topk=self.topk) + num_correct_acc_topk = [acc * labels.size(0) / 100 for acc in topk_acc] + + self._predictions.append( + {"num_correct_topk": num_correct_acc_topk, "num_samples": labels.size(0)} + ) + + def evaluate(self): + if not dist.is_main_process(): + return {} + else: + predictions = self._predictions + + total_correct_num = OrderedDict() + for top_k in self.topk: + total_correct_num["Acc@" + str(top_k)] = 0 + + total_samples = 0 + for prediction in predictions: + for top_k, num_correct_n in zip(self.topk, prediction["num_correct_topk"]): + total_correct_num["Acc@" + str(top_k)] += int(num_correct_n) + + total_samples += int(prediction["num_samples"]) + + self._results = OrderedDict() + for top_k, topk_correct_num in total_correct_num.items(): + self._results[top_k] = topk_correct_num / total_samples * 100 + + return copy.deepcopy(self._results) diff --git a/libai/evaluation/evaluator.py b/libai/evaluation/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..1414cdaa008c456eea55383d6017111223dd9b17 --- /dev/null +++ b/libai/evaluation/evaluator.py @@ -0,0 +1,295 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import logging +import time +from collections import OrderedDict, abc +from contextlib import ExitStack, contextmanager +from typing import Callable, List, Union + +import oneflow as flow + +from libai.utils import distributed as dist +from libai.utils.logger import log_every_n_seconds + +from .utils import pad_batch + +# -------------------------------------------------------- +# References: +# https://github.com/facebookresearch/detectron2/blob/main/detectron2/evaluation/evaluator.py +# -------------------------------------------------------- + + +class DatasetEvaluator: + """ + Base class for a dataset evaluator. + The function :func:`inference_on_dataset` runs the model over + all samples in the dataset, and have a DatasetEvaluator to process the inputs/outputs. + This class will accumulate information of the inputs/outputs (by :meth:`process`), + and produce evaluation results in the end (by :meth:`evaluate`). + """ + + def reset(self): + """ + Preparation for a new round of evaluation. + Should be called before starting a round of evaluation. + """ + + def process(self, inputs, outputs): + """ + Process the pair of inputs and outputs. + + .. code-block:: python + + pred_logits = outputs["prediction_scores"] + labels = inputs["labels"] + # do evaluation on pred_logits/labels pair + ... + + Args: + inputs (dict): the inputs that's used to call the model. + outputs (dict): the return dict of `model(**inputs)` + """ + + def evaluate(self): + """ + Evaluate/summarize the performance after processing all input/output pairs. + + Returns: + dict: + A new evaluator class can return a dict of arbitrary format + as long as the user can process the results. + In our train_net.py, we expect the following format: + + * key: the name of the task (e.g., Classification) + * value: a dict of {metric name: score}, e.g.: {"Acc@1": 75.0} + """ + + +class DatasetEvaluators(DatasetEvaluator): + """ + Wrapper class to combine multiple :class:`DatasetEvaluator` instances. + This class dispatches every evaluation call to + all of its :class:`DatasetEvaluator`. + """ + + def __init__(self, evaluators): + """ + Args: + evaluators (list): the evaluators to combine. + """ + super().__init__() + self._evaluators = evaluators + + def reset(self): + for evaluator in self._evaluators: + evaluator.reset() + + def process(self, inputs, outputs): + for evaluator in self._evaluators: + evaluator.process(inputs, outputs) + + def evaluate(self): + results = OrderedDict() + for evaluator in self._evaluators: + result = evaluator.evaluate() + if dist.is_main_process() and result is not None: + for k, v in result.items(): + assert ( + k not in results + ), "Different evaluators produce results with the same key {}".format(k) + results[k] = v + return results + + +def inference_on_dataset( + model, + data_loader, + batch_size, + eval_iter, + get_batch: Callable, + input_placement_device: str, + evaluator: Union[DatasetEvaluator, List[DatasetEvaluator], None], +): + """ + Run model on the data_loader and evaluate the metrics with evaluator. + Also benchmark the inference speed of `model.__call__` accurately. + The model will be used in eval mode. + + Args: + model (callable): a callable which takes an object from + `data_loader` and returns some outputs. + If it's an nn.Module, it will be temporarily set to `eval` mode. + If you wish to evaluate a model in `training` mode instead, you can + wrap the given model and override its behavior of `.eval()` and `.train()`. + batch_size: batch size for inference + data_loader: an iterable object with a length. + The elements it generates will be the inputs to the model. + eval_iter: running steps for evaluation + get_batch: a Callable function for getting data from dataloader + input_placement_device: used in get_batch, set it to `cuda` or `cpu`. + see input_placement_device in `libai.configs.common.train.py` for more details. + evaluator: the evaluator(s) to run. Use `None` if you only want to benchmark, + but don't want to do any evaluation. + + Returns: + The return value of `evaluator.evaluate()` + """ + num_devices = dist.get_world_size() + logger = logging.getLogger(__name__) + + total_samples = len(data_loader.dataset) # inference data loader must have a fixed length + if evaluator is None: + # create a no-op evaluator + evaluator = DatasetEvaluators([]) + if isinstance(evaluator, abc.MutableSequence): + evaluator = DatasetEvaluators(evaluator) + evaluator.reset() + + num_warmup = min(5, len(data_loader) - 1) + start_time = time.perf_counter() + total_data_time = 0 + total_compute_time = 0 + total_eval_time = 0 + consumed_samples = 0 + dps = dist.get_data_parallel_size() + last_batch_lack = (dps - (total_samples % dps)) % dps + + # reset total samples + real_eval_iter = min(eval_iter, len(data_loader)) + total_samples = min(real_eval_iter * batch_size, len(data_loader.dataset)) + logger.info( + f"with eval_iter {eval_iter}, " + f"reset total samples {len(data_loader.dataset)} to {total_samples}" + ) + logger.info(f"Start inference on {total_samples} samples") + + with ExitStack() as stack: + if isinstance(model, (flow.nn.Module, flow.nn.Graph)): + stack.enter_context(inference_context(model)) + stack.enter_context(flow.no_grad()) + + start_data_time = time.perf_counter() + for idx, inputs in enumerate(data_loader): + if idx >= real_eval_iter: + break + total_data_time += time.perf_counter() - start_data_time + if idx == num_warmup: + start_time = time.perf_counter() + total_data_time = 0 + total_compute_time = 0 + total_eval_time = 0 + + start_compute_time = time.perf_counter() + # model forward + data = get_batch(inputs, input_placement_device) + is_last_batch = idx == len(data_loader) - 1 + paded_data, valid_sample = pad_batch(data, batch_size, last_batch_lack, is_last_batch) + outputs = model(**paded_data) + + # get valid sample + valid_data = { + key: dist.tensor_to_rank0(value, to_local=True)[:valid_sample] + for key, value in data.items() + } + valid_outputs = {} + for key, value in outputs.items(): + value = dist.tensor_to_rank0(value, to_local=True) + if value.ndim > 1: + valid_outputs[key] = value[:valid_sample] # Slice if it's batched output + else: + valid_outputs[key] = value + + if flow.cuda.is_available(): + dist.synchronize() + total_compute_time += time.perf_counter() - start_compute_time + + start_eval_time = time.perf_counter() + if dist.is_main_process(): + evaluator.process(valid_data, valid_outputs) + dist.synchronize() + total_eval_time += time.perf_counter() - start_eval_time + + consumed_samples += valid_sample + iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup) + data_seconds_per_iter = total_data_time / iters_after_start + compute_seconds_per_iter = total_compute_time / iters_after_start + eval_seconds_per_iter = total_eval_time / iters_after_start + total_seconds_per_iter = (time.perf_counter() - start_time) / iters_after_start + if idx >= num_warmup * 2 or compute_seconds_per_iter > 5: + eta = datetime.timedelta( + seconds=int(total_seconds_per_iter * (total_samples // batch_size - idx - 1)) + ) + log_every_n_seconds( + logging.INFO, + ( + f"Inference done {consumed_samples}/{total_samples}. " + f"Dataloading: {data_seconds_per_iter:.4f} s/iter. " + f"Inference: {compute_seconds_per_iter:.4f} s/iter. " + f"Eval: {eval_seconds_per_iter:.4f} s/iter. " + f"Total: {total_seconds_per_iter:.4f} s/iter. " + f"ETA={eta}" + ), + n=5, + ) + start_data_time = time.perf_counter() + + # Measure the time only for this worker (before the synchronization barrier) + total_time = time.perf_counter() - start_time + total_time_str = str(datetime.timedelta(seconds=total_time)) + # NOTE this format is parsed by grep + logger.info("Total valid samples: {}".format(consumed_samples)) + logger.info( + "Total inference time: {} ({:.6f} s / iter per device, on {} devices)".format( + total_time_str, total_time / (total_samples - num_warmup), num_devices + ) + ) + total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time))) + logger.info( + "Total inference pure compute time: {} ({:.6f} s / iter per device, on {} devices)".format( + total_compute_time_str, + total_compute_time / (total_samples - num_warmup), + num_devices, + ) + ) + + results = evaluator.evaluate() + # An evaluator may return None when not in main process. + # Replace it by an empty dict instead to make it easier for downstream code to handle + if results is None: + results = {} + return results + + +@contextmanager +def inference_context(model): + """ + A context where the model is temporarily changed to eval mode, + and restored to previous mode afterwards. + Args: + model: eager or graph mode in oneflow + """ + training_mode = model.model.training if isinstance(model, flow.nn.Graph) else model.training + if isinstance(model, flow.nn.Graph): + model.model.eval() + else: + model.eval() + yield + if isinstance(model, flow.nn.Graph): + model.model.train(training_mode) + else: + model.train(training_mode) diff --git a/libai/evaluation/ppl_evaluator.py b/libai/evaluation/ppl_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..259d8b1ab8ea3506eca2f1aa99c147fe22604e2b --- /dev/null +++ b/libai/evaluation/ppl_evaluator.py @@ -0,0 +1,60 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import math +from collections import OrderedDict + +from libai.utils import distributed as dist + +from .evaluator import DatasetEvaluator + + +class PPLEvaluator(DatasetEvaluator): + """ + Evaluate perplexity for Language Model. + + Perplexity is a measurement of how well a probability distribution or + probability model predicts a sample. + """ + + def __init__(self): + self._predictions = [] + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + for k, v in outputs.items(): + ppl = math.exp(min(20, v.item())) + self._predictions.append({f"{k}_PPL": ppl}) + + def evaluate(self): + if not dist.is_main_process(): + return {} + else: + predictions = self._predictions + + self._results = OrderedDict() + for prediction in predictions: + for k, v in prediction.items(): + if k not in self._results: + self._results[k] = 0 + self._results[k] += v + + for k in self._results.keys(): + self._results[k] /= len(predictions) + + return copy.deepcopy(self._results) diff --git a/libai/evaluation/reg_evaluator.py b/libai/evaluation/reg_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..8e4184ef3f3a62f40b90dfd917bc1c51d153951e --- /dev/null +++ b/libai/evaluation/reg_evaluator.py @@ -0,0 +1,68 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import logging +from collections import OrderedDict + +import numpy as np +from scipy.stats import pearsonr, spearmanr + +from libai.utils import distributed as dist + +from .evaluator import DatasetEvaluator + +logger = logging.getLogger(__name__) + + +class RegEvaluator(DatasetEvaluator): + def __init__(self): + self._predictions = [] + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + pred_logits = outputs["prediction_scores"] + labels = inputs["labels"] + + # measure accuracy + preds = pred_logits.cpu().topk(1)[1].squeeze(1).numpy() + labels = labels.cpu().numpy() + + self._predictions.append({"preds": preds, "labels": labels}) + + def evaluate(self): + if not dist.is_main_process(): + return {} + else: + predictions = self._predictions + + preds = np.array([]) + labels = np.array([]) + for prediction in predictions: + preds = np.concatenate((preds, prediction["preds"])) + labels = np.concatenate((labels, prediction["labels"])) + + pearson_corr = pearsonr(preds, labels)[0] + spearman_corr = spearmanr(preds, labels)[0] + corr = (pearson_corr + spearman_corr) / 2 + + self._results = OrderedDict() + self._results["pearson"] = pearson_corr + self._results["spearman"] = spearman_corr + self._results["corr"] = corr + + return copy.deepcopy(self._results) diff --git a/libai/evaluation/utils.py b/libai/evaluation/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a1c36ca90894178924ae32d7c0b39bcec43e80eb --- /dev/null +++ b/libai/evaluation/utils.py @@ -0,0 +1,93 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from collections.abc import Mapping + +import oneflow as flow + +from libai.utils import distributed as dist + + +def pad_batch(x_dict, batch_size, last_batch_lack, is_last_batch): + x = list(x_dict.values())[0] + tensor_batch = x.shape[0] + assert tensor_batch <= batch_size + + if tensor_batch == batch_size and not is_last_batch: + return x_dict, batch_size + + valid_sample = tensor_batch - last_batch_lack + data_parallel_size = dist.get_data_parallel_size() + assert tensor_batch % data_parallel_size == 0 + tensor_micro_batch_size = tensor_batch // data_parallel_size + padded_dict = {} + for key, xi in x_dict.items(): + pad_shape = (batch_size, *xi.shape[1:]) + local_xi = xi.to_global( + sbp=flow.sbp.broadcast, placement=flow.env.all_device_placement("cuda") + ).to_local() + padded_xi = flow.zeros(pad_shape, dtype=xi.dtype, device="cuda") + padded_xi[:tensor_batch, ...] = padded_xi[:tensor_batch, ...] + local_xi + for i in range(last_batch_lack - 1): + start_idx = tensor_micro_batch_size * (data_parallel_size - i - 1) - 1 + padded_xi[start_idx:-1] = padded_xi[start_idx + 1 :] + padded_xi = padded_xi.to_global( + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), placement=xi.placement + ).to_global(sbp=xi.sbp) + padded_dict[key] = padded_xi + return padded_dict, valid_sample + + +def print_csv_format(results): + """ + Print main metrics in a particular format + so that they are easy to copypaste into a spreadsheet. + + Args: + results (OrderedDict[dict]): task_name -> {metric -> score} + unordered dict can also be printed, but in arbitrary order + """ + assert isinstance(results, Mapping) or not len(results), results + logger = logging.getLogger(__name__) + for task, res in results.items(): + if isinstance(res, Mapping): + # Don't print "AP-category" metrics since they are usually not tracked. + important_res = [(k, v) for k, v in res.items() if "-" not in k] + logger.info("copypaste: Task: {}".format(task)) + logger.info("copypaste: " + ",".join([k[0] for k in important_res])) + logger.info("copypaste: " + ",".join(["{0:.4f}".format(k[1]) for k in important_res])) + else: + logger.info(f"copypaste: {task}={res}") + + +def flatten_results_dict(results): + """ + Expand a hierarchical dict of scalars into a flat dict of scalars. + If results[k1][k2][k3] = v, the returned dict will have the entry + {"k1/k2/k3": v}. + + Args: + results (dict): + """ + r = {} + for k, v in results.items(): + if isinstance(v, Mapping): + v = flatten_results_dict(v) + for kk, vv in v.items(): + r[k + "/" + kk] = vv + else: + r[k] = v + return r diff --git a/libai/inference/basic.py b/libai/inference/basic.py new file mode 100644 index 0000000000000000000000000000000000000000..2b27057448cefd64c5767b2d4827d8bfa34031b0 --- /dev/null +++ b/libai/inference/basic.py @@ -0,0 +1,189 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from abc import ABCMeta, abstractmethod +from typing import Any, Dict + +import oneflow as flow + +from libai.config import LazyConfig, try_get_key +from libai.engine import DefaultTrainer +from libai.utils import distributed as dist +from libai.utils.logger import setup_logger + +logger = setup_logger(distributed_rank=dist.get_rank()) +logger = logging.getLogger("libai.inference") + + +class BasePipeline(metaclass=ABCMeta): + """ + Base class for all task pipeline + """ + + def __init__( + self, + config_file, + data_parallel=None, + tensor_parallel=None, + pipeline_parallel=None, + pipeline_stage_id=None, + pipeline_num_layers=None, + model_path=None, + mode="libai", + **kwargs, + ): + # init cfg + self.cfg = LazyConfig.load(config_file) + flow.boxing.nccl.set_fusion_threshold_mbytes( + try_get_key(self.cfg, "train.nccl_fusion_threshold_mb", default=16) + ) + flow.boxing.nccl.set_fusion_max_ops_num( + try_get_key(self.cfg, "train.nccl_fusion_max_ops", default=24) + ) + self.update_cfg( + data_parallel, + tensor_parallel, + pipeline_parallel, + pipeline_stage_id, + pipeline_num_layers, + ) + dist.setup_dist_util(self.cfg.train.dist) + assert ( + self.cfg.train.dist.data_parallel_size == 1 + ), "not support data parallel yet, only support tensor and pipeline parallel" + logger.info(self.cfg.train.dist) + + # initial and load model + + self.model = self.load_pretrain_weight(self.cfg.model, model_path, mode=mode) + self.model._apply(dist.convert_to_distributed_default_setting) + self.model = self.model.eval() + + # initial tokenizer + if dist.is_main_process(): + self.tokenizer = self.build_tokenizer(self.cfg) + else: + self.tokenizer = None + self.tokenizer = dist.broadcast_py_object(self.tokenizer, src=0) + + # set parameters + ( + self._preprocess_params, + self._forward_params, + self._postprocess_params, + ) = self._parse_parameters(**kwargs) + + def update_cfg( + self, + data_parallel=1, + tensor_parallel=1, + pipeline_parallel=1, + pipeline_stage_id=None, + pipeline_num_layers=None, + ): + self.cfg.train.dist.data_parallel_size = data_parallel + self.cfg.train.dist.tensor_parallel_size = tensor_parallel + self.cfg.train.dist.pipeline_parallel_size = pipeline_parallel + self.cfg.train.dist.custom_pipeline_stage_id = pipeline_stage_id + if pipeline_num_layers is not None: + self.cfg.train.dist.pipeline_num_layers = pipeline_num_layers + + if self.cfg.train.dist.pipeline_parallel_size > 1: + assert ( + try_get_key(self.cfg.train.dist, "pipeline_num_layers") is not None + ), "cfg.train.dist.pipeline_num_layers must be set when run pipeline parallel" + + def load_pretrain_weight( + self, + libai_cfg_model, + model_path, + mode="libai", + ): + """load pretrained model. + + Args: + libai_cfg_model (libai.models): Lazy config Model in Libai, you can import it + by `from libai.config.configs.common.models.bert + import pretrain_model as libai_cfg_model` + model_path (str): The directory path of pretrained model + mode (str): set it to `libai` for loading trained model from libai, + set it to `random` for quickly debugging by random initialized model + """ + if mode == "libai": + from libai.models.utils.model_utils.base_loader import ModelLoaderLiBai + + model_loader = ModelLoaderLiBai(libai_cfg_model, libai_cfg_model.cfg, model_path) + model_loader.base_model_prefix_1 = None + model_loader.base_model_prefix_2 = "" + return model_loader.load() + elif mode == "random": + return DefaultTrainer.build_model(self.cfg) + else: + raise NotImplementedError + + def build_tokenizer(self, cfg): + tokenizer = None + if try_get_key(cfg, "tokenization") is not None: + tokenizer = DefaultTrainer.build_tokenizer(cfg) + return tokenizer + + @abstractmethod + def _parse_parameters(self, **pipeline_parameters): + raise NotImplementedError("_parse_parameters not implemented") + + def __call__(self, inputs, *args, batch_size=None, **kwargs) -> dict: + + preprocess_params, forward_params, postprocess_params = self._parse_parameters( + **kwargs + ) # noqa + + # Fuse __init__ params and __call__ params without modifying the __init__ ones. + preprocess_params = {**self._preprocess_params, **preprocess_params} + forward_params = {**self._forward_params, **forward_params} + postprocess_params = {**self._postprocess_params, **postprocess_params} + + with flow.no_grad(): + model_inputs_dict = self.preprocess(inputs, **preprocess_params) + model_outputs_dict = self.forward(model_inputs_dict, **forward_params) + model_outputs_dict = self.to_local(model_outputs_dict) + if dist.is_main_process(): + outputs_dict = self.postprocess(model_outputs_dict, **postprocess_params) + else: + outputs_dict = {} + dist.synchronize() + return outputs_dict + + def to_local(self, model_outputs_dict): + for key, value in model_outputs_dict.items(): + if isinstance(value, flow.Tensor) and value.is_global: + model_outputs_dict[key] = dist.ttol( + value, ranks=[0] if value.placement.ranks.ndim == 1 else [[0]] + ) + if flow.cuda.is_available(): + dist.synchronize() + return model_outputs_dict + + @abstractmethod + def preprocess(self, input_: Any, **preprocess_parameters: Dict) -> dict: + raise NotImplementedError("preprocess not implemented") + + @abstractmethod + def forward(self, **kwargs: Dict) -> dict: + raise NotImplementedError("forward not implemented") + + @abstractmethod + def postprocess(self, **kwargs: Dict) -> dict: + raise NotImplementedError("postprocess not implemented") diff --git a/libai/inference/generator/generation_beam_search.py b/libai/inference/generator/generation_beam_search.py new file mode 100644 index 0000000000000000000000000000000000000000..5fb24f5b5c2c25fd6fa3f483960580df36ca877d --- /dev/null +++ b/libai/inference/generator/generation_beam_search.py @@ -0,0 +1,373 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright 2020 The Google AI Language Team Authors, Facebook AI Research authors and +# The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +from abc import ABC, abstractmethod +from collections import UserDict +from typing import Optional, Tuple + +import oneflow as flow + +from libai.utils import distributed as dist + + +class BeamScorer(ABC): + @abstractmethod + def process( + self, + input_ids: flow.Tensor, + next_scores: flow.Tensor, + next_tokens: flow.Tensor, + next_indices: flow.Tensor, + **kwargs, + ): + raise NotImplementedError("This is an abstract method.") + + +class BeamHypotheses: + def __init__(self, num_beams: int, length_penalty: float, early_stopping: bool): + """ + Initialize n-best list of hypotheses. + """ + self.length_penalty = length_penalty + self.early_stopping = early_stopping + self.num_beams = num_beams + self.beams = [] + self.worst_score = 1e9 + + def __len__(self) -> int: + """ + Number of hypotheses in the list. + """ + return len(self.beams) + + def add( + self, hyp: flow.Tensor, sum_logprobs: float, beam_indices: Optional[flow.Tensor] = None + ): + """ + Add a new hypothesis to the list. + """ + score = sum_logprobs / (hyp.shape[-1] ** self.length_penalty) + if len(self) < self.num_beams or score > self.worst_score: + self.beams.append((score, hyp, beam_indices)) + if len(self) > self.num_beams: + sorted_next_scores = sorted([(s, idx) for idx, (s, _, _) in enumerate(self.beams)]) + del self.beams[sorted_next_scores[0][1]] + self.worst_score = sorted_next_scores[1][0] + else: + self.worst_score = min(score, self.worst_score) + + def is_done(self, best_sum_logprobs: float, cur_len: int) -> bool: + """ + If there are enough hypotheses and that none of the hypotheses being generated + can become better than the worst one in the heap, then we are done with this sentence. + """ + + if len(self) < self.num_beams: + return False + elif self.early_stopping: + return True + else: + cur_score = best_sum_logprobs / cur_len ** self.length_penalty + ret = self.worst_score >= cur_score + return ret + + +class BeamSearchScorer(BeamScorer): + def __init__( + self, + batch_size: int, + num_beams: int, + length_penalty: Optional[float] = 1.0, + do_early_stopping: Optional[bool] = False, + num_beam_hyps_to_keep: Optional[int] = 1, + num_beam_groups: Optional[int] = 1, + **kwargs, + ): + self.num_beams = num_beams + self.length_penalty = length_penalty + self.do_early_stopping = do_early_stopping + self.num_beam_hyps_to_keep = num_beam_hyps_to_keep + self.num_beam_groups = num_beam_groups + self.group_size = self.num_beams // self.num_beam_groups + + self._is_init = False + self._beam_hyps = [ + BeamHypotheses( + num_beams=self.num_beams, + length_penalty=self.length_penalty, + early_stopping=self.do_early_stopping, + ) + for _ in range(batch_size) + ] + + self._done = flow.tensor( + [False for _ in range(batch_size)], + dtype=flow.bool, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=flow.placement("cuda", list(range(dist.get_world_size()))), + ) + + if not isinstance(num_beams, int) or num_beams <= 1: + raise ValueError( + f"`num_beams` has to be an integer strictly greater than 1, but is {num_beams}." + "For `num_beams` == 1, one should make use of `greedy_search` instead." + ) + + if ( + not isinstance(num_beam_groups, int) + or (num_beam_groups > num_beams) + or (num_beams % num_beam_groups != 0) + ): + raise ValueError( + "`num_beam_groups` has to be an integer smaller or equal than `num_beams` and " + f"`num_beams` has to be divisible by `num_beam_groups`, but is {num_beam_groups}" + f"with `num_beams` being {num_beams}." + ) + + if "max_length" in kwargs: + warnings.warn( + "Passing `max_length` to BeamSearchScorer is deprecated and has no effect. " + "`max_length` should be passed directly to `beam_search(...)`, `beam_sample(...)`" + ", or `group_beam_search(...)`." + ) + + @property + def is_done(self) -> bool: + return self._done.all() + + def process( + self, + input_ids: flow.Tensor, + next_scores: flow.Tensor, + next_tokens: flow.Tensor, + next_indices: flow.Tensor, + pad_token_id: Optional[int] = None, + eos_token_id: Optional[int] = None, + beam_indices: Optional[flow.Tensor] = None, + ) -> Tuple[flow.Tensor]: + cur_len = input_ids.shape[-1] + batch_size = len(self._beam_hyps) + if not (batch_size == (input_ids.shape[0] // self.group_size)): + if self.num_beam_groups > 1: + raise ValueError( + f"A group beam size of {input_ids.shape[0]} is used as the input, but a group " + f"beam size of {self.group_size} is expected by the beam scorer." + ) + else: + raise ValueError( + f"A beam size of {input_ids.shape[0]} is used as the input, but a beam size of " + f"{self.group_size} is expected by the beam scorer." + ) + next_beam_scores = flow.zeros( + (batch_size, self.group_size), + dtype=next_scores.dtype, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=flow.placement("cuda", list(range(dist.get_world_size()))), + ) + next_beam_tokens = flow.zeros( + (batch_size, self.group_size), + dtype=next_tokens.dtype, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=flow.placement("cuda", list(range(dist.get_world_size()))), + ) + next_beam_indices = flow.zeros( + (batch_size, self.group_size), + dtype=next_indices.dtype, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=flow.placement("cuda", list(range(dist.get_world_size()))), + ) + + for batch_idx, beam_hyp in enumerate(self._beam_hyps): + if self._done[batch_idx]: + if self.num_beams < len(beam_hyp): + raise ValueError( + f"Batch can only be done if at least {self.num_beams} beams have " + "been generated" + ) + if eos_token_id is None or pad_token_id is None: + raise ValueError( + "Generated beams >= num_beams -> eos_token_id and pad_token have " + "to be defined" + ) + # pad the batch + next_beam_scores[batch_idx, :] = 0 + next_beam_tokens[batch_idx, :] = pad_token_id + next_beam_indices[batch_idx, :] = 0 + continue + + # next tokens for this sentence + beam_idx = 0 + for beam_token_rank, (next_token, next_score, next_index) in enumerate( + zip(next_tokens[batch_idx], next_scores[batch_idx], next_indices[batch_idx]) + ): + batch_beam_idx = batch_idx * self.group_size + next_index + # add to generated hypotheses if end of sentence + if (eos_token_id is not None) and (next_token.item() == eos_token_id): + # if beam_token does not belong to top num_beams tokens, it should not be added + is_beam_token_worse_than_top_num_beams = beam_token_rank >= self.group_size + if is_beam_token_worse_than_top_num_beams: + continue + if beam_indices is not None: + beam_index = beam_indices[batch_beam_idx] + beam_index = beam_index + (next_index,) + else: + beam_index = None + + beam_hyp.add( + input_ids[batch_beam_idx].clone(), + next_score.item(), + beam_indices=beam_index, + ) + else: + # add next predicted token since it is not eos_token + next_beam_scores[batch_idx, beam_idx] = next_score + next_beam_tokens[batch_idx, beam_idx] = next_token + next_beam_indices[batch_idx, beam_idx] = batch_beam_idx + beam_idx += 1 + + # once the beam for next step is full, don't add more tokens to it. + if beam_idx == self.group_size: + break + + if beam_idx < self.group_size: + raise ValueError( + f"At most {self.group_size} tokens in {next_tokens[batch_idx]} can be equal " + f"to `eos_token_id: {eos_token_id}`. Make sure {next_tokens[batch_idx]} " + "are corrected." + ) + + # Check if we are done so that we can save a pad step if all(done) + self._done[batch_idx] = self._done[batch_idx] or beam_hyp.is_done( + next_scores[batch_idx].max().item(), cur_len + ) + + return UserDict( + { + "next_beam_scores": next_beam_scores.view(-1), + "next_beam_tokens": next_beam_tokens.view(-1), + "next_beam_indices": next_beam_indices.view(-1), + } + ) + + def finalize( + self, + input_ids: flow.Tensor, + final_beam_scores: flow.Tensor, + final_beam_tokens: flow.Tensor, + final_beam_indices: flow.Tensor, + max_length: int, + pad_token_id: Optional[int] = None, + eos_token_id: Optional[int] = None, + beam_indices: Optional[flow.Tensor] = None, + ): + batch_size = len(self._beam_hyps) + # finalize all open beam hypotheses and add to generated hypotheses + for batch_idx, beam_hyp in enumerate(self._beam_hyps): + if self._done[batch_idx]: + continue + + # all open beam hypotheses are added to the beam hypothesis + # beam hypothesis class automatically keeps the best beams + for beam_id in range(self.num_beams): + batch_beam_idx = batch_idx * self.num_beams + beam_id + final_score = final_beam_scores[batch_beam_idx].item() + final_tokens = input_ids[batch_beam_idx] + beam_index = beam_indices[batch_beam_idx] if beam_indices is not None else None + beam_hyp.add(final_tokens, final_score, beam_indices=beam_index) + + # select the best hypotheses + sent_lengths = flow.zeros( + batch_size * self.num_beam_hyps_to_keep, + dtype=flow.long, + sbp=input_ids.sbp, + placement=input_ids.placement, + ) + best = [] + best_indices = [] + best_scores = flow.zeros( + batch_size * self.num_beam_hyps_to_keep, + dtype=flow.float32, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=flow.placement("cuda", list(range(dist.get_world_size()))), + ) + + # retrieve best hypotheses + for i, beam_hyp in enumerate(self._beam_hyps): + sorted_hyps = sorted(beam_hyp.beams, key=lambda x: x[0]) + for j in range(self.num_beam_hyps_to_keep): + best_hyp_tuple = sorted_hyps.pop() + best_score = best_hyp_tuple[0] + best_hyp = best_hyp_tuple[1] + best_index = best_hyp_tuple[2] + sent_lengths[self.num_beam_hyps_to_keep * i + j] = len(best_hyp) + + # append hyp to lists + best.append(best_hyp) + + # append indices to list + best_indices.append(best_index) + + best_scores[i * self.num_beam_hyps_to_keep + j] = best_score + + # prepare for adding eos + sent_lengths_max = sent_lengths.max().item() + 1 + sent_max_len = ( + min(sent_lengths_max, max_length) if max_length is not None else sent_lengths_max + ) + decoded = flow.zeros( + (batch_size * self.num_beam_hyps_to_keep, sent_max_len), + dtype=flow.long, + sbp=input_ids.sbp, + placement=input_ids.placement, + ) + + if len(best_indices) > 0 and best_indices[0] is not None: + indices = flow.zeros( + (batch_size * self.num_beam_hyps_to_keep, sent_max_len), + dtype=flow.long, + sbp=input_ids.sbp, + placement=input_ids.placement, + ) + else: + indices = None + + # shorter batches are padded if needed + if sent_lengths.min().item() != sent_lengths.max().item(): + assert pad_token_id is not None, "`pad_token_id` has to be defined" + decoded.fill_(pad_token_id) + + if indices is not None: + indices.fill_(-1) + + # fill with hypotheses and eos_token_id if the latter fits in + for i, (hypo, best_idx) in enumerate(zip(best, best_indices)): + decoded[i, : sent_lengths[i]] = hypo + + if indices is not None: + indices[i, : len(best_idx)] = flow.tensor(best_idx) + + if sent_lengths[i] < sent_max_len: + decoded[i, sent_lengths[i]] = eos_token_id + + return UserDict( + { + "sequences": decoded, + "sequence_scores": best_scores, + "beam_indices": indices, + } + ) diff --git a/libai/inference/generator/generation_logits_processor.py b/libai/inference/generator/generation_logits_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..7ca04684b0b3544aab3b82b88cc9be60e544b605 --- /dev/null +++ b/libai/inference/generator/generation_logits_processor.py @@ -0,0 +1,385 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright 2020 The Google AI Language Team Authors, Facebook AI Research authors and +# The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import math +from typing import Callable, List, Tuple + +import oneflow as flow + + +class LogitsProcessorList(list): + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor, **kwargs) -> flow.Tensor: + for processor in self: + function_args = inspect.signature(processor.__call__).parameters + if len(function_args) > 2: + if not all(arg in kwargs for arg in list(function_args.keys())[2:]): + raise ValueError( + f"Make sure that all the required parameters: {list(function_args.keys())} " + "for {processor.__class__} are passed to the logits processor." + ) + scores = processor(input_ids, scores, **kwargs) + else: + scores = processor(input_ids, scores) + return scores + + +class NormalizationLogitsProcessor(object): + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor) -> flow.Tensor: + scores = scores.log_softmax(dim=-1) + return scores + + +class InfNanRemoveLogitsProcessor(object): + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor) -> flow.Tensor: + scores[scores != scores] = 0.0 + scores[scores == float("inf")] = flow.finfo(scores.dtype).max + return scores + + +class ForcedEOSTokenLogitsProcessor(object): + def __init__(self, max_length: int, eos_token_id: int): + self.max_length = max_length + self.eos_token_id = eos_token_id + + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor) -> flow.Tensor: + cur_len = input_ids.shape[-1] + if cur_len == self.max_length - 1: + num_tokens = scores.shape[1] + scores[:, [i for i in range(num_tokens) if i != self.eos_token_id]] = -float("inf") + scores[:, self.eos_token_id] = 0 + return scores + + +class ForcedBOSTokenLogitsProcessor(object): + def __init__(self, bos_token_id: int): + self.bos_token_id = bos_token_id + + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor) -> flow.Tensor: + cur_len = input_ids.shape[-1] + if cur_len == 1: + num_tokens = scores.shape[1] + scores[:, [i for i in range(num_tokens) if i != self.bos_token_id]] = -float("inf") + scores[:, self.bos_token_id] = 0 + return scores + + +class RepetitionPenaltyLogitsProcessor(object): + def __init__(self, penalty: float): + if not isinstance(penalty, float) or not (penalty > 0): + raise ValueError(f"`penalty` has to be a strictly positive float, but is {penalty}") + self.penalty = penalty + + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor) -> flow.Tensor: + score = flow.gather(scores, 1, input_ids) + score = flow.where(score < 0, score * self.penalty, score / self.penalty) + scores = flow.scatter(scores, 1, input_ids, score) + return scores + + +class HammingDiversityLogitsProcessor(object): + def __init__(self, diversity_penalty: float, num_beams: int, num_beam_groups: int): + if not isinstance(diversity_penalty, float) or (not diversity_penalty > 0.0): + raise ValueError("`diversity_penalty` should be a float strictly larger than 0.") + self._diversity_penalty = diversity_penalty + if not isinstance(num_beams, int) or num_beams < 2: + raise ValueError("`num_beams` should be an integer strictly larger than 1.") + self._num_beams = num_beams + if not isinstance(num_beam_groups, int) or num_beam_groups < 2: + raise ValueError("`num_beam_groups` should be an integer strictly larger than 1.") + if num_beam_groups > num_beams: + raise ValueError("`beam_groups` has to be smaller or equal to `num_beams`.") + self._num_sub_beams = num_beams // num_beam_groups + + def __call__(self, input_ids, scores, current_tokens, beam_group_idx) -> flow.Tensor: + scores = scores.numpy() + + batch_size = current_tokens.shape[0] // self._num_beams + group_start_idx = beam_group_idx * self._num_sub_beams + group_end_idx = min(group_start_idx + self._num_sub_beams, self._num_beams) + group_size = group_end_idx - group_start_idx + vocab_size = scores.shape[-1] + + if group_start_idx == 0: + return scores + + for batch_idx in range(batch_size): + # predicted tokens of last time step of previous groups + previous_group_tokens = current_tokens[ + batch_idx * self._num_beams : batch_idx * self._num_beams + group_start_idx + ] + token_frequency = flow.bincount(previous_group_tokens, minlength=vocab_size) + scores[batch_idx * group_size : (batch_idx + 1) * group_size] = ( + scores[batch_idx * group_size : (batch_idx + 1) * group_size] + - self._diversity_penalty * token_frequency + ) + + return scores + + +def _get_ngrams(ngram_size: int, prev_input_ids: flow.Tensor, num_hypos: int): + generated_ngrams = [{} for _ in range(num_hypos)] + for idx in range(num_hypos): + gen_tokens = prev_input_ids[idx].tolist() + generated_ngram = generated_ngrams[idx] + for ngram in zip(*[gen_tokens[i:] for i in range(ngram_size)]): + prev_ngram_tuple = tuple(ngram[:-1]) + generated_ngram[prev_ngram_tuple] = generated_ngram.get(prev_ngram_tuple, []) + [ + ngram[-1] + ] + return generated_ngrams + + +def _get_generated_ngrams(banned_ngrams, prev_input_ids, ngram_size, cur_len): + start_idx = cur_len + 1 - ngram_size + ngram_idx = tuple(prev_input_ids[start_idx:cur_len].tolist()) + return banned_ngrams.get(ngram_idx, []) + + +def _calc_banned_ngram_tokens( + ngram_size: int, prev_input_ids: flow.Tensor, num_hypos: int, cur_len: int +): + if cur_len + 1 < ngram_size: + return [[] for _ in range(num_hypos)] + + generated_ngrams = _get_ngrams(ngram_size, prev_input_ids, num_hypos) + + banned_tokens = [ + _get_generated_ngrams( + generated_ngrams[hypo_idx], prev_input_ids[hypo_idx], ngram_size, cur_len + ) + for hypo_idx in range(num_hypos) + ] + return banned_tokens + + +class NoRepeatNGramLogitsProcessor(object): + def __init__(self, ngram_size: int): + if not isinstance(ngram_size, int) or ngram_size <= 0: + raise ValueError( + f"`ngram_size` has to be a strictly positive integer, but is {ngram_size}" + ) + self.ngram_size = ngram_size + + def __call__(self, input_ids, scores) -> flow.Tensor: + num_batch_hypotheses = scores.shape[0] + cur_len = input_ids.shape[-1] + banned_batch_tokens = _calc_banned_ngram_tokens( + self.ngram_size, input_ids, num_batch_hypotheses, cur_len + ) + + for i, banned_tokens in enumerate(banned_batch_tokens): + scores[i, banned_tokens] = -float("inf") + + return scores + + +class EncoderNoRepeatNGramLogitsProcessor(object): + def __init__(self, encoder_ngram_size: int, encoder_input_ids: flow.Tensor): + if not isinstance(encoder_ngram_size, int) or encoder_ngram_size <= 0: + raise ValueError( + "`encoder_ngram_size` has to be a strictly positive integer, but is " + f"{encoder_ngram_size}" + ) + self.ngram_size = encoder_ngram_size + if len(encoder_input_ids.shape) == 1: + encoder_input_ids = encoder_input_ids.unsqueeze(0) + self.batch_size = encoder_input_ids.shape[0] + self.generated_ngrams = _get_ngrams(encoder_ngram_size, encoder_input_ids, self.batch_size) + + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor) -> flow.Tensor: + # B x num_beams + num_hypos = scores.shape[0] + num_beams = num_hypos // self.batch_size + cur_len = input_ids.shape[-1] + banned_batch_tokens = [ + _get_generated_ngrams( + self.generated_ngrams[hypo_idx // num_beams], + input_ids[hypo_idx], + self.ngram_size, + cur_len, + ) + for hypo_idx in range(num_hypos) + ] + + for i, banned_tokens in enumerate(banned_batch_tokens): + scores[i, banned_tokens] = -float("inf") + + return scores + + +class MinLengthLogitsProcessor(object): + def __init__(self, min_length: int, eos_token_id: int): + if not isinstance(min_length, int) or min_length < 0: + raise ValueError(f"`min_length` has to be a positive integer, but is {min_length}") + + if not isinstance(eos_token_id, int) or eos_token_id < 0: + raise ValueError(f"`eos_token_id` has to be a positive integer, but is {eos_token_id}") + + self.min_length = min_length + self.eos_token_id = eos_token_id + + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor) -> flow.Tensor: + cur_len = input_ids.shape[-1] + if cur_len < self.min_length: + scores[:, self.eos_token_id] = -float("inf") + return scores + + +class PrefixConstrainedLogitsProcessor(object): + def __init__( + self, prefix_allowed_tokens_fn: Callable[[int, flow.Tensor], List[int]], num_beams: int + ): + self._prefix_allowed_tokens_fn = prefix_allowed_tokens_fn + self._num_beams = num_beams + + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor) -> flow.Tensor: + mask = flow.full_like(scores, -math.inf) + for batch_id, beam_sent in enumerate( + input_ids.view(-1, self._num_beams, input_ids.shape[-1]) + ): + for beam_id, sent in enumerate(beam_sent): + mask[ + batch_id * self._num_beams + beam_id, + self._prefix_allowed_tokens_fn(batch_id, sent), + ] = 0 + + return scores + mask + + +class ExponentialDecayLengthPenalty(object): + def __init__( + self, exponential_decay_length_penalty: Tuple, eos_token_id: int, input_ids_seq_length: int + ): + self.regulation_start = exponential_decay_length_penalty[0] + input_ids_seq_length + self.regulation_factor = exponential_decay_length_penalty[1] + self.eos_token_id = eos_token_id + + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor) -> flow.Tensor: + cur_len = input_ids.shape[-1] + if cur_len > self.regulation_start: + scores[:, self.eos_token_id] = scores[:, self.eos_token_id] * pow( + self.regulation_factor, cur_len - self.regulation_start + ) + return scores + + +class TemperatureLogitsWarper(object): + def __init__(self, temperature: float): + if not isinstance(temperature, float) or not (temperature > 0): + raise ValueError( + f"`temperature` has to be a strictly positive float, but is {temperature}" + ) + self.temperature = temperature + + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor) -> flow.Tensor: + scores = scores / self.temperature + return scores + + +class TopPLogitsWarper(object): + def __init__( + self, top_p: float, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1 + ): + top_p = float(top_p) + if top_p < 0 or top_p > 1.0: + raise ValueError(f"`top_p` has to be a float > 0 and < 1, but is {top_p}") + + self.top_p = top_p + self.filter_value = filter_value + self.min_tokens_to_keep = min_tokens_to_keep + + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor) -> flow.Tensor: + sorted_logits, sorted_indices = flow.sort(scores, descending=True) + cumulative_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1) + + # Remove tokens with cumulative top_p above the threshold (token with 0 are kept) + sorted_indices_to_remove = cumulative_probs > self.top_p + if self.min_tokens_to_keep > 1: + # Keep at least min_tokens_to_keep (set to min_tokens_to_keep-1 + # because we add the first one below) + sorted_indices_to_remove[..., : self.min_tokens_to_keep - 1] = 0 + + # Shift the indices to the right to keep also the first token above the threshold + sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() + sorted_indices_to_remove[..., 0] = 0 + + # scatter sorted tensors to original indexing + indices_to_remove = flow.scatter( + sorted_indices_to_remove, 1, sorted_indices, sorted_indices_to_remove + ) + scores = scores.masked_fill(indices_to_remove, self.filter_value) + return scores + + +class TopKLogitsWarper(object): + def __init__( + self, top_k: int, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1 + ): + if not isinstance(top_k, int) or top_k <= 0: + raise ValueError(f"`top_k` has to be a strictly positive integer, but is {top_k}") + + self.top_k = top_k + self.filter_value = filter_value + self.min_tokens_to_keep = min_tokens_to_keep + + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor) -> flow.Tensor: + top_k = min(max(self.top_k, self.min_tokens_to_keep), scores.size(-1)) # Safety check + # Remove all tokens with a probability less than the last token of the top-k + indices_to_remove = scores < flow.topk(scores, top_k)[0][..., -1, None] + scores = scores.masked_fill(indices_to_remove, self.filter_value) + return scores + + +class TypicalLogitsWarper(object): + def __init__( + self, mass: float = 0.9, filter_value: float = -float("Inf"), min_tokens_to_keep: int = 1 + ): + mass = float(mass) + if not (mass > 0 and mass < 1): + raise ValueError(f"`typical_p` has to be a float > 0 and < 1, but is {mass}") + + self.filter_value = filter_value + self.mass = mass + self.min_tokens_to_keep = min_tokens_to_keep + + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor) -> flow.Tensor: + + # calculate entropy + normalized = flow.nn.functional.log_softmax(scores, dim=-1) + p = flow.exp(normalized) + ent = -flow.nansum(normalized * p, dim=-1, keepdim=True) + + # shift and sort + shifted_scores = flow.abs((-normalized) - ent) + sorted_scores, sorted_indices = flow.sort(shifted_scores, descending=False) + sorted_logits = scores.gather(-1, sorted_indices) + cumulative_probs = sorted_logits.softmax(dim=-1).cumsum(dim=-1) + + # Remove tokens with cumulative mass above the threshold + last_ind = (cumulative_probs < self.mass).sum(dim=1) + last_ind[last_ind < 0] = 0 + sorted_indices_to_remove = sorted_scores > sorted_scores.gather(1, last_ind.view(-1, 1)) + if self.min_tokens_to_keep > 1: + # Keep at least min_tokens_to_keep + # (set to min_tokens_to_keep-1 because we add the first one below) + sorted_indices_to_remove[..., : self.min_tokens_to_keep] = 0 + indices_to_remove = flow.scatter( + sorted_indices_to_remove, 1, sorted_indices, sorted_indices_to_remove + ) + scores = scores.masked_fill(indices_to_remove, self.filter_value) + return scores diff --git a/libai/inference/generator/generation_stopping_criteria.py b/libai/inference/generator/generation_stopping_criteria.py new file mode 100644 index 0000000000000000000000000000000000000000..d7dec3b692a0635d241e335166a9491d614bacfa --- /dev/null +++ b/libai/inference/generator/generation_stopping_criteria.py @@ -0,0 +1,64 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright 2020 The Google AI Language Team Authors, Facebook AI Research authors and +# The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +import warnings +from copy import deepcopy + +import oneflow as flow + + +class StoppingCriteriaList(list): + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor, **kwargs) -> bool: + return any(criteria(input_ids, scores) for criteria in self) + + @property + def max_length(self): + for stopping_criterium in self: + if isinstance(stopping_criterium, MaxLengthCriteria): + return stopping_criterium.max_length + return None + + +class MaxLengthCriteria(object): + def __init__(self, max_length: int): + self.max_length = max_length + + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor) -> bool: + return input_ids.shape[-1] >= self.max_length + + +class MaxTimeCriteria(object): + def __init__(self, max_time: float, initial_timestamp: float = None): + self.max_time = max_time + self.initial_timestamp = time.time() if initial_timestamp is None else initial_timestamp + + def __call__(self, input_ids: flow.Tensor, scores: flow.Tensor, **kwargs) -> bool: + return time.time() - self.initial_timestamp > self.max_time + + +def validate_stopping_criteria(stopping_criteria: StoppingCriteriaList, max_length: int): + stopping_max_length = stopping_criteria.max_length + new_stopping_criteria = deepcopy(stopping_criteria) + if stopping_max_length is not None and stopping_max_length != max_length: + warnings.warn( + "You set different `max_length` for stopping criteria and `max_length` parameter", + UserWarning, + ) + elif stopping_max_length is None: + new_stopping_criteria.append(MaxLengthCriteria(max_length=max_length)) + return new_stopping_criteria diff --git a/libai/inference/generator/generation_utils.py b/libai/inference/generator/generation_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..73df306b675c593b57245d64a0401367b699bf2e --- /dev/null +++ b/libai/inference/generator/generation_utils.py @@ -0,0 +1,1073 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright 2020 The Google AI Language Team Authors, Facebook AI Research authors and +# The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import logging +import warnings +from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union + +import oneflow as flow +from oneflow import nn + +from libai.utils import distributed as dist + +from .generation_beam_search import BeamScorer, BeamSearchScorer +from .generation_logits_processor import ( + EncoderNoRepeatNGramLogitsProcessor, + ExponentialDecayLengthPenalty, + ForcedBOSTokenLogitsProcessor, + ForcedEOSTokenLogitsProcessor, + HammingDiversityLogitsProcessor, + InfNanRemoveLogitsProcessor, + LogitsProcessorList, + MinLengthLogitsProcessor, + NoRepeatNGramLogitsProcessor, + NormalizationLogitsProcessor, + PrefixConstrainedLogitsProcessor, + RepetitionPenaltyLogitsProcessor, + TemperatureLogitsWarper, + TopKLogitsWarper, + TopPLogitsWarper, + TypicalLogitsWarper, +) +from .generation_stopping_criteria import ( + MaxLengthCriteria, + MaxTimeCriteria, + StoppingCriteriaList, + validate_stopping_criteria, +) + +logger = logging.getLogger(__name__) + + +class Generator: + def _prepare_model_inputs( + self, + inputs: Optional[flow.Tensor] = None, + bos_token_id: Optional[int] = None, + model_kwargs: Optional[Dict[str, flow.Tensor]] = None, + ): + if self.cfg.is_encoder_decoder: + input_name = "encoder_input_ids" + else: + input_name = "input_ids" + + model_kwargs = {k: v for k, v in model_kwargs.items() if v is not None or k != input_name} + + inputs_kwarg = model_kwargs.pop(input_name, None) + if inputs_kwarg is not None and inputs is not None: + raise ValueError( + f"`inputs`: {inputs}` were passed alongside " + f"{input_name} which is not allowed." + f"Make sure to either pass {inputs} or {input_name}=..." + ) + elif inputs_kwarg is not None: + inputs = inputs_kwarg + + if inputs is None: + inputs = self._prepare_input_ids_for_generation( + bos_token_id, model_kwargs.get("encoder_outputs", None) + ) + return inputs, input_name, model_kwargs + + def prepare_inputs_for_generation(self, input_ids: flow.Tensor, **kwargs): + """ + Implement in subclasses of [`PreTrainedModel`] for custom behavior to prepare inputs in the + generate method. + """ + return {"input_ids": input_ids} + + def _prepare_input_ids_for_generation( + self, bos_token_id: Optional[int], encoder_outputs: Optional[flow.Tensor] + ): + if self.cfg.is_encoder_decoder and encoder_outputs is not None: + shape = encoder_outputs.size()[:-1] + return ( + flow.ones( + shape, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=flow.placement("cuda", list(range(dist.get_world_size()))), + ) + * -100 + ) + + if bos_token_id is None: + raise ValueError("`bos_token_id` has to be defined when no `input_ids` are provided.") + return ( + flow.ones( + (1, 1), + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=flow.placement("cuda", list(range(dist.get_world_size()))), + ) + * bos_token_id + ) + + def _prepare_attention_mask_for_generation( + self, + inputs: flow.Tensor, + pad_token_id: Optional[int], + eos_token_id: Optional[int], + ): + is_input_ids = len(inputs.shape) == 2 and inputs.dtype in [flow.int64, flow.long] + is_pad_token_in_inputs = (pad_token_id is not None) and (pad_token_id in inputs) + is_pad_token_not_equal_to_eos_token_id = (eos_token_id is None) or ( + (eos_token_id is not None) and (pad_token_id != eos_token_id) + ) + # Check if input is input_ids and padded -> only then is attention_mask defined + if is_input_ids and is_pad_token_in_inputs and is_pad_token_not_equal_to_eos_token_id: + return inputs.ne(pad_token_id).bool() + else: + return flow.ones( + inputs.shape[:2], + dtype=flow.bool, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=flow.placement("cuda", list(range(dist.get_world_size()))), + ) + + def _prepare_encoder_decoder_kwargs_for_generation( + self, inputs_tensor: flow.Tensor, model_kwargs, model_input_name: str + ): + only_encoder = True + model_kwargs[model_input_name] = inputs_tensor + if "encoder_decoder_attn_mask" in set(inspect.signature(self.forward).parameters): + model_kwargs["encoder_decoder_attn_mask"] = model_kwargs["encoder_attn_mask"] + model_kwargs["encoder_outputs"] = self(**model_kwargs, only_encoder=only_encoder) + model_kwargs.pop(model_input_name) + return model_kwargs + + def _prepare_decoder_input_ids_for_generation( + self, + batch_size: int, + decoder_start_token_id: int = None, + bos_token_id: int = None, + model_kwargs=None, + ): + if model_kwargs is not None and "decoder_input_ids" in model_kwargs: + return model_kwargs.pop("decoder_input_ids") + else: + decoder_start_token_id = ( + decoder_start_token_id + if decoder_start_token_id + else self.cfg.decoder_start_token_id + ) + return ( + flow.ones( + (batch_size, 1), + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=flow.placement("cuda", list(range(dist.get_world_size()))), + ) + * decoder_start_token_id + ) + + def _get_decoder_start_token_id( + self, decoder_start_token_id: int = None, bos_token_id: int = None + ): + if decoder_start_token_id is not None: + return decoder_start_token_id + elif self.cfg.is_encoder_decoder: + return self.cfg.decoder_start_token_id + elif bos_token_id is not None: + return bos_token_id + else: + return self.cfg.bos_token_idx + + @staticmethod + def _expand_inputs_for_generation( + input_ids: flow.Tensor, + expand_size: int = 1, + is_encoder_decoder: bool = False, + attention_mask: Optional[flow.Tensor] = None, + encoder_outputs: Optional[flow.Tensor] = None, + **model_kwargs, + ): + expanded_return_idx = ( + flow.arange(input_ids.shape[0]).view(-1, 1).repeat(1, expand_size).view(-1) + ) + expanded_return_idx = expanded_return_idx.to_global( + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=flow.placement("cuda", list(range(dist.get_world_size()))), + ) + + input_ids = input_ids.index_select(0, expanded_return_idx) + + # token_type ids not supported. + + if attention_mask is not None: + model_kwargs["attention_mask"] = attention_mask.index_select(0, expanded_return_idx) + + if is_encoder_decoder: + if encoder_outputs is None: + raise ValueError( + "If `is_encoder_decoder` is True, make sure that `encoder_outputs` is defined." + ) + encoder_outputs = encoder_outputs.to_global(placement=expanded_return_idx.placement) + encoder_outputs = encoder_outputs.index_select(0, expanded_return_idx) + model_kwargs["encoder_outputs"] = encoder_outputs + model_kwargs["encoder_attn_mask"] = model_kwargs["encoder_attn_mask"].index_select( + 0, expanded_return_idx + ) + model_kwargs["encoder_decoder_attn_mask"] = model_kwargs["encoder_attn_mask"] + return input_ids, model_kwargs + + def _update_model_kwargs_for_generation( + self, outputs, model_kwargs, is_encoder_decoder: bool = False + ): + if "past_key_values" in outputs: + model_kwargs["past"] = outputs["past_key_values"] + elif "mems" in outputs: + model_kwargs["past"] = outputs["mems"] + elif "past_buckets_states" in outputs: + model_kwargs["past"] = outputs["past_buckets_states"] + elif self.past_key_values[-1] is not None: + model_kwargs["past"] = self.past_key_values + else: + model_kwargs["past"] = None + + # update attention mask + if "attention_mask" in model_kwargs and not is_encoder_decoder: + attention_mask = model_kwargs["attention_mask"] + pad = flow.ones( + (attention_mask.shape[0], 1), + sbp=attention_mask.sbp, + placement=attention_mask.placement, + ) + model_kwargs["attention_mask"] = flow.cat([attention_mask, pad], dim=-1) + + if "decoder_attn_mask" in model_kwargs and is_encoder_decoder: + attention_mask = model_kwargs["decoder_attn_mask"] + pad = flow.ones( + (attention_mask.shape[0], 1), + sbp=attention_mask.sbp, + placement=attention_mask.placement, + ) + model_kwargs["decoder_attn_mask"] = flow.cat([attention_mask, pad], dim=-1) + + return model_kwargs + + def _reorder_cache(self, past, beam_idx): + raise NotImplementedError( + "Make sure that a `_reorder_cache` function is correctly implemented in " + f"{self.__class__.__module__} to enable beam search for {self.__class__}" + ) + + def _get_logits_warper( + self, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + typical_p: Optional[float] = None, + temperature: Optional[float] = None, + num_beams: Optional[int] = None, + renormalize_logits: Optional[bool] = None, + ): + # instantiate warpers list + warpers = LogitsProcessorList() + + # all samplers can be found in `generation_utils_samplers.py` + if temperature is not None and temperature != 1.0: + warpers.append(TemperatureLogitsWarper(temperature)) + if top_k is not None and top_k != 0: + warpers.append( + TopKLogitsWarper(top_k=top_k, min_tokens_to_keep=(2 if num_beams > 1 else 1)) + ) + if top_p is not None and top_p < 1.0: + warpers.append( + TopPLogitsWarper(top_p=top_p, min_tokens_to_keep=(2 if num_beams > 1 else 1)) + ) + if typical_p is not None and typical_p < 1.0: + warpers.append( + TypicalLogitsWarper(mass=typical_p, min_tokens_to_keep=(2 if num_beams > 1 else 1)) + ) + # `LogitNormalization` should always be the last logit processor, when present + if renormalize_logits is True: + warpers.append(NormalizationLogitsProcessor()) + return warpers + + def _get_logits_processor( + self, + repetition_penalty: float, + no_repeat_ngram_size: int, + encoder_no_repeat_ngram_size: int, + input_ids_seq_length: int, + encoder_input_ids: flow.Tensor, + min_length: int, + max_length: int, + eos_token_id: int, + forced_bos_token_id: int, + forced_eos_token_id: int, + prefix_allowed_tokens_fn: Callable[[int, flow.Tensor], List[int]], + num_beams: int, + num_beam_groups: int, + diversity_penalty: float, + remove_invalid_values: bool, + exponential_decay_length_penalty: Tuple, + logits_processor: Optional[LogitsProcessorList], + renormalize_logits: Optional[bool], + ): + """ + This class returns a [`LogitsProcessorList`] list object that contains all relevant + [`LogitsProcessor`] instances used to modify the scores of the language model head. + """ + processors = LogitsProcessorList() + + # instantiate processors list + if diversity_penalty is not None and diversity_penalty > 0.0: + processors.append( + HammingDiversityLogitsProcessor( + diversity_penalty=diversity_penalty, + num_beams=num_beams, + num_beam_groups=num_beam_groups, + ) + ) + if repetition_penalty is not None and repetition_penalty != 1.0: + processors.append(RepetitionPenaltyLogitsProcessor(penalty=repetition_penalty)) + if no_repeat_ngram_size is not None and no_repeat_ngram_size > 0: + processors.append(NoRepeatNGramLogitsProcessor(no_repeat_ngram_size)) + if encoder_no_repeat_ngram_size is not None and encoder_no_repeat_ngram_size > 0: + if self.cfg.is_encoder_decoder: + processors.append( + EncoderNoRepeatNGramLogitsProcessor( + encoder_no_repeat_ngram_size, encoder_input_ids + ) + ) + else: + raise ValueError( + "It's impossible to use `encoder_no_repeat_ngram_size` with decoder-only " + "architecture" + ) + if min_length is not None and eos_token_id is not None and min_length > 0: + processors.append(MinLengthLogitsProcessor(min_length, eos_token_id)) + if prefix_allowed_tokens_fn is not None: + processors.append( + PrefixConstrainedLogitsProcessor( + prefix_allowed_tokens_fn, num_beams // num_beam_groups + ) + ) + if forced_bos_token_id is not None: + processors.append(ForcedBOSTokenLogitsProcessor(forced_bos_token_id)) + if forced_eos_token_id is not None: + processors.append(ForcedEOSTokenLogitsProcessor(max_length, forced_eos_token_id)) + if remove_invalid_values is True: + processors.append(InfNanRemoveLogitsProcessor()) + if exponential_decay_length_penalty is not None: + processors.append( + ExponentialDecayLengthPenalty( + exponential_decay_length_penalty, eos_token_id, input_ids_seq_length + ) + ) + processors = self._merge_criteria_processor_list(processors, logits_processor) + # `LogitNormalization` should always be the last logit processor, when present + if renormalize_logits is True: + processors.append(NormalizationLogitsProcessor()) + return processors + + def _get_stopping_criteria( + self, + max_length: Optional[int], + max_time: Optional[float], + stopping_criteria: Optional[StoppingCriteriaList], + ): + criteria = StoppingCriteriaList() + if max_length is not None: + criteria.append(MaxLengthCriteria(max_length=max_length)) + if max_time is not None: + criteria.append(MaxTimeCriteria(max_time=max_time)) + criteria = self._merge_criteria_processor_list(criteria, stopping_criteria) + return criteria + + def _merge_criteria_processor_list(self, default_list, custom_list): + if len(custom_list) == 0: + return default_list + for default in default_list: + for custom in custom_list: + if type(custom) is type(default): + raise ValueError("Criteria repetition error.") + default_list.extend(custom_list) + return default_list + + def compute_transition_beam_scores( + self, + sequences: flow.Tensor, + scores: Tuple[flow.Tensor], + beam_indices: flow.Tensor, + eos_token_id: int = None, + ): + scores = flow.stack(scores).reshape(len(scores), -1).transpose(0, 1) + + beam_indices_mask = beam_indices < 0 + max_beam_length = (1 - beam_indices_mask.long()).sum(-1).max() + beam_indices = beam_indices[:, :max_beam_length] + beam_indices_mask = beam_indices_mask[:, :max_beam_length] + + beam_indices[beam_indices_mask] = 0 + + beam_sequence_indices = beam_indices * self.cfg.vocab_size + + cut_idx = sequences.shape[-1] - max_beam_length + indices = sequences[:, cut_idx:] + beam_sequence_indices + + transition_scores = scores.gather(0, indices) + + transition_scores[beam_indices_mask] = 0 + + return transition_scores + + def _validate_model_kwargs(self, model_kwargs): + if self.cfg.is_encoder_decoder: + for key in ["decoder_input_ids"]: + model_kwargs.pop(key, None) + unused_model_args = [] + model_args = set(inspect.signature(self.prepare_inputs_for_generation).parameters) + if "kwargs" in model_args: + model_args |= set(inspect.signature(self.forward).parameters) + for key, value in model_kwargs.items(): + if value is not None and key not in model_args: + unused_model_args.append(key) + if unused_model_args: + raise ValueError( + f"The following `model_kwargs` are not used by the model: {unused_model_args} " + "(note: typos in the generate arguments will also show up in this list)" + ) + + def greedy_search( + self, + input_ids: flow.Tensor, + logits_processor: Optional[LogitsProcessorList] = None, + stopping_criteria: Optional[StoppingCriteriaList] = None, + max_length: Optional[int] = None, + pad_token_id: Optional[int] = None, + eos_token_id: Optional[int] = None, + is_encoder_decoder: bool = False, + output_scores: bool = False, + **model_kwargs, + ): + pad_token_id = pad_token_id if pad_token_id is not None else self.cfg.pad_token_id + eos_token_id = eos_token_id if eos_token_id is not None else self.cfg.eos_token_id + output_scores = output_scores if output_scores is not None else self.cfg.output_scores + scores = () if output_scores else None + logits_processor = ( + logits_processor if logits_processor is not None else LogitsProcessorList() + ) + stopping_criteria = ( + stopping_criteria if stopping_criteria is not None else StoppingCriteriaList() + ) + if max_length is not None: + warnings.warn( + "`max_length` is deprecated in this function, use MaxLengthCriteria" " instead.", + UserWarning, + ) + stopping_criteria = validate_stopping_criteria(stopping_criteria, max_length) + + # keep track of which sequences are already finished + unfinished_sequences = flow.ones(input_ids.shape[0]) + cur_len = input_ids.shape[-1] + while True: + # prepare model inputs + model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs) + + # generate + outputs = self(**model_inputs) + next_token_logits = outputs["logits"][:, -1, :] + + # logits_processor + next_token_scores = logits_processor(input_ids, next_token_logits) + + # Store scores + if output_scores: + scores += (next_token_scores,) + + # argmax + next_tokens = flow.argmax(next_token_scores, dim=-1) + next_tokens = next_tokens.to_global(placement=input_ids.placement) + unfinished_sequences = unfinished_sequences.to_global( + sbp=next_tokens.sbp, placement=next_tokens.placement + ) + + if eos_token_id is not None: + if pad_token_id is None: + raise ValueError( + "If `eos_token_id` is defined, make sure that `pad_token_id` is defined." + ) + next_tokens = next_tokens * unfinished_sequences + pad_token_id * ( + 1 - unfinished_sequences + ) + + next_tokens = next_tokens.to(flow.long) + input_ids = flow.cat([input_ids, next_tokens[:, None]], dim=-1) + model_kwargs = self._update_model_kwargs_for_generation( + outputs, model_kwargs, is_encoder_decoder=is_encoder_decoder + ) + cur_len = cur_len + 1 + + # if eos_token was found in one sentence, set sentence to finished + if eos_token_id is not None: + unfinished_sequences = flow.mul( + unfinished_sequences, (next_tokens != eos_token_id).long() + ) + + if unfinished_sequences.max() == 0 or stopping_criteria(input_ids, scores): + break + + # Release records + if "past_key_values" in self.__dir__(): + self.past_key_values = [None] * self.cfg.hidden_layers + if "encoder_states" in self.__dir__(): + self.encoder_states = None + + return input_ids + + def multinomial_sample( + self, + input_ids: flow.Tensor, + logits_processor: Optional[LogitsProcessorList] = None, + stopping_criteria: Optional[StoppingCriteriaList] = None, + logits_warper: Optional[LogitsProcessorList] = None, + max_length: Optional[int] = None, + pad_token_id: Optional[int] = None, + eos_token_id: Optional[int] = None, + is_encoder_decoder: bool = False, + output_scores: bool = False, + **model_kwargs, + ): + # init values + pad_token_id = pad_token_id if pad_token_id is not None else self.cfg.pad_token_id + eos_token_id = eos_token_id if eos_token_id is not None else self.cfg.eos_token_id + output_scores = output_scores if output_scores is not None else self.cfg.output_scores + scores = () if output_scores else None + logits_processor = ( + logits_processor if logits_processor is not None else LogitsProcessorList() + ) + stopping_criteria = ( + stopping_criteria if stopping_criteria is not None else StoppingCriteriaList() + ) + if max_length is not None: + warnings.warn( + "`max_length` is deprecated in this function, use " + "`stopping_criteria=StoppingCriteriaList(MaxLengthCriteria(max_length=max_length))`" + "instead.", + UserWarning, + ) + stopping_criteria = validate_stopping_criteria(stopping_criteria, max_length) + logits_warper = logits_warper if logits_warper is not None else LogitsProcessorList() + + unfinished_sequences = flow.ones(input_ids.shape[0]) + cur_len = input_ids.shape[-1] + + while True: + # prepare model inputs + model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs) + + # generate + outputs = self(**model_inputs) + next_token_logits = outputs["logits"][:, -1, :] + + # pre-process distribution + next_token_scores = logits_processor(input_ids, next_token_logits) + next_token_scores = logits_warper(input_ids, next_token_scores) + + # Store scores + if output_scores: + scores += (next_token_scores,) + + # sample + probs = nn.functional.softmax(next_token_scores, dim=-1) + probs = probs.to_global( + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=flow.placement("cuda", list(range(dist.get_world_size()))), + ).to_local() + next_tokens = flow.multinomial(probs, num_samples=1).squeeze(1) + next_tokens = next_tokens.to_global( + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=flow.placement("cuda", list(range(dist.get_world_size()))), + ) + unfinished_sequences = unfinished_sequences.to_global( + sbp=next_tokens.sbp, placement=next_tokens.placement + ) + + if eos_token_id is not None: + if pad_token_id is None: + raise ValueError( + "If `eos_token_id` is defined, make sure that `pad_token_id` is defined." + ) + next_tokens = next_tokens * unfinished_sequences + pad_token_id * ( + 1 - unfinished_sequences + ) + + next_tokens = next_tokens.to(flow.long) + input_ids = flow.cat([input_ids, next_tokens[:, None]], dim=-1) + + model_kwargs = self._update_model_kwargs_for_generation( + outputs, model_kwargs, is_encoder_decoder=is_encoder_decoder + ) + cur_len = cur_len + 1 + + if eos_token_id is not None: + unfinished_sequences = flow.mul( + unfinished_sequences, (next_tokens != eos_token_id).long() + ) + + if unfinished_sequences.max() == 0 or stopping_criteria(input_ids, scores): + break + + # Release records + if "past_key_values" in self.__dir__(): + self.past_key_values = [None] * self.cfg.hidden_layers + if "encoder_states" in self.__dir__(): + self.encoder_states = None + + return input_ids + + def beam_search( + self, + input_ids: flow.Tensor, + beam_scorer: BeamScorer, + logits_processor: Optional[LogitsProcessorList] = None, + stopping_criteria: Optional[StoppingCriteriaList] = None, + max_length: Optional[int] = None, + pad_token_id: Optional[int] = None, + eos_token_id: Optional[int] = None, + is_encoder_decoder: bool = False, + output_scores: bool = False, + **model_kwargs, + ): + pad_token_id = pad_token_id if pad_token_id is not None else self.cfg.pad_token_id + eos_token_id = eos_token_id if eos_token_id is not None else self.cfg.eos_token_id + output_scores = output_scores if output_scores is not None else self.cfg.output_scores + scores = () if output_scores else None + logits_processor = ( + logits_processor if logits_processor is not None else LogitsProcessorList() + ) + stopping_criteria = ( + stopping_criteria if stopping_criteria is not None else StoppingCriteriaList() + ) + if max_length is not None: + warnings.warn( + "`max_length` is deprecated in this function, use " + "`stopping_criteria=StoppingCriteriaList(MaxLengthCriteria(max_length=max_length))`" + "instead.", + UserWarning, + ) + stopping_criteria = validate_stopping_criteria(stopping_criteria, max_length) + if len(stopping_criteria) == 0: + warnings.warn( + "You don't have defined any stopping_criteria, this will likely loop forever", + UserWarning, + ) + + batch_size = len(beam_scorer._beam_hyps) + num_beams = beam_scorer.num_beams + + batch_beam_size, cur_len = input_ids.shape + + if num_beams * batch_size != batch_beam_size: + raise ValueError( + f"Batch dimension of `input_ids` should be {num_beams * batch_size}, " + f"but is {batch_beam_size}." + ) + + beam_indices = None + + beam_scores = flow.zeros( + (batch_size, num_beams), + dtype=flow.float, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=flow.placement("cuda", list(range(dist.get_world_size()))), + ) + beam_scores[:, 1:] = -1e9 + beam_scores = beam_scores.view((batch_size * num_beams,)) + + while True: + # prepare model inputs + model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs) + + outputs = self(**model_inputs) + next_token_logits = outputs["logits"][:, -1, :] + + next_token_scores = nn.functional.log_softmax( + next_token_logits, dim=-1 + ) # (batch_size * num_beams, vocab_size) + next_token_scores = next_token_scores.to_global( + sbp=input_ids.sbp, placement=input_ids.placement + ) + + next_token_scores_processed = logits_processor(input_ids, next_token_scores) + next_token_scores = next_token_scores_processed + beam_scores[:, None].expand_as( + next_token_scores + ) + + # Store scores + if output_scores: + scores += (next_token_scores,) + + # reshape for beam search + vocab_size = next_token_scores.shape[-1] + next_token_scores = next_token_scores.view(batch_size, num_beams * vocab_size) + + next_token_scores, next_tokens = flow.topk( + next_token_scores, 2 * num_beams, dim=1, largest=True, sorted=True + ) + next_indices = next_tokens // vocab_size + next_tokens = next_tokens % vocab_size + + beam_outputs = beam_scorer.process( + input_ids, + next_token_scores, + next_tokens, + next_indices, + pad_token_id=pad_token_id, + eos_token_id=eos_token_id, + beam_indices=beam_indices, + ) + + beam_scores = beam_outputs["next_beam_scores"] + beam_next_tokens = beam_outputs["next_beam_tokens"] + beam_idx = beam_outputs["next_beam_indices"] + + input_ids = flow.cat([input_ids[beam_idx, :], beam_next_tokens.unsqueeze(-1)], dim=-1) + + model_kwargs = self._update_model_kwargs_for_generation( + outputs, model_kwargs, is_encoder_decoder=is_encoder_decoder + ) + + # update past_key_value + if model_kwargs["past"] is not None: + model_kwargs["past"] = self._reorder_cache(beam_idx) + + # increase cur_len + cur_len = cur_len + 1 + + if beam_scorer.is_done or stopping_criteria(input_ids, scores): + break + + sequence_outputs = beam_scorer.finalize( + input_ids, + beam_scores, + next_tokens, + next_indices, + pad_token_id=pad_token_id, + eos_token_id=eos_token_id, + max_length=stopping_criteria.max_length, + beam_indices=beam_indices, + ) + + # Release records + if "past_key_values" in self.__dir__(): + self.past_key_values = [None] * self.cfg.hidden_layers + if "encoder_states" in self.__dir__(): + self.encoder_states = None + + return sequence_outputs["sequences"] + + @flow.no_grad() + def generate( + self, + inputs: Optional[flow.Tensor] = None, + max_length: Optional[int] = None, + min_length: Optional[int] = None, + do_sample: Optional[bool] = None, + early_stopping: Optional[bool] = None, + num_beams: Optional[int] = None, + temperature: Optional[float] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + typical_p: Optional[float] = None, + repetition_penalty: Optional[float] = None, + force_words_ids: Optional[Union[Iterable[int], Iterable[Iterable[int]]]] = None, + bos_token_id: Optional[int] = None, + pad_token_id: Optional[int] = None, + eos_token_id: Optional[int] = None, + length_penalty: Optional[float] = None, + no_repeat_ngram_size: Optional[int] = None, + encoder_no_repeat_ngram_size: Optional[int] = None, + num_return_sequences: Optional[int] = None, + max_time: Optional[float] = None, + max_new_tokens: Optional[int] = None, + decoder_start_token_id: Optional[int] = None, + use_cache: Optional[bool] = None, + num_beam_groups: Optional[int] = None, + diversity_penalty: Optional[float] = None, + prefix_allowed_tokens_fn: Optional[Callable[[int, flow.Tensor], List[int]]] = None, + logits_processor: Optional[LogitsProcessorList] = LogitsProcessorList(), + renormalize_logits: Optional[bool] = None, + stopping_criteria=StoppingCriteriaList(), + constraints=None, + output_scores: Optional[bool] = None, + forced_bos_token_id: Optional[int] = None, + forced_eos_token_id: Optional[int] = None, + remove_invalid_values: Optional[bool] = None, + exponential_decay_length_penalty: Optional[Tuple[Union[int, float]]] = None, + **model_kwargs, + ): + # 0. Validate model kwargs + self._validate_model_kwargs(model_kwargs.copy()) + + # 1. Set generation parameters if not already defined + bos_token_id = bos_token_id if bos_token_id is not None else self.cfg.bos_token_id + num_beams = num_beams if num_beams is not None else self.cfg.num_beams + length_penalty = length_penalty if length_penalty is not None else self.cfg.length_penalty + early_stopping = early_stopping if early_stopping is not None else self.cfg.early_stopping + num_beam_groups = ( + num_beam_groups if num_beam_groups is not None else self.cfg.num_beam_groups + ) + do_sample = do_sample if do_sample is not None else self.cfg.do_sample + num_return_sequences = ( + num_return_sequences + if num_return_sequences is not None + else self.cfg.num_return_sequences + ) + + pad_token_id = pad_token_id if pad_token_id is not None else self.cfg.pad_token_id + eos_token_id = eos_token_id if eos_token_id is not None else self.cfg.eos_token_id + + output_scores = output_scores if output_scores is not None else self.cfg.output_scores + + # 2. Prepare model inputs + inputs_tensor, model_input_name, model_kwargs = self._prepare_model_inputs( + inputs, bos_token_id, model_kwargs + ) + batch_size = inputs_tensor.shape[0] + + # 3. Prepare other model kwargs + model_kwargs["use_cache"] = use_cache if use_cache is not None else self.cfg.use_cache + + if self.cfg.is_encoder_decoder: + att_mask_name = "encoder_attn_mask" + accepts_attention_mask = att_mask_name in set( + inspect.signature(self.forward).parameters.keys() + ) + else: + att_mask_name = "attention_mask" + accepts_attention_mask = att_mask_name in set( + inspect.signature(self.forward).parameters.keys() + ) + requires_attention_mask = "encoder_outputs" not in model_kwargs + + if ( + model_kwargs.get(att_mask_name, None) is None + and requires_attention_mask + and accepts_attention_mask + ): + model_kwargs[att_mask_name] = self._prepare_attention_mask_for_generation( + inputs_tensor, pad_token_id, eos_token_id + ) + if self.cfg.is_encoder_decoder and "encoder_outputs" not in model_kwargs: + # if model is encoder decoder encoder_outputs are created + # and added to `model_kwargs` + model_kwargs = self._prepare_encoder_decoder_kwargs_for_generation( + inputs_tensor, model_kwargs, model_input_name + ) + + # 4. Prepare `input_ids` which will be used for auto-regressive generation + if self.cfg.is_encoder_decoder: + input_ids = self._prepare_decoder_input_ids_for_generation( + batch_size, + decoder_start_token_id=decoder_start_token_id, + bos_token_id=bos_token_id, + model_kwargs=model_kwargs, + ) + else: + # if decoder-only then inputs_tensor has to be `input_ids` + input_ids = inputs_tensor + + # 5. Prepare `max_length` depending on other stopping criteria. + input_ids_seq_length = input_ids.shape[-1] + if max_length is None and max_new_tokens is None: + if dist.is_main_process(): + warnings.warn( + "Neither `max_length` nor `max_new_tokens` has been set, `max_length` will " + f"default to {self.cfg.max_length} (`self.cfg.max_length`). we recommend using" + " `max_new_tokens` to control the maximum length of the generation.", + UserWarning, + ) + elif max_length is None and max_new_tokens is not None: + max_length = max_new_tokens + input_ids_seq_length + elif max_length is not None and max_new_tokens is not None: + raise ValueError( + "Both `max_new_tokens` and `max_length` have been set but they serve the same" + ) + + # default to cfg if still None + max_length = max_length if max_length is not None else self.cfg.max_length + min_length = min_length if min_length is not None else self.cfg.min_length + + if min_length is not None and min_length > max_length: + raise ValueError( + f"Unfeasable length constraints: the minimum length ({min_length}) is larger than" + f"the maximum length ({max_length})" + ) + if input_ids_seq_length >= max_length: + input_ids_string = "decoder_input_ids" if self.cfg.is_encoder_decoder else "input_ids" + logger.warning( + f"Input length of {input_ids_string} is {input_ids_seq_length}, but `max_length` is" + f" set to {max_length}. This can lead to unexpected behavior. You should consider " + "increasing `max_new_tokens`." + ) + + # 6. Determine generation mode + is_constraint_gen_mode = constraints is not None or force_words_ids is not None + is_greedy_gen_mode = ( + (num_beams == 1) + and (num_beam_groups == 1) + and do_sample is False + and not is_constraint_gen_mode + ) + is_sample_gen_mode = ( + (num_beams == 1) + and (num_beam_groups == 1) + and do_sample is True + and not is_constraint_gen_mode + ) + is_beam_gen_mode = ( + (num_beams > 1) + and (num_beam_groups == 1) + and do_sample is False + and not is_constraint_gen_mode + ) + # is_beam_sample_gen_mode = ( + # (num_beams > 1) + # and (num_beam_groups == 1) + # and do_sample is True + # and not is_constraint_gen_mode + # ) + is_group_beam_gen_mode = ( + (num_beams > 1) and (num_beam_groups > 1) and not is_constraint_gen_mode + ) + + if num_beam_groups > num_beams: + raise ValueError("`num_beam_groups` has to be smaller or equal to `num_beams`") + if is_group_beam_gen_mode and do_sample is True: + raise ValueError( + "Diverse beam search cannot be used in sampling mode. Make sure that `do_sample` is" + " set to `False`." + ) + + # 7. Prepare distribution pre_processing samplers + logits_processor = self._get_logits_processor( + repetition_penalty=repetition_penalty, + no_repeat_ngram_size=no_repeat_ngram_size, + encoder_no_repeat_ngram_size=encoder_no_repeat_ngram_size, + input_ids_seq_length=input_ids_seq_length, + encoder_input_ids=inputs_tensor, + min_length=min_length, + max_length=max_length, + eos_token_id=eos_token_id, + forced_bos_token_id=forced_bos_token_id, + forced_eos_token_id=forced_eos_token_id, + prefix_allowed_tokens_fn=prefix_allowed_tokens_fn, + num_beams=num_beams, + num_beam_groups=num_beam_groups, + diversity_penalty=diversity_penalty, + remove_invalid_values=remove_invalid_values, + exponential_decay_length_penalty=exponential_decay_length_penalty, + logits_processor=logits_processor, + renormalize_logits=renormalize_logits, + ) + + # 8. Prepare stopping criteria + stopping_criteria = self._get_stopping_criteria( + max_length=max_length, max_time=max_time, stopping_criteria=stopping_criteria + ) + + # 9. Go into different generation modes + if is_greedy_gen_mode: + if num_return_sequences > 1: + raise ValueError( + f"num_return_sequences has to be 1, but is {num_return_sequences} when doing" + " greedy search." + ) + + # 10. Run greedy search + return self.greedy_search( + input_ids, + logits_processor=logits_processor, + stopping_criteria=stopping_criteria, + pad_token_id=pad_token_id, + eos_token_id=eos_token_id, + output_scores=output_scores, + **model_kwargs, + ) + + elif is_sample_gen_mode: + # 10. Prepare logits warper + logits_warper = self._get_logits_warper( + top_k=top_k, + top_p=top_p, + typical_p=typical_p, + temperature=temperature, + num_beams=num_beams, + renormalize_logits=renormalize_logits, + ) + + # 11. Expand input_ids with `num_return_sequences` additional sequences per batch + input_ids, model_kwargs = self._expand_inputs_for_generation( + input_ids, + expand_size=num_return_sequences, + is_encoder_decoder=self.cfg.is_encoder_decoder, + **model_kwargs, + ) + + # 12. Run multinomial sample + return self.multinomial_sample( + input_ids, + logits_processor=logits_processor, + logits_warper=logits_warper, + stopping_criteria=stopping_criteria, + pad_token_id=pad_token_id, + eos_token_id=eos_token_id, + output_scores=output_scores, + **model_kwargs, + ) + + elif is_beam_gen_mode: + if num_return_sequences > num_beams: + raise ValueError( + "`num_return_sequences` has to be smaller or equal to `num_beams`." + ) + + if stopping_criteria.max_length is None: + raise ValueError("`max_length` needs to be a stopping_criteria for now.") + + # 10. Prepare beam search scorer + beam_scorer = BeamSearchScorer( + batch_size=batch_size, + num_beams=num_beams, + length_penalty=length_penalty, + do_early_stopping=early_stopping, + num_beam_hyps_to_keep=num_return_sequences, + ) + + # 11. Interleave input_ids with `num_beams` additional sequences per batch + input_ids, model_kwargs = self._expand_inputs_for_generation( + input_ids, + expand_size=num_beams, + is_encoder_decoder=self.cfg.is_encoder_decoder, + **model_kwargs, + ) + + # 12. Run beam search + return self.beam_search( + input_ids, + beam_scorer, + logits_processor=logits_processor, + stopping_criteria=stopping_criteria, + pad_token_id=pad_token_id, + eos_token_id=eos_token_id, + output_scores=output_scores, + **model_kwargs, + ) diff --git a/libai/inference/image_classification.py b/libai/inference/image_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..50eaa8d4d2c0c553e6b0b1fc3746cd65cc5bf4c8 --- /dev/null +++ b/libai/inference/image_classification.py @@ -0,0 +1,153 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import oneflow as flow +from PIL import Image + +from libai.config import instantiate +from libai.data.structures import DistTensorData, Instance +from libai.inference.basic import BasePipeline + + +class ImageClassificationPipeline(BasePipeline): + def __init__( + self, + config_file, + data_parallel=None, + tensor_parallel=None, + pipeline_parallel=None, + pipeline_stage_id=None, + pipeline_num_layers=None, + model_path=None, + mode="libai", + **kwargs, + ): + super().__init__( + config_file, + data_parallel, + tensor_parallel, + pipeline_parallel, + pipeline_stage_id, + pipeline_num_layers, + model_path, + mode, + **kwargs, + ) + if "num_classes" in self.cfg.model: + self.num_classes = self.cfg.model.num_classes + elif "num_classes" in self.cfg.model.cfg: + self.num_classes = self.cfg.model.cfg.num_classes + else: + raise AttributeError("The model's config must contain num_classes") + label2id = self.label2id(self.num_classes) + self.id2label = {ind: label for label, ind in label2id.items()} + self.transform = instantiate(self.cfg.dataloader.test[0].dataset.transform) + + def _parse_parameters(self, **pipeline_parameters): + preprocess_params = {} + forward_params = {} + postprocess_params = {**pipeline_parameters} + return preprocess_params, forward_params, postprocess_params + + def preprocess( + self, + inputs, + **kwargs, + ) -> dict: + assert os.path.exists(inputs), "inputs must be an existing image path!" + with open(inputs, "rb") as f: + img = Image.open(f).convert("RGB") + img = self.transform(img) + img = img.unsqueeze(0) + + # to global tensor + model_input = Instance( + images=DistTensorData(img), + ) + mdoel_input_dict = {} + for key, value in model_input.get_fields().items(): + value.to_global() + mdoel_input_dict[key] = value.tensor + return mdoel_input_dict + + def forward(self, mdoel_input_dict) -> dict: + model_outputs_dict = self.model(**mdoel_input_dict) + return model_outputs_dict + + def postprocess( + self, model_outputs_dict, function_to_apply=None, return_all_scores=False, **kwargs + ) -> dict: + # prepare + num_labels = self.num_classes + if function_to_apply is not None: + function_to_apply = function_to_apply.lower() + assert function_to_apply in [ + "sigmoid", + "softmax", + "none", + ], f"Unrecognized `function_to_apply` argument: {function_to_apply}" + else: + if num_labels == 1: + function_to_apply = "sigmoid" + elif num_labels > 1: + function_to_apply = "softmax" + + # process, logits: [num_labels] + logits = model_outputs_dict["prediction_scores"][0] + + if function_to_apply == "sigmoid": + scores = flow.sigmoid(logits) + elif function_to_apply == "softmax": + + scores = flow.softmax(logits) + else: + scores = logits + scores = scores.detach().numpy() + + if return_all_scores: + return [ + {"label": self.id2label[i], "score": score.item()} for i, score in enumerate(scores) + ] + else: + return { + "label": self.id2label[scores.argmax().item()], + "score": scores.max().item(), + } + + def label2id(self, num_classes): + """ + Args: + num_classes (int): the number of total classes + Returns: + labels (list): a dict contains all the labels for inference, + each item should be the form as follows: + { + "tench": 0, + "tiger": 1, + "xxx", n, + } + + """ + from libai.inference.utils.imagenet_class import IMAGENET_LABELS as labels + + assert num_classes == len(labels), "number of labels must be equal to num_classes" + return {label: i for (i, label) in enumerate(labels)} + + +if __name__ == "__main__": + pipeline = ImageClassificationPipeline("/home/chengpeng/config.yaml", 1, 1, 1) + print(pipeline("data_test/inference_test_data/ILSVRC2012_val_00000293.JPEG")) diff --git a/libai/inference/text_classification.py b/libai/inference/text_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..dced0763d24f3a10390742ca946cd99eac98f703 --- /dev/null +++ b/libai/inference/text_classification.py @@ -0,0 +1,144 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import oneflow as flow + +from libai.data.structures import DistTensorData, Instance +from libai.inference.basic import BasePipeline + + +class TextClassificationPipeline(BasePipeline): + def __init__( + self, + config_file, + data_parallel=None, + tensor_parallel=None, + pipeline_parallel=None, + pipeline_stage_id=None, + pipeline_num_layers=None, + model_path=None, + mode="libai", + **kwargs, + ): + super().__init__( + config_file, + data_parallel, + tensor_parallel, + pipeline_parallel, + pipeline_stage_id, + model_path, + pipeline_num_layers, + mode, + **kwargs, + ) + + def update_cfg( + self, + data_parallel=1, + tensor_parallel=1, + pipeline_parallel=1, + pipeline_stage_id=None, + pipeline_num_layers=None, + ): + super().update_cfg( + data_parallel, + tensor_parallel, + pipeline_parallel, + pipeline_stage_id, + pipeline_num_layers, + ) + self.cfg.model.cfg.hidden_dropout_prob = 0.0 + self.cfg.model.cfg.attention_probs_dropout_prob = 0.0 + + assert "num_labels" in self.cfg.model.cfg, "The model's config must contain num_labels" + if "label2id" not in self.cfg.model.cfg: + label2id = {"Label_" + str(i): i for i in range(self.cfg.model.cfg.num_labels)} + id2label = {ind: label for label, ind in label2id.items()} + self.cfg.model.cfg["label2id"] = label2id + self.cfg.model.cfg["id2label"] = id2label + + def _parse_parameters(self, **pipeline_parameters): + preprocess_params = {} + forward_params = {} + postprocess_params = {**pipeline_parameters} + return preprocess_params, forward_params, postprocess_params + + def preprocess( + self, + inputs, + pad: bool = False, + **kwargs, + ) -> dict: + # tokenizer encoder + input_ids = flow.tensor(np.array(self.tokenizer.encode(inputs))) + padding_mask = flow.tensor(np.ones(input_ids.shape), dtype=flow.bool) + # set batch size = 1 + input_ids = input_ids.unsqueeze(0) + padding_mask = padding_mask.unsqueeze(0) + + # to global tensor + model_input = Instance( + input_ids=DistTensorData(input_ids), + attention_mask=DistTensorData(padding_mask), + ) + mdoel_input_dict = {} + for key, value in model_input.get_fields().items(): + value.to_global() + mdoel_input_dict[key] = value.tensor + return mdoel_input_dict + + def forward(self, mdoel_input_dict) -> dict: + model_outputs_dict = self.model(**mdoel_input_dict) + return model_outputs_dict + + def postprocess( + self, model_outputs_dict, function_to_apply=None, return_all_scores=False, **kwargs + ) -> dict: + # prepare + num_labels = self.cfg.model.cfg.num_labels + if function_to_apply is not None: + function_to_apply = function_to_apply.lower() + assert function_to_apply in [ + "sigmoid", + "softmax", + "none", + ], f"Unrecognized `function_to_apply` argument: {function_to_apply}" + else: + if num_labels == 1: + function_to_apply = "sigmoid" + elif num_labels > 1: + function_to_apply = "softmax" + + # process, logits: [num_labels] + logits = model_outputs_dict["logits"][0] + + if function_to_apply == "sigmoid": + scores = flow.sigmoid(logits) + elif function_to_apply == "softmax": + scores = flow.softmax(logits) + else: + scores = logits + scores = scores.detach().numpy() + if return_all_scores: + return [ + {"label": self.cfg.model.cfg.id2label[i], "score": score.item()} + for i, score in enumerate(scores) + ] + else: + return { + "label": self.cfg.model.cfg.id2label[scores.argmax().item()], + "score": scores.max().item(), + } diff --git a/libai/inference/text_generation.py b/libai/inference/text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..d789f28a179fa02a028247461c387639cde510e9 --- /dev/null +++ b/libai/inference/text_generation.py @@ -0,0 +1,108 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libai.inference.basic import BasePipeline +from libai.utils import distributed as dist + + +class TextGenerationPipeline(BasePipeline): + def load_pretrain_weight(self, libai_cfg_model, model_path, mode="huggingface"): + """load pretrained model. + + Args: + libai_cfg_model (libai.models): Lazy config Model in Libai, you can import it + by `from libai.config.configs.common.models.bert + import pretrain_model as libai_cfg_model` + model_path (str): The directory path of pretrained model, + """ + if mode == "huggingface": + from projects.MT5.utils.mt5_loader import T5LoaderHuggerFace + + model_loader = T5LoaderHuggerFace( + libai_cfg_model, + libai_cfg_model.cfg, + model_path, + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + embedding_dropout_prob=0.0, + ) + return model_loader.load() + elif mode == "libai": + from projects.MT5.utils.mt5_loader import T5LoaderLibai + + model_loader = T5LoaderLibai( + libai_cfg_model, + libai_cfg_model.cfg, + model_path, + ) + return model_loader.load() + elif mode == "random": + from libai.engine import DefaultTrainer + + return DefaultTrainer.build_model(self.cfg) + else: + raise NotImplementedError + + def _parse_parameters(self, **pipeline_parameters): + preprocess_params = {} + forward_params = {**pipeline_parameters} + postprocess_params = {} + + return preprocess_params, forward_params, postprocess_params + + def preprocess( + self, + inputs, + pad: bool = False, + **kwargs, + ) -> dict: + # tokenizer encoder + encoder_ids = self.tokenizer.encode(inputs, return_tensors="of", is_global=True) + + encoder_input_dict = { + "encoder_ids": encoder_ids, + } + + return encoder_input_dict + + def forward(self, encoder_input_dict, **kwargs) -> dict: + outputs = self.model.generate(encoder_input_dict["encoder_ids"], **kwargs) + return {"return_ids": outputs} + + def postprocess(self, model_output_dict, **kwargs) -> dict: + return_ids = model_output_dict["return_ids"] + records = [ + {"generated_text": self.tokenizer.decode(return_ids[i], skip_special_tokens=True)} + for i in range(return_ids.size(0)) + ] + return records + + +if __name__ == "__main__": + pipeline = TextGenerationPipeline( + "/path/to/libai/projects/MT5/configs/t5_inference.py", + data_parallel=1, + tensor_parallel=2, + pipeline_parallel=2, + pipeline_stage_id=[0] * 12 + [1] * 12, + pipeline_num_layers=12 * 2, + model_path="/path/to/t5-base", + mode="huggingface", + ) + + text = ["summarize: She is a student, She is tall, She loves study"] + dict1 = pipeline(text) + if dist.is_main_process(): + print(dict1) diff --git a/libai/inference/utils/imagenet_class.py b/libai/inference/utils/imagenet_class.py new file mode 100644 index 0000000000000000000000000000000000000000..1782caaa669dec0451feefcf4bcb52ca133d6375 --- /dev/null +++ b/libai/inference/utils/imagenet_class.py @@ -0,0 +1,1002 @@ +IMAGENET_LABELS = [ + "tench, Tinca tinca", + "goldfish, Carassius auratus", + "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", # noqa: E501 + "tiger shark, Galeocerdo cuvieri", + "hammerhead, hammerhead shark", + "electric ray, crampfish, numbfish, torpedo", + "stingray", + "cock", + "hen", + "ostrich, Struthio camelus", + "brambling, Fringilla montifringilla", + "goldfinch, Carduelis carduelis", + "house finch, linnet, Carpodacus mexicanus", + "junco, snowbird", + "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "robin, American robin, Turdus migratorius", + "bulbul", + "jay", + "magpie", + "chickadee", + "water ouzel, dipper", + "kite", + "bald eagle, American eagle, Haliaeetus leucocephalus", + "vulture", + "great grey owl, great gray owl, Strix nebulosa", + "European fire salamander, Salamandra salamandra", + "common newt, Triturus vulgaris", + "eft", + "spotted salamander, Ambystoma maculatum", + "axolotl, mud puppy, Ambystoma mexicanum", + "bullfrog, Rana catesbeiana", + "tree frog, tree-frog", + "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "loggerhead, loggerhead turtle, Caretta caretta", + "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", # noqa: E501 + "mud turtle", + "terrapin", + "box turtle, box tortoise", + "banded gecko", + "common iguana, iguana, Iguana iguana", + "American chameleon, anole, Anolis carolinensis", + "whiptail, whiptail lizard", + "agama", + "frilled lizard, Chlamydosaurus kingi", + "alligator lizard", + "Gila monster, Heloderma suspectum", + "green lizard, Lacerta viridis", + "African chameleon, Chamaeleo chamaeleon", + "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", # noqa: E501 + "African crocodile, Nile crocodile, Crocodylus niloticus", + "American alligator, Alligator mississipiensis", + "triceratops", + "thunder snake, worm snake, Carphophis amoenus", + "ringneck snake, ring-necked snake, ring snake", + "hognose snake, puff adder, sand viper", + "green snake, grass snake", + "king snake, kingsnake", + "garter snake, grass snake", + "water snake", + "vine snake", + "night snake, Hypsiglena torquata", + "boa constrictor, Constrictor constrictor", + "rock python, rock snake, Python sebae", + "Indian cobra, Naja naja", + "green mamba", + "sea snake", + "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "sidewinder, horned rattlesnake, Crotalus cerastes", + "trilobite", + "harvestman, daddy longlegs, Phalangium opilio", + "scorpion", + "black and gold garden spider, Argiope aurantia", + "barn spider, Araneus cavaticus", + "garden spider, Aranea diademata", + "black widow, Latrodectus mactans", + "tarantula", + "wolf spider, hunting spider", + "tick", + "centipede", + "black grouse", + "ptarmigan", + "ruffed grouse, partridge, Bonasa umbellus", + "prairie chicken, prairie grouse, prairie fowl", + "peacock", + "quail", + "partridge", + "African grey, African gray, Psittacus erithacus", + "macaw", + "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "lorikeet", + "coucal", + "bee eater", + "hornbill", + "hummingbird", + "jacamar", + "toucan", + "drake", + "red-breasted merganser, Mergus serrator", + "goose", + "black swan, Cygnus atratus", + "tusker", + "echidna, spiny anteater, anteater", + "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", # noqa: E501 + "wallaby, brush kangaroo", + "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", # noqa: E501 + "wombat", + "jellyfish", + "sea anemone, anemone", + "brain coral", + "flatworm, platyhelminth", + "nematode, nematode worm, roundworm", + "conch", + "snail", + "slug", + "sea slug, nudibranch", + "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "chambered nautilus, pearly nautilus, nautilus", + "Dungeness crab, Cancer magister", + "rock crab, Cancer irroratus", + "fiddler crab", + "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", # noqa: E501 + "American lobster, Northern lobster, Maine lobster, Homarus americanus", # noqa: E501 + "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", # noqa: E501 + "crayfish, crawfish, crawdad, crawdaddy", + "hermit crab", + "isopod", + "white stork, Ciconia ciconia", + "black stork, Ciconia nigra", + "spoonbill", + "flamingo", + "little blue heron, Egretta caerulea", + "American egret, great white heron, Egretta albus", + "bittern", + "crane", + "limpkin, Aramus pictus", + "European gallinule, Porphyrio porphyrio", + "American coot, marsh hen, mud hen, water hen, Fulica americana", + "bustard", + "ruddy turnstone, Arenaria interpres", + "red-backed sandpiper, dunlin, Erolia alpina", + "redshank, Tringa totanus", + "dowitcher", + "oystercatcher, oyster catcher", + "pelican", + "king penguin, Aptenodytes patagonica", + "albatross, mollymawk", + "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", # noqa: E501 + "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "dugong, Dugong dugon", + "sea lion", + "Chihuahua", + "Japanese spaniel", + "Maltese dog, Maltese terrier, Maltese", + "Pekinese, Pekingese, Peke", + "Shih-Tzu", + "Blenheim spaniel", + "papillon", + "toy terrier", + "Rhodesian ridgeback", + "Afghan hound, Afghan", + "basset, basset hound", + "beagle", + "bloodhound, sleuthhound", + "bluetick", + "black-and-tan coonhound", + "Walker hound, Walker foxhound", + "English foxhound", + "redbone", + "borzoi, Russian wolfhound", + "Irish wolfhound", + "Italian greyhound", + "whippet", + "Ibizan hound, Ibizan Podenco", + "Norwegian elkhound, elkhound", + "otterhound, otter hound", + "Saluki, gazelle hound", + "Scottish deerhound, deerhound", + "Weimaraner", + "Staffordshire bullterrier, Staffordshire bull terrier", + "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", # noqa: E501 + "Bedlington terrier", + "Border terrier", + "Kerry blue terrier", + "Irish terrier", + "Norfolk terrier", + "Norwich terrier", + "Yorkshire terrier", + "wire-haired fox terrier", + "Lakeland terrier", + "Sealyham terrier, Sealyham", + "Airedale, Airedale terrier", + "cairn, cairn terrier", + "Australian terrier", + "Dandie Dinmont, Dandie Dinmont terrier", + "Boston bull, Boston terrier", + "miniature schnauzer", + "giant schnauzer", + "standard schnauzer", + "Scotch terrier, Scottish terrier, Scottie", + "Tibetan terrier, chrysanthemum dog", + "silky terrier, Sydney silky", + "soft-coated wheaten terrier", + "West Highland white terrier", + "Lhasa, Lhasa apso", + "flat-coated retriever", + "curly-coated retriever", + "golden retriever", + "Labrador retriever", + "Chesapeake Bay retriever", + "German short-haired pointer", + "vizsla, Hungarian pointer", + "English setter", + "Irish setter, red setter", + "Gordon setter", + "Brittany spaniel", + "clumber, clumber spaniel", + "English springer, English springer spaniel", + "Welsh springer spaniel", + "cocker spaniel, English cocker spaniel, cocker", + "Sussex spaniel", + "Irish water spaniel", + "kuvasz", + "schipperke", + "groenendael", + "malinois", + "briard", + "kelpie", + "komondor", + "Old English sheepdog, bobtail", + "Shetland sheepdog, Shetland sheep dog, Shetland", + "collie", + "Border collie", + "Bouvier des Flandres, Bouviers des Flandres", + "Rottweiler", + "German shepherd, German shepherd dog, German police dog, alsatian", + "Doberman, Doberman pinscher", + "miniature pinscher", + "Greater Swiss Mountain dog", + "Bernese mountain dog", + "Appenzeller", + "EntleBucher", + "boxer", + "bull mastiff", + "Tibetan mastiff", + "French bulldog", + "Great Dane", + "Saint Bernard, St Bernard", + "Eskimo dog, husky", + "malamute, malemute, Alaskan malamute", + "Siberian husky", + "dalmatian, coach dog, carriage dog", + "affenpinscher, monkey pinscher, monkey dog", + "basenji", + "pug, pug-dog", + "Leonberg", + "Newfoundland, Newfoundland dog", + "Great Pyrenees", + "Samoyed, Samoyede", + "Pomeranian", + "chow, chow chow", + "keeshond", + "Brabancon griffon", + "Pembroke, Pembroke Welsh corgi", + "Cardigan, Cardigan Welsh corgi", + "toy poodle", + "miniature poodle", + "standard poodle", + "Mexican hairless", + "timber wolf, grey wolf, gray wolf, Canis lupus", + "white wolf, Arctic wolf, Canis lupus tundrarum", + "red wolf, maned wolf, Canis rufus, Canis niger", + "coyote, prairie wolf, brush wolf, Canis latrans", + "dingo, warrigal, warragal, Canis dingo", + "dhole, Cuon alpinus", + "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "hyena, hyaena", + "red fox, Vulpes vulpes", + "kit fox, Vulpes macrotis", + "Arctic fox, white fox, Alopex lagopus", + "grey fox, gray fox, Urocyon cinereoargenteus", + "tabby, tabby cat", + "tiger cat", + "Persian cat", + "Siamese cat, Siamese", + "Egyptian cat", + "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", # noqa: E501 + "lynx, catamount", + "leopard, Panthera pardus", + "snow leopard, ounce, Panthera uncia", + "jaguar, panther, Panthera onca, Felis onca", + "lion, king of beasts, Panthera leo", + "tiger, Panthera tigris", + "cheetah, chetah, Acinonyx jubatus", + "brown bear, bruin, Ursus arctos", + "American black bear, black bear, Ursus americanus, Euarctos americanus", # noqa: E501 + "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "sloth bear, Melursus ursinus, Ursus ursinus", + "mongoose", + "meerkat, mierkat", + "tiger beetle", + "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "ground beetle, carabid beetle", + "long-horned beetle, longicorn, longicorn beetle", + "leaf beetle, chrysomelid", + "dung beetle", + "rhinoceros beetle", + "weevil", + "fly", + "bee", + "ant, emmet, pismire", + "grasshopper, hopper", + "cricket", + "walking stick, walkingstick, stick insect", + "cockroach, roach", + "mantis, mantid", + "cicada, cicala", + "leafhopper", + "lacewing, lacewing fly", + "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", # noqa: E501 + "damselfly", + "admiral", + "ringlet, ringlet butterfly", + "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "cabbage butterfly", + "sulphur butterfly, sulfur butterfly", + "lycaenid, lycaenid butterfly", + "starfish, sea star", + "sea urchin", + "sea cucumber, holothurian", + "wood rabbit, cottontail, cottontail rabbit", + "hare", + "Angora, Angora rabbit", + "hamster", + "porcupine, hedgehog", + "fox squirrel, eastern fox squirrel, Sciurus niger", + "marmot", + "beaver", + "guinea pig, Cavia cobaya", + "sorrel", + "zebra", + "hog, pig, grunter, squealer, Sus scrofa", + "wild boar, boar, Sus scrofa", + "warthog", + "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "ox", + "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "bison", + "ram, tup", + "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", # noqa: E501 + "ibex, Capra ibex", + "hartebeest", + "impala, Aepyceros melampus", + "gazelle", + "Arabian camel, dromedary, Camelus dromedarius", + "llama", + "weasel", + "mink", + "polecat, fitch, foulmart, foumart, Mustela putorius", + "black-footed ferret, ferret, Mustela nigripes", + "otter", + "skunk, polecat, wood pussy", + "badger", + "armadillo", + "three-toed sloth, ai, Bradypus tridactylus", + "orangutan, orang, orangutang, Pongo pygmaeus", + "gorilla, Gorilla gorilla", + "chimpanzee, chimp, Pan troglodytes", + "gibbon, Hylobates lar", + "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "guenon, guenon monkey", + "patas, hussar monkey, Erythrocebus patas", + "baboon", + "macaque", + "langur", + "colobus, colobus monkey", + "proboscis monkey, Nasalis larvatus", + "marmoset", + "capuchin, ringtail, Cebus capucinus", + "howler monkey, howler", + "titi, titi monkey", + "spider monkey, Ateles geoffroyi", + "squirrel monkey, Saimiri sciureus", + "Madagascar cat, ring-tailed lemur, Lemur catta", + "indri, indris, Indri indri, Indri brevicaudatus", + "Indian elephant, Elephas maximus", + "African elephant, Loxodonta africana", + "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "barracouta, snoek", + "eel", + "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", # noqa: E501 + "rock beauty, Holocanthus tricolor", + "anemone fish", + "sturgeon", + "gar, garfish, garpike, billfish, Lepisosteus osseus", + "lionfish", + "puffer, pufferfish, blowfish, globefish", + "abacus", + "abaya", + "academic gown, academic robe, judge's robe", + "accordion, piano accordion, squeeze box", + "acoustic guitar", + "aircraft carrier, carrier, flattop, attack aircraft carrier", + "airliner", + "airship, dirigible", + "altar", + "ambulance", + "amphibian, amphibious vehicle", + "analog clock", + "apiary, bee house", + "apron", + "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", # noqa: E501 + "assault rifle, assault gun", + "backpack, back pack, knapsack, packsack, rucksack, haversack", + "bakery, bakeshop, bakehouse", + "balance beam, beam", + "balloon", + "ballpoint, ballpoint pen, ballpen, Biro", + "Band Aid", + "banjo", + "bannister, banister, balustrade, balusters, handrail", + "barbell", + "barber chair", + "barbershop", + "barn", + "barometer", + "barrel, cask", + "barrow, garden cart, lawn cart, wheelbarrow", + "baseball", + "basketball", + "bassinet", + "bassoon", + "bathing cap, swimming cap", + "bath towel", + "bathtub, bathing tub, bath, tub", + "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", # noqa: E501 + "beacon, lighthouse, beacon light, pharos", + "beaker", + "bearskin, busby, shako", + "beer bottle", + "beer glass", + "bell cote, bell cot", + "bib", + "bicycle-built-for-two, tandem bicycle, tandem", + "bikini, two-piece", + "binder, ring-binder", + "binoculars, field glasses, opera glasses", + "birdhouse", + "boathouse", + "bobsled, bobsleigh, bob", + "bolo tie, bolo, bola tie, bola", + "bonnet, poke bonnet", + "bookcase", + "bookshop, bookstore, bookstall", + "bottlecap", + "bow", + "bow tie, bow-tie, bowtie", + "brass, memorial tablet, plaque", + "brassiere, bra, bandeau", + "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "breastplate, aegis, egis", + "broom", + "bucket, pail", + "buckle", + "bulletproof vest", + "bullet train, bullet", + "butcher shop, meat market", + "cab, hack, taxi, taxicab", + "caldron, cauldron", + "candle, taper, wax light", + "cannon", + "canoe", + "can opener, tin opener", + "cardigan", + "car mirror", + "carousel, carrousel, merry-go-round, roundabout, whirligig", + "carpenter's kit, tool kit", + "carton", + "car wheel", + "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", # noqa: E501 + "cassette", + "cassette player", + "castle", + "catamaran", + "CD player", + "cello, violoncello", + "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "chain", + "chainlink fence", + "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", # noqa: E501 + "chain saw, chainsaw", + "chest", + "chiffonier, commode", + "chime, bell, gong", + "china cabinet, china closet", + "Christmas stocking", + "church, church building", + "cinema, movie theater, movie theatre, movie house, picture palace", + "cleaver, meat cleaver, chopper", + "cliff dwelling", + "cloak", + "clog, geta, patten, sabot", + "cocktail shaker", + "coffee mug", + "coffeepot", + "coil, spiral, volute, whorl, helix", + "combination lock", + "computer keyboard, keypad", + "confectionery, confectionary, candy store", + "container ship, containership, container vessel", + "convertible", + "corkscrew, bottle screw", + "cornet, horn, trumpet, trump", + "cowboy boot", + "cowboy hat, ten-gallon hat", + "cradle", + "crane", + "crash helmet", + "crate", + "crib, cot", + "Crock Pot", + "croquet ball", + "crutch", + "cuirass", + "dam, dike, dyke", + "desk", + "desktop computer", + "dial telephone, dial phone", + "diaper, nappy, napkin", + "digital clock", + "digital watch", + "dining table, board", + "dishrag, dishcloth", + "dishwasher, dish washer, dishwashing machine", + "disk brake, disc brake", + "dock, dockage, docking facility", + "dogsled, dog sled, dog sleigh", + "dome", + "doormat, welcome mat", + "drilling platform, offshore rig", + "drum, membranophone, tympan", + "drumstick", + "dumbbell", + "Dutch oven", + "electric fan, blower", + "electric guitar", + "electric locomotive", + "entertainment center", + "envelope", + "espresso maker", + "face powder", + "feather boa, boa", + "file, file cabinet, filing cabinet", + "fireboat", + "fire engine, fire truck", + "fire screen, fireguard", + "flagpole, flagstaff", + "flute, transverse flute", + "folding chair", + "football helmet", + "forklift", + "fountain", + "fountain pen", + "four-poster", + "freight car", + "French horn, horn", + "frying pan, frypan, skillet", + "fur coat", + "garbage truck, dustcart", + "gasmask, respirator, gas helmet", + "gas pump, gasoline pump, petrol pump, island dispenser", + "goblet", + "go-kart", + "golf ball", + "golfcart, golf cart", + "gondola", + "gong, tam-tam", + "gown", + "grand piano, grand", + "greenhouse, nursery, glasshouse", + "grille, radiator grille", + "grocery store, grocery, food market, market", + "guillotine", + "hair slide", + "hair spray", + "half track", + "hammer", + "hamper", + "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "hand-held computer, hand-held microcomputer", + "handkerchief, hankie, hanky, hankey", + "hard disc, hard disk, fixed disk", + "harmonica, mouth organ, harp, mouth harp", + "harp", + "harvester, reaper", + "hatchet", + "holster", + "home theater, home theatre", + "honeycomb", + "hook, claw", + "hoopskirt, crinoline", + "horizontal bar, high bar", + "horse cart, horse-cart", + "hourglass", + "iPod", + "iron, smoothing iron", + "jack-o'-lantern", + "jean, blue jean, denim", + "jeep, landrover", + "jersey, T-shirt, tee shirt", + "jigsaw puzzle", + "jinrikisha, ricksha, rickshaw", + "joystick", + "kimono", + "knee pad", + "knot", + "lab coat, laboratory coat", + "ladle", + "lampshade, lamp shade", + "laptop, laptop computer", + "lawn mower, mower", + "lens cap, lens cover", + "letter opener, paper knife, paperknife", + "library", + "lifeboat", + "lighter, light, igniter, ignitor", + "limousine, limo", + "liner, ocean liner", + "lipstick, lip rouge", + "Loafer", + "lotion", + "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", # noqa: E501 + "loupe, jeweler's loupe", + "lumbermill, sawmill", + "magnetic compass", + "mailbag, postbag", + "mailbox, letter box", + "maillot", + "maillot, tank suit", + "manhole cover", + "maraca", + "marimba, xylophone", + "mask", + "matchstick", + "maypole", + "maze, labyrinth", + "measuring cup", + "medicine chest, medicine cabinet", + "megalith, megalithic structure", + "microphone, mike", + "microwave, microwave oven", + "military uniform", + "milk can", + "minibus", + "miniskirt, mini", + "minivan", + "missile", + "mitten", + "mixing bowl", + "mobile home, manufactured home", + "Model T", + "modem", + "monastery", + "monitor", + "moped", + "mortar", + "mortarboard", + "mosque", + "mosquito net", + "motor scooter, scooter", + "mountain bike, all-terrain bike, off-roader", + "mountain tent", + "mouse, computer mouse", + "mousetrap", + "moving van", + "muzzle", + "nail", + "neck brace", + "necklace", + "nipple", + "notebook, notebook computer", + "obelisk", + "oboe, hautboy, hautbois", + "ocarina, sweet potato", + "odometer, hodometer, mileometer, milometer", + "oil filter", + "organ, pipe organ", + "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "overskirt", + "oxcart", + "oxygen mask", + "packet", + "paddle, boat paddle", + "paddlewheel, paddle wheel", + "padlock", + "paintbrush", + "pajama, pyjama, pj's, jammies", + "palace", + "panpipe, pandean pipe, syrinx", + "paper towel", + "parachute, chute", + "parallel bars, bars", + "park bench", + "parking meter", + "passenger car, coach, carriage", + "patio, terrace", + "pay-phone, pay-station", + "pedestal, plinth, footstall", + "pencil box, pencil case", + "pencil sharpener", + "perfume, essence", + "Petri dish", + "photocopier", + "pick, plectrum, plectron", + "pickelhaube", + "picket fence, paling", + "pickup, pickup truck", + "pier", + "piggy bank, penny bank", + "pill bottle", + "pillow", + "ping-pong ball", + "pinwheel", + "pirate, pirate ship", + "pitcher, ewer", + "plane, carpenter's plane, woodworking plane", + "planetarium", + "plastic bag", + "plate rack", + "plow, plough", + "plunger, plumber's helper", + "Polaroid camera, Polaroid Land camera", + "pole", + "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", # noqa: E501 + "poncho", + "pool table, billiard table, snooker table", + "pop bottle, soda bottle", + "pot, flowerpot", + "potter's wheel", + "power drill", + "prayer rug, prayer mat", + "printer", + "prison, prison house", + "projectile, missile", + "projector", + "puck, hockey puck", + "punching bag, punch bag, punching ball, punchball", + "purse", + "quill, quill pen", + "quilt, comforter, comfort, puff", + "racer, race car, racing car", + "racket, racquet", + "radiator", + "radio, wireless", + "radio telescope, radio reflector", + "rain barrel", + "recreational vehicle, RV, R.V.", + "reel", + "reflex camera", + "refrigerator, icebox", + "remote control, remote", + "restaurant, eating house, eating place, eatery", + "revolver, six-gun, six-shooter", + "rifle", + "rocking chair, rocker", + "rotisserie", + "rubber eraser, rubber, pencil eraser", + "rugby ball", + "rule, ruler", + "running shoe", + "safe", + "safety pin", + "saltshaker, salt shaker", + "sandal", + "sarong", + "sax, saxophone", + "scabbard", + "scale, weighing machine", + "school bus", + "schooner", + "scoreboard", + "screen, CRT screen", + "screw", + "screwdriver", + "seat belt, seatbelt", + "sewing machine", + "shield, buckler", + "shoe shop, shoe-shop, shoe store", + "shoji", + "shopping basket", + "shopping cart", + "shovel", + "shower cap", + "shower curtain", + "ski", + "ski mask", + "sleeping bag", + "slide rule, slipstick", + "sliding door", + "slot, one-armed bandit", + "snorkel", + "snowmobile", + "snowplow, snowplough", + "soap dispenser", + "soccer ball", + "sock", + "solar dish, solar collector, solar furnace", + "sombrero", + "soup bowl", + "space bar", + "space heater", + "space shuttle", + "spatula", + "speedboat", + "spider web, spider's web", + "spindle", + "sports car, sport car", + "spotlight, spot", + "stage", + "steam locomotive", + "steel arch bridge", + "steel drum", + "stethoscope", + "stole", + "stone wall", + "stopwatch, stop watch", + "stove", + "strainer", + "streetcar, tram, tramcar, trolley, trolley car", + "stretcher", + "studio couch, day bed", + "stupa, tope", + "submarine, pigboat, sub, U-boat", + "suit, suit of clothes", + "sundial", + "sunglass", + "sunglasses, dark glasses, shades", + "sunscreen, sunblock, sun blocker", + "suspension bridge", + "swab, swob, mop", + "sweatshirt", + "swimming trunks, bathing trunks", + "swing", + "switch, electric switch, electrical switch", + "syringe", + "table lamp", + "tank, army tank, armored combat vehicle, armoured combat vehicle", + "tape player", + "teapot", + "teddy, teddy bear", + "television, television system", + "tennis ball", + "thatch, thatched roof", + "theater curtain, theatre curtain", + "thimble", + "thresher, thrasher, threshing machine", + "throne", + "tile roof", + "toaster", + "tobacco shop, tobacconist shop, tobacconist", + "toilet seat", + "torch", + "totem pole", + "tow truck, tow car, wrecker", + "toyshop", + "tractor", + "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", # noqa: E501 + "tray", + "trench coat", + "tricycle, trike, velocipede", + "trimaran", + "tripod", + "triumphal arch", + "trolleybus, trolley coach, trackless trolley", + "trombone", + "tub, vat", + "turnstile", + "typewriter keyboard", + "umbrella", + "unicycle, monocycle", + "upright, upright piano", + "vacuum, vacuum cleaner", + "vase", + "vault", + "velvet", + "vending machine", + "vestment", + "viaduct", + "violin, fiddle", + "volleyball", + "waffle iron", + "wall clock", + "wallet, billfold, notecase, pocketbook", + "wardrobe, closet, press", + "warplane, military plane", + "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "washer, automatic washer, washing machine", + "water bottle", + "water jug", + "water tower", + "whiskey jug", + "whistle", + "wig", + "window screen", + "window shade", + "Windsor tie", + "wine bottle", + "wing", + "wok", + "wooden spoon", + "wool, woolen, woollen", + "worm fence, snake fence, snake-rail fence, Virginia fence", + "wreck", + "yawl", + "yurt", + "web site, website, internet site, site", + "comic book", + "crossword puzzle, crossword", + "street sign", + "traffic light, traffic signal, stoplight", + "book jacket, dust cover, dust jacket, dust wrapper", + "menu", + "plate", + "guacamole", + "consomme", + "hot pot, hotpot", + "trifle", + "ice cream, icecream", + "ice lolly, lolly, lollipop, popsicle", + "French loaf", + "bagel, beigel", + "pretzel", + "cheeseburger", + "hotdog, hot dog, red hot", + "mashed potato", + "head cabbage", + "broccoli", + "cauliflower", + "zucchini, courgette", + "spaghetti squash", + "acorn squash", + "butternut squash", + "cucumber, cuke", + "artichoke, globe artichoke", + "bell pepper", + "cardoon", + "mushroom", + "Granny Smith", + "strawberry", + "orange", + "lemon", + "fig", + "pineapple, ananas", + "banana", + "jackfruit, jak, jack", + "custard apple", + "pomegranate", + "hay", + "carbonara", + "chocolate sauce, chocolate syrup", + "dough", + "meat loaf, meatloaf", + "pizza, pizza pie", + "potpie", + "burrito", + "red wine", + "espresso", + "cup", + "eggnog", + "alp", + "bubble", + "cliff, drop, drop-off", + "coral reef", + "geyser", + "lakeside, lakeshore", + "promontory, headland, head, foreland", + "sandbar, sand bar", + "seashore, coast, seacoast, sea-coast", + "valley, vale", + "volcano", + "ballplayer, baseball player", + "groom, bridegroom", + "scuba diver", + "rapeseed", + "daisy", + "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", # noqa: E501 + "corn", + "acorn", + "hip, rose hip, rosehip", + "buckeye, horse chestnut, conker", + "coral fungus", + "agaric", + "gyromitra", + "stinkhorn, carrion fungus", + "earthstar", + "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", # noqa: E501 + "bolete", + "ear, spike, capitulum", + "toilet tissue, toilet paper, bathroom tissue", +] diff --git a/libai/layers/__init__.py b/libai/layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..81dd8326cb1b3242135745fd500033a5039857d3 --- /dev/null +++ b/libai/layers/__init__.py @@ -0,0 +1,44 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .activation import build_activation +from .cross_entropy import ParallelCrossEntropyLoss +from .embedding import Embedding, SinePositionalEmbedding, VocabEmbedding, PatchEmbedding +from .layer_norm import LayerNorm, RMSLayerNorm +from .linear import Linear, Linear1D +from .lm_logits import LMLogits +from .mlp import MLP +from .transformer_layer import TransformerLayer +from .attention import MultiheadAttention +from .droppath import DropPath, drop_path + +__all__ = [ + "Embedding", + "VocabEmbedding", + "SinePositionalEmbedding", + "PatchEmbedding", + "build_activation", + "Linear", + "Linear1D", + "MLP", + "LayerNorm", + "RMSLayerNorm", + "TransformerLayer", + "MultiheadAttention", + "ParallelCrossEntropyLoss", + "LMLogits", + "drop_path", + "DropPath", +] diff --git a/libai/layers/__pycache__/__init__.cpython-39.pyc b/libai/layers/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..773f7495cd21cba7d912ddf31093e43d706d2264 Binary files /dev/null and b/libai/layers/__pycache__/__init__.cpython-39.pyc differ diff --git a/libai/layers/__pycache__/activation.cpython-39.pyc b/libai/layers/__pycache__/activation.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..46b05764516beb515406463501e146c0b22b6373 Binary files /dev/null and b/libai/layers/__pycache__/activation.cpython-39.pyc differ diff --git a/libai/layers/__pycache__/attention.cpython-39.pyc b/libai/layers/__pycache__/attention.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..74a8e6df9a280da6f4bb99c82e095750b1331893 Binary files /dev/null and b/libai/layers/__pycache__/attention.cpython-39.pyc differ diff --git a/libai/layers/__pycache__/cross_entropy.cpython-39.pyc b/libai/layers/__pycache__/cross_entropy.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4d118f5e217952d1f341e35efef3a3660d07c7af Binary files /dev/null and b/libai/layers/__pycache__/cross_entropy.cpython-39.pyc differ diff --git a/libai/layers/__pycache__/droppath.cpython-39.pyc b/libai/layers/__pycache__/droppath.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..90dd397b0ed19a43d23fd6cc19cf1a1e42e2cda5 Binary files /dev/null and b/libai/layers/__pycache__/droppath.cpython-39.pyc differ diff --git a/libai/layers/__pycache__/embedding.cpython-39.pyc b/libai/layers/__pycache__/embedding.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..334df1501185dc3b9409d7077d9e0b67c7563566 Binary files /dev/null and b/libai/layers/__pycache__/embedding.cpython-39.pyc differ diff --git a/libai/layers/__pycache__/layer_norm.cpython-39.pyc b/libai/layers/__pycache__/layer_norm.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b80a479e3f78bed0d22863de9602c247b6ad0fad Binary files /dev/null and b/libai/layers/__pycache__/layer_norm.cpython-39.pyc differ diff --git a/libai/layers/__pycache__/linear.cpython-39.pyc b/libai/layers/__pycache__/linear.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..96df298fa66c92dc41a7e414b211510ec1ae0d1b Binary files /dev/null and b/libai/layers/__pycache__/linear.cpython-39.pyc differ diff --git a/libai/layers/__pycache__/lm_logits.cpython-39.pyc b/libai/layers/__pycache__/lm_logits.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e98563ba0c313c936c7925ede6967427b4542d28 Binary files /dev/null and b/libai/layers/__pycache__/lm_logits.cpython-39.pyc differ diff --git a/libai/layers/__pycache__/mlp.cpython-39.pyc b/libai/layers/__pycache__/mlp.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3d000faf2a0cfa78481d2f12efcf92505f6f46f3 Binary files /dev/null and b/libai/layers/__pycache__/mlp.cpython-39.pyc differ diff --git a/libai/layers/__pycache__/transformer_layer.cpython-39.pyc b/libai/layers/__pycache__/transformer_layer.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b351bdfb711ed7b729154d3a99cf1de86bad2ff Binary files /dev/null and b/libai/layers/__pycache__/transformer_layer.cpython-39.pyc differ diff --git a/libai/layers/activation.py b/libai/layers/activation.py new file mode 100644 index 0000000000000000000000000000000000000000..0aa6e7da8d7e2e4ecc610a3fccfbf6e037e10028 --- /dev/null +++ b/libai/layers/activation.py @@ -0,0 +1,86 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum +from typing import Optional + +import oneflow as flow +from oneflow import nn + + +class Activation(str, Enum): + SquaredReLU = "squared_relu" + GeLU = "gelu" + GeLUTanh = "gelu_tanh" + LeakyReLU = "leaky_relu" + ReLU = "relu" + Tanh = "tanh" + QuickGELU = "quick_gelu" + + +# For unit testing / parity comparisons, probably not the fastest way +class SquaredReLU(nn.Module): + def __init__(self) -> None: + super().__init__() + + def forward(self, x: flow.Tensor) -> flow.Tensor: + x_ = flow._C.relu(x) + return x_ * x_ + + +class Passthrough(nn.Module): + def __init__(self) -> None: + super().__init__() + + def forward(self, x: flow.Tensor) -> flow.Tensor: + return x + + +class GeLUTanh(nn.Module): + def __init__(self) -> None: + super().__init__() + + def forward(self, x: flow.Tensor) -> flow.Tensor: + """When the approximate argument is 'tanh', Gelu is estimated with: + 0.5 * x * (1.0 + flow.tanh(math.sqrt(2.0 / math.pi) * (x + 0.044715 * flow.pow(x, 3.0)))) + """ + return flow.nn.functional.gelu(x, approximate="tanh") + + +class QuickGELU(nn.Module): + def __init__(self) -> None: + super().__init__() + + def forward(self, x: flow.Tensor) -> flow.Tensor: + return x * flow.sigmoid(1.702 * x) + + +def build_activation(activation: Optional[Activation]): + """ + Fetching activation layers by name, e.g., + ``build_activation("gelu")`` returns ``nn.GELU()`` module. + """ + if not activation: + return Passthrough() + + return { + Activation.ReLU: nn.ReLU, + Activation.GeLU: nn.GELU, + Activation.GeLUTanh: GeLUTanh, + Activation.LeakyReLU: nn.LeakyReLU, + Activation.SquaredReLU: SquaredReLU, + Activation.Tanh: nn.Tanh, + Activation.QuickGELU: QuickGELU, + }[activation]() diff --git a/libai/layers/attention.py b/libai/layers/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..0bec6ebc1f1206c0c05d6d442b633bcbb849ee2c --- /dev/null +++ b/libai/layers/attention.py @@ -0,0 +1,281 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import enum +import math +from typing import Tuple + +import oneflow as flow +from oneflow import nn + +from .linear import Linear + + +class AttnMaskType(enum.Enum): + padding = 1 + causal = 2 + + +class MultiheadAttention(nn.Module): + """Multi-head attention layer, support self attention and cross attention. + + Args: + hidden_size: size of hidden state. + num_attention_heads: number of attention heads. + is_cross_attention: used to specify whether it is self attention or cross attention. + Defaults to False. + attention_dropout_prob: dropout probability of attention weights. + Defaults to 0.0. + output_dropout_prob: dropout probability of output. Defaults to 0.0. + init_method: method to initialize the input layer weights. + Defaults to ``init.xavier_normal_``. + output_layer_init_method: method to initialize the output layer weights. + If None, use ``init_method``. + bias_dropout_fusion: whether to fuse add bias and dropout. + Defaults to False. + scale_mask_softmax_fusion: whether to fuse scale, mask and softmax. + Defaults to False. + apply_query_key_layer_scaling: if `True`, scaling the attention score by layer index. + Defaults to False. + layer_idx: a layer_idx sign which determines the placements. + It will be used in pipeline parallelism. Defaults to 0. + """ + + def __init__( + self, + hidden_size, + num_attention_heads, + is_cross_attention=False, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + attn_mask_type=AttnMaskType.padding, + *, + layer_idx=0 + ): + super().__init__() + self.hidden_size = hidden_size + if output_layer_init_method is None: + output_layer_init_method = init_method + + assert ( + hidden_size % num_attention_heads == 0 + ), "hidden_size must be divisible by num_attention_heads." + + self.num_heads = num_attention_heads + self.head_size = hidden_size // num_attention_heads + self.attn_mask_type = attn_mask_type + + self.attention_dropout_prob = attention_dropout_prob + self.dropout = nn.Dropout(p=attention_dropout_prob) + self.norm_factor = 1.0 / math.sqrt(float(self.head_size)) + self.coeff = None + if apply_query_key_layer_scaling: + self.coeff = layer_idx + 1 + self.norm_factor /= self.coeff + + self.is_cross_attention = is_cross_attention + self.scale_mask_softmax_fusion = scale_mask_softmax_fusion + self.bias_dropout_fusion = bias_dropout_fusion + + if self.bias_dropout_fusion: + self.output_dropout_prob = output_dropout_prob + else: + self.output_dropout = nn.Dropout(p=output_dropout_prob) + + if self.is_cross_attention: + self.query = Linear( + self.hidden_size, + self.hidden_size, + parallel="col", + init_method=init_method, + layer_idx=layer_idx, + ) + self.key_value = Linear( + self.hidden_size, + self.hidden_size * 2, + parallel="col", + init_method=init_method, + layer_idx=layer_idx, + ) + else: + self.query_key_value = Linear( + self.hidden_size, + self.hidden_size * 3, + parallel="col", + init_method=init_method, + layer_idx=layer_idx, + ) + + self.dense = Linear( + self.hidden_size, + self.hidden_size, + parallel="row", + init_method=output_layer_init_method, + skip_bias_add=self.bias_dropout_fusion, + layer_idx=layer_idx, + ) + + def forward( + self, + hidden_states: flow.Tensor, + encoder_states: flow.Tensor = None, + attention_mask: flow.Tensor = None, + past_key_value: Tuple[flow.Tensor, flow.Tensor] = None, + use_cache: bool = False, + ): + """ + + Args: + hidden_states (flow.Tensor): shape is [bsz, tgt_len, hidden_size]. + encoder_states (flow.Tensor, optional): shape is [bsz, src_len, hidden_size]. + Defaults to None. + attention_mask (flow.Tensor, optional): shape is [bsz, 1, tgt_len, src_len]. + It should be the combination of padding mask and casual mask. + It is the padding mask of source input when used with self-attention in encoder. + And it is the combination of padding mask of target input and casual mask when + used with self-attention in decoder. It is the padding mask of source input when + used with cross-attention in decoder. + Defaults to None. + past_key_value (Tuple[flow.Tensor, flow.Tensor], optional): tuple of key and value, + each shape is [bsz, num_heads, src_len, head_size]. Defaults to None. + use_cache (bool, optional): it will be set to True, when the model is in the inference + phase and used for incremental decoding. Defaults to False. + """ + + # hidden_states, encoder_states: [S(0), B] + # attention_mask: [S(0), B] + + if encoder_states is not None: + encoder_states = encoder_states.to_global(placement=hidden_states.placement) + + if attention_mask is not None: + attention_mask = attention_mask.to_global(placement=hidden_states.placement) + + bsz, tgt_len = hidden_states.size()[:2] + + if self.is_cross_attention: + # if it is cross attention, key and value should be calculated only once, and the + # result can be reused. + query = self.query(hidden_states) + query = query.view(bsz, -1, self.num_heads, self.head_size) + query = query.permute(0, 2, 1, 3) + if past_key_value is not None: + key, value = past_key_value + elif encoder_states is not None: + key_value = self.key_value(encoder_states) + key_value = key_value.view(bsz, -1, self.num_heads, 2 * self.head_size) + key_value = key_value.permute(0, 2, 1, 3) + key, value = flow.chunk(key_value, chunks=2, dim=-1) + else: + raise ValueError( + "past_key_value and encoder_states cannot be None at the same time." + ) + else: + # if it is self attention, query, key, and value are all obtained from hidden_states. + # when in the inference phase of an incremental decoder, + # hidden_states is the last-added state, + # the full key and value could be obtained by concatenating with past_key_value. + query_key_value = self.query_key_value(hidden_states) + query_key_value = query_key_value.view(bsz, -1, self.num_heads, 3 * self.head_size) + query_key_value = query_key_value.permute( + 0, 2, 1, 3 + ) # [bsz, num_heads, src_len, 3 * head_size] + query, key, value = flow.chunk(query_key_value, chunks=3, dim=-1) + if past_key_value is not None: + past_key, past_value = past_key_value + key = flow.cat((past_key.type_as(key), key), dim=2) + value = flow.cat((past_value.type_as(value), value), dim=2) + + # query, key, value: [S(0), S(1)], shape: [bsz, num_heads, seq_length, head_size] + if use_cache: + past_key_value = (key, value) + + # [bsz, num_heads, tgt_len, src_len] with [S(0), S(1)] + attention_scores = flow.matmul(query, key, transpose_b=True, alpha=self.norm_factor) + + # [S(0), S(1)] x [S(0), B] = [S(0), S(1)] + if attention_mask is not None: + if self.scale_mask_softmax_fusion: + if self.attn_mask_type == AttnMaskType.padding: + attention_mask = ( + attention_mask.expand_as(attention_scores) if use_cache else attention_mask + ) + attention_weights = flow._C.fused_scale_mask_softmax_dropout( + attention_scores, + attention_mask, + fill_value=-10000.0, + scale=self.coeff, + p=self.attention_dropout_prob, + )[0] + else: + if self.coeff is not None: + attention_scores *= self.coeff + attention_scores = flow.mul(attention_scores, attention_mask) + attention_scores = attention_scores - 10000.0 * (1 - attention_mask) + # TODO(xingyu.liao): graph will occur `where_scalar` errors + # when using `masked_fill` + # attention_scores = attention_scores.masked_fill(1 - attention_mask, -10000.0) + attention_weights = flow.softmax(attention_scores, dim=-1) + # [bsz, num_heads, tgt_len, src_len] + attention_weights = self.dropout(attention_weights) + else: + if self.scale_mask_softmax_fusion and self.attn_mask_type == AttnMaskType.causal: + attention_weights = flow._C.fused_scale_tril_softmax_mask_scale( + attention_scores, + p=self.attention_dropout_prob, + diagonal=0, + tril_scale_value=self.coeff, + tril_fill_value=-10000.0, + )[0] + else: + attention_weights = flow.softmax(attention_scores, dim=-1) + # [bsz, num_heads, tgt_len, src_len] + attention_weights = self.dropout(attention_weights) + + # Context shape: [bsz, num_heads, tgt_len, head_size] with [S(0), S(1)] + context = flow.matmul(attention_weights, value) + # Change shape: [bsz, num_heads, tgt_len, head_size] -> [bsz, tgt_len, num_heads, head_size] + context = context.transpose(1, 2) + + # Concat multi-head results from + # [bsz, tgt_len, num_heads, head_size] -> [bsz, tgt_len, num_heads * head_size] + # SBP sign: [S(0), S(2)] + # [S(0), S(2)] x [B, S(0)] = [S(0), P] -> [S(0), B] + output = self.dense(context.flatten(2)) + + if self.bias_dropout_fusion: + output, bias = output + output = flow._C.fused_bias_add_dropout( + output, bias, p=self.output_dropout_prob, axis=output.ndim - 1 + ) + else: + output = self.output_dropout(output) + + if use_cache: + output = (output, past_key_value) + + return output + + def extra_repr(self) -> str: + return "hidden_size={}, num_heads={}, is_cross_attention={}".format( + self.hidden_size, + self.num_heads, + self.is_cross_attention, + ) diff --git a/libai/layers/cross_entropy.py b/libai/layers/cross_entropy.py new file mode 100644 index 0000000000000000000000000000000000000000..cde6b163205b47bcd9ca249e611bb80f4eb60d2c --- /dev/null +++ b/libai/layers/cross_entropy.py @@ -0,0 +1,48 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import oneflow as flow +from oneflow import nn + + +class ParallelCrossEntropyLoss(nn.Module): + """This criterion acts like :class:`~flow.nn.CrossEntropyLoss` except it will + execute distributed cross entropy loss computation cross different GPUs. + """ + + def forward(self, logits: flow.Tensor, target: flow.Tensor): + """Function for the distributed cross entropy. + + Args: + logits (flow.Tensor): vocab_parallel_logits with shape + (batch_size, seq_length, vocab_size) and sbp signature is [S(0), S(2)]. + target (flow.Tensor): target with shape (batch_size, seq_length) and + sbp signature is [S(0), B]. + """ + assert logits.ndim == 3 + assert target.ndim == 2 + assert logits.shape[0:2] == target.shape + + target = target.to_global(placement=logits.placement) + + # Change -1 in target to 0 because sparse_softmax_cross_entropy don't accept -1 + target = target * (target >= 0) + + lm_loss = flow._C.sparse_softmax_cross_entropy( + logits.view(-1, logits.shape[-1]), + target.view(-1), + ) + return lm_loss diff --git a/libai/layers/droppath.py b/libai/layers/droppath.py new file mode 100644 index 0000000000000000000000000000000000000000..fd628dbe21ffddbce49291e66562364b1b1a43b9 --- /dev/null +++ b/libai/layers/droppath.py @@ -0,0 +1,46 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +import oneflow.nn as nn + + +def drop_path(x, drop_prob: float = 0.5, training: bool = False, scale_by_keep: bool = True): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" + + if drop_prob == 0.0 or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + + # similar opeartion to new_tensor(shape).bernoulli_(keep_prob) + random_tensor = flow.rand(*shape, dtype=x.dtype, sbp=x.sbp, placement=x.placement) + random_tensor = (random_tensor < keep_prob).to(flow.float32) + + if keep_prob > 0.0 and scale_by_keep: + random_tensor = random_tensor / keep_prob + return x * random_tensor + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" + + def __init__(self, drop_prob: float = 0.0, scale_by_keep: bool = True): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + self.scale_by_keep = scale_by_keep + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training, self.scale_by_keep) diff --git a/libai/layers/embedding.py b/libai/layers/embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..86b25549b98ac3240a4ed71f8f18d62372b98df7 --- /dev/null +++ b/libai/layers/embedding.py @@ -0,0 +1,286 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import oneflow as flow +from oneflow import nn +from oneflow.nn import init + +from libai.utils import distributed as dist + + +class Embedding(nn.Module): + """Construct the trainable embedding module, which does not support parallelization. + This can be used for positional embedding and token type embedding. + + Arguments: + num_embeddings: size of vocabulary. + embedding_dim: dimension of embeddings. + padding_idx: pad index. Defaults to None. + init_method: method to initialize weights. Defaults to ``flow.nn.init.xavier_normal_``. + amp_enabled: fp16 option for embedding weight. Defaults to False. + """ + + def __init__( + self, + num_embeddings, + embedding_dim, + padding_idx=None, + init_method=init.xavier_normal_, + amp_enabled=False, + layer_idx=0, + ): + super().__init__() + self.num_embeddings = num_embeddings + self.embedding_dim = embedding_dim + if padding_idx is not None: + if padding_idx > 0: + assert ( + padding_idx < self.num_embeddings + ), "Padding_idx must be within num_embeddings" + elif padding_idx < 0: + assert ( + padding_idx >= -self.num_embeddings + ), "Padding_idx must be within num_embeddings" + padding_idx = self.num_embeddings + padding_idx + self.padding_idx = padding_idx + self.init_method = init_method + self.amp_enabled = amp_enabled + + assert num_embeddings > 0 + self.weight = nn.Parameter( + flow.empty( + (num_embeddings, embedding_dim), + dtype=flow.float32, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + ) + self.init_method(self.weight) + # FIXME(lxy): Fill padding_idx is not supported in nd_sbp right now. + # self._fill_padding_idx_with_zero() + + def forward(self, input_ids): + weight = flow._C.amp_white_identity(self.weight) if self.amp_enabled else self.weight + # embeddings with sbp sign: [B, B] + # [B, B] x [S(0), B] --> [S(0), B] + # ↑ ↑ ↑ + # embed pos_ids pos_embed + input_embeds = flow._C.gather(weight, input_ids, axis=0) + return input_embeds + + def _fill_padding_idx_with_zero(self) -> None: + if self.padding_idx is not None: + with flow.no_grad(): + self.weight[self.padding_idx] = flow.zeros( + self.embedding_dim, + placement=dist.get_layer_placement(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + + def extra_repr(self) -> str: + s = "num_embeddings={num_embeddings}, embedding_dim={embedding_dim}" + if self.padding_idx is not None: + s += ", padding_idx={padding_idx}" + return s.format(**self.__dict__) + + +class VocabEmbedding(nn.Module): + """Construct the word embeddings, which may be split along vocabulary dimension. + + Arguments: + num_embeddings: size of vocabulary. + embedding_dim: dimension of embeddings. + padding_idx: pad index. Defaults to None. + init_method: method to initialize weights. Defaults to ``flow.nn.init.xavier_normal_``. + amp_enabled: fp16 option for embedding weight. Defaults to False. + """ + + def __init__( + self, + num_embeddings, + embedding_dim, + padding_idx=None, + init_method=init.xavier_normal_, + amp_enabled=False, + ): + super().__init__() + self.num_embeddings = num_embeddings + self.embedding_dim = embedding_dim + if padding_idx is not None: + if padding_idx > 0: + assert ( + padding_idx < self.num_embeddings + ), "Padding_idx must be within num_embeddings" + elif padding_idx < 0: + assert ( + padding_idx >= -self.num_embeddings + ), "Padding_idx must be within num_embeddings" + padding_idx = self.num_embeddings + padding_idx + self.padding_idx = padding_idx + self.init_method = init_method + self.amp_enabled = amp_enabled + + # Word token embedding shape with (vocab_size, hidden_size) + # sbp: [B, S(0)] + self.weight = nn.Parameter( + flow.empty( + (num_embeddings, embedding_dim), + dtype=flow.float32, + placement=dist.get_layer_placement(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.split(0)]), + ) + ) + # Initialize the word embedding + self.init_method(self.weight) + # FIXME(Lxy): Fill padding_idx is not supported in nd_sbp right now. + # self._fill_padding_idx_with_zero() + + def forward(self, input_ids): + weight = flow._C.amp_white_identity(self.weight) if self.amp_enabled else self.weight + # input_ids with shape (batch_size, seq_len), and sbp sign: [S(0), B] + + # Gather forward sbp sign + # [B, S(0)] x [S(0), B] --> [S(0), P] + # ↑ ↑ ↑ + # embed input_ids input_embeds + input_embeds = flow._C.gather(weight, input_ids, axis=0) + # Set the embeds sbp from [S(0), P] --> [S(0), B] to get complete embedding results. + input_embeds = input_embeds.to_global(sbp=dist.get_hidden_sbp()) + + return input_embeds + + def _fill_padding_idx_with_zero(self) -> None: + if self.padding_idx is not None: + with flow.no_grad(): + self.weight[self.padding_idx] = flow.zeros( + self.embedding_dim, + placement=dist.get_layer_placement(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + + def extra_repr(self) -> str: + s = "num_embeddings={num_embeddings}, embedding_dim={embedding_dim}" + if self.padding_idx is not None: + s += ", padding_idx={padding_idx}" + return s.format(**self.__dict__) + + +class SinePositionalEmbedding(nn.Module): + """Construct the sinusoidal positional embeddings. + + Arguments: + num_embeddings: size of vocabulary. + embedding_dim: dimension of embeddings. + """ + + def __init__(self, num_embeddings, embedding_dim): + super().__init__() + + self.embedding_dim = embedding_dim + self.num_embeddings = num_embeddings + + position_embedding = flow.zeros( + num_embeddings, + embedding_dim, + dtype=flow.float32, + placement=dist.get_layer_placement(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + position = flow._C.global_arange( + start=0, + end=num_embeddings, + placement=dist.get_layer_placement(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + dtype=flow.float32, + ).unsqueeze(1) + position_range = flow._C.global_arange( + start=0, + end=embedding_dim, + step=2, + placement=dist.get_layer_placement(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + dtype=flow.float32, + ) + div_term = flow.exp(position_range * (-math.log(10000.0) / embedding_dim)) + + position_embedding[:, 0::2] = flow.sin(position * div_term) + position_embedding[:, 1::2] = flow.cos(position * div_term) + self.register_buffer("position_embedding", position_embedding) + + def forward(self, position_ids): + position_embeds = flow._C.gather(self.position_embedding, position_ids, axis=0) + return position_embeds + + def extra_repr(self) -> str: + s = "num_embeddings={num_embeddings}, embedding_dim={embedding_dim}" + return s.format(**self.__dict__) + + +class PatchEmbedding(nn.Module): + """2D Image to Patch Embedding + + Arguments: + img_size: size of input image. Default to 224. + patch_size: embedded patch size. Default to 16. + in_chans: input channel's size. Default to 3. + embed_dim: dimension of embedded patch. Default to 768. + norm_layer: normalization patch embedding or not. Default to None. + flatten: flatten patch embedding or keep the 2-D shape. Default to True. + layer_idx: A layer_idx sign which determines the placement. It will be used in pipeline + parallelism. Default to 0. + """ + + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + norm_layer=None, + flatten=True, + *, + layer_idx=0, + ): + super().__init__() + img_size = img_size if isinstance(img_size, tuple) else (img_size, img_size) + patch_size = patch_size if isinstance(patch_size, tuple) else (patch_size, patch_size) + self.img_size = img_size + self.patch_size = patch_size + self.grid_size = (img_size[0] // patch_size[0], img_size[1] // patch_size[1]) + self.num_patches = self.grid_size[0] * self.grid_size[1] + self.flatten = flatten + self.proj = nn.Conv2d( + in_chans, embed_dim, kernel_size=patch_size, stride=patch_size + ).to_global( + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(layer_idx), + ) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + def forward(self, x): + B, C, H, W = x.shape + assert ( + H == self.img_size[0] + ), f"Input image height ({H}) doesn't match model ({self.img_size[0]})." + assert ( + W == self.img_size[1] + ), f"Input image width ({W}) doesn't match model ({self.img_size[1]})." + x = self.proj(x) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + x = self.norm(x) + return x diff --git a/libai/layers/layer_norm.py b/libai/layers/layer_norm.py new file mode 100644 index 0000000000000000000000000000000000000000..b154645cc47a92a688b7c10391c647147071f866 --- /dev/null +++ b/libai/layers/layer_norm.py @@ -0,0 +1,129 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +from oneflow import nn + +from libai.utils import distributed as dist + + +class LayerNorm(nn.Module): + """Applies Layer Normalization over a mini-batch of inputs in 1D parallelism. + + Args: + normalized_shape: input shape from an expected input of size. + eps: a value added to the denominator for numerical stability. Defaults to 1e-5. + elementwise_affine: a boolean value that when set to ``True``, this module + has learnable per-element affine parameters initialized to ones (for weights) + and zeros (for biases). Default: ``True``. + elementwise_affine: a boolean value that when set to ``True``, this module + has learnable per-element affine parameters initialized to ones (for weights) + and zeros (for biases). Default: ``True``. + bias: If set to ``False``, the layer will not learn an additive bias. Defaults to ``True``. + layer_idx: a layer_idx sign which determines the placement. It will be used in pipeline + parallelism. Defaults to 0. + """ + + def __init__( + self, normalized_shape, eps=1e-5, elementwise_affine=True, bias=True, *, layer_idx=0 + ): + super().__init__() + if isinstance(normalized_shape, int): + normalized_shape = (normalized_shape,) + self.normalized_shape = tuple(normalized_shape) + self.eps = eps + self.elementwise_affine = elementwise_affine + self.layer_idx = layer_idx + + if elementwise_affine: + self.weight = nn.Parameter( + flow.ones( + normalized_shape, + dtype=flow.float32, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + ) + self.bias = nn.Parameter( + flow.zeros( + normalized_shape, + dtype=flow.float32, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ), + requires_grad=bias, + ) + else: + self.weight = None + self.bias = None + + def forward(self, x): + assert x.shape[-len(self.normalized_shape) :] == self.normalized_shape + begin_norm_axis = x.ndim - len(self.normalized_shape) + begin_params_axis = x.ndim - len(self.normalized_shape) + if self.elementwise_affine: + y = flow._C.layer_norm_affine( + x, + self.weight, + self.bias, + begin_norm_axis=begin_norm_axis, + begin_params_axis=begin_params_axis, + epsilon=self.eps, + ) + else: + y = flow._C.layer_norm( + x, + begin_norm_axis=begin_norm_axis, + begin_params_axis=begin_params_axis, + epsilon=self.eps, + ) + return y + + def extra_repr(self) -> str: + return "{normalized_shape}, eps={eps}, elementwise_affine={elementwise_affine}".format( + **self.__dict__ + ) + + +class RMSLayerNorm(nn.Module): + """T5 uses a layer_norm which only scales and doesn't shift, which is also known as + Root Mean Square Layer Normalization thus varience is calculated w/o mean and + there is no bias. More details see: https://arxiv.org/abs/1910.07467. + + Args: + normalized_shape: input shape from an expected input of size. + eps: a value added to the denominator for numerical stability. Defaults to 1e-5. + elementwise_affine: a boolean value that when set to ``True``, this module + has learnable per-element affine parameters initialized to ones (for weights) + and zeros (for biases). Default: ``True``. + layer_idx: a layer_idx sign which determines the placement. It will be used in pipeline + parallelism. Defaults to 0. + """ + + def __init__(self, normalized_shape, eps=1e-6, layer_idx=0): + super().__init__() + self.layer_idx = layer_idx + self.weight = flow.nn.Parameter( + flow.ones( + normalized_shape, + dtype=flow.float32, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + ) + self.l2norm_epsilon = eps + + def forward(self, hidden_states): + return flow._C.rms_layer_norm(hidden_states, self.weight, self.l2norm_epsilon) diff --git a/libai/layers/linear.py b/libai/layers/linear.py new file mode 100644 index 0000000000000000000000000000000000000000..648200a982880cf01aba19429bcfbf674725430d --- /dev/null +++ b/libai/layers/linear.py @@ -0,0 +1,175 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import oneflow as flow +from oneflow import nn + +from libai.utils import distributed as dist + + +class Linear1D(nn.Module): + r"""Linear layer with 1D parallelism which includes column parallelism and row parallelism. + The linear layer is defined as :math:`y = xA^T + b`. + + In column parallelism, A^T is parallelized along the second dimension + as :math:`A^T = [A_1, ..., A_p]`. + + In row parallelism, A^T is parallelized along the first dimension and X along its second + dimension as: + + .. math:: + A^T = \begin{bmatrix} + A\_1 \\ + . \\ + . \\ + . \\ + A\_p + \end{bmatrix} + x = \begin{bmatrix} + x\_1 & ... & x\_p + \end{bmatrix} + + Arguments: + in_features: size of each input sample. + out_features: size of each output sample. + bias: If set to ``False``, the layer will not learn an additive bias. Defaults to ``True``. + parallel: Parallel mode. Defaults to "data". + init_method: method to initialize weight. Defaults to :func:`nn.init.xavier_normal_`. + skip_bias_add: skip adding bias but instead return it, so that adding bias can be fused with + other elementwise operations. Defaults to ``False``. + layer_idx: A layer_idx sign which determines the placement. It will be used in pipeline + parallelism. Defaults to 0. + """ + + def __init__( + self, + in_features, + out_features, + bias=True, + parallel="data", + init_method=nn.init.xavier_normal_, + skip_bias_add=False, + *, + layer_idx=0, # enforce layer_idx passed with keyword + ): + super().__init__() + self.in_features = in_features + self.out_features = out_features + self.parallel = parallel + self.skip_bias_add = skip_bias_add + + if parallel == "col": + # Column parallel + # weight sbp sign: [B, S(0)], weight will be transposed when performing matmul + # so weight sbp sign actually be [B, S(1)] + # bias sbp sign: [B, S(0)] + weight_sbp = dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.split(0)]) + bias_sbp = dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.split(0)]) + elif parallel == "row": + # Row parallel + # weight sbp sign: [B, S(1)], weight will be transposed when performing matmul + # so weight sbp sign actually be [B, S(1)] + # bias sbp sign: [B, B] + weight_sbp = dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.split(1)]) + bias_sbp = dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]) + elif parallel == "data": + weight_sbp = dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]) + bias_sbp = dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]) + else: + raise KeyError(f"{parallel} is not supported! Only support ('data', 'row' and 'col')") + + self.weight = flow.nn.Parameter( + flow.empty( + (out_features, in_features), + dtype=flow.float32, + placement=dist.get_layer_placement(layer_idx), # for pipeline parallelism placement + sbp=weight_sbp, + ) + ) + init_method(self.weight) + + self.bias = ( + flow.nn.Parameter( + flow.zeros( + (out_features,), + dtype=flow.float32, + placement=dist.get_layer_placement(layer_idx), + sbp=bias_sbp, + ) + ) + if bias + else None + ) + + def forward(self, x): + if dist.same_sbp(self.weight.sbp, dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.split(0)])): + # If the last dim of weight sbp sign is S(0), then last dim of weight.t sbp + # sign is S(1), so the last dim of x sbp sign must be B. + if self.weight.sbp[-1] == flow.sbp.split(0): + x_sbp = x.sbp[:-1] + (flow.sbp.broadcast,) + x = x.to_global(sbp=x_sbp) + + # x.grad sbp must be x.sbp, otherwise backward pass cannot be performed correctly. + x = x.to_global(grad_sbp=x.sbp) + x = flow.matmul(x, self.weight, transpose_b=True) + + elif dist.same_sbp( + self.weight.sbp, dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.split(1)]) + ): + # If the last dim of weight sbp sign is S(1), then last dim of weight.t sbp + # sign is S(0), so the last dim of x sbp sign must be S(ndim-1). + if self.weight.sbp[-1] == flow.sbp.split(1): + x_sbp = x.sbp[:-1] + (flow.sbp.split(x.ndim - 1),) + x = x.to_global(sbp=x_sbp) + out_sbp = x.sbp[:-1] + (flow.sbp.broadcast,) + else: + out_sbp = x.sbp + + x = flow.matmul(x, self.weight, transpose_b=True) + # Change x.sbp for followup forward pass. + # This line can be removed when sbp can be auto inferred. + x = x.to_global(sbp=out_sbp) + elif dist.same_sbp( + self.weight.sbp, dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]) + ): + # x.grad sbp must be x.sbp, otherwise backward pass cannot be performed correctly. + x = x.to_global(grad_sbp=x.sbp) + # NOTE(chengcheng): when input x is [S(0), B], there is no need to change sbp for x. + # x = x.to_global(sbp=dist.get_nd_sbp([flow.sbp.split(0), flow.sbp.split(0)])) + x = flow.matmul(x, self.weight, transpose_b=True) + else: + # Not supported weight_sbp, deduce sbp and communicate with nccl automatically. + x = flow.matmul(x, self.weight, transpose_b=True) + + if self.bias is not None: + if self.skip_bias_add: + return x, self.bias + else: + return x + self.bias + else: + return x + + def extra_repr(self) -> str: + return "in_features={}, out_features={}, bias={}, parallel={}".format( + self.in_features, + self.out_features, + self.bias is not None, + self.parallel, + ) + + +# Give an alias for Linear1d +Linear = Linear1D diff --git a/libai/layers/lm_logits.py b/libai/layers/lm_logits.py new file mode 100644 index 0000000000000000000000000000000000000000..3d3c877a61b7c85a9c930f2cf4403b0c1c806b44 --- /dev/null +++ b/libai/layers/lm_logits.py @@ -0,0 +1,61 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +from oneflow import nn + +from libai.utils import distributed as dist + + +class LMLogits(nn.Module): + def __init__(self, vocab_size, bias=False): + super().__init__() + self.bias = ( + nn.Parameter( + flow.zeros( + (vocab_size,), + dtype=flow.float32, + placement=dist.get_layer_placement(-1), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.split(0)]), + ) + ) + if bias + else None + ) + + def forward(self, input, word_embeddings): + """LM logits using word embedding weights""" + # input with sbp sign [S(0), B] and word_embeddings with sbp sign [S(0), B] + + # NOTE(l1aoxingyu): This is for pipeline parallelism + # change word embedding placement from stage(0) to stage(-1) + w = word_embeddings.to_global(placement=input.placement) + + # NOTE(l1aoxingyu): input x embed^T = logits with sbp sign + # [S(0), B] x [B, S(1)] --> [S(0), S(1)] + # ↑ ↑ ↑ + # input embed^T logits + # Backward pass input.grad = logits.grad x embed with sbp sign + # [S(0), S(1)] x [B, S(0)] --> [S(0), P] + # ↑ ↑ ↑ + # logits.grad embed input.grad + # When use input.grad as head node for backward pass, need to convert + # its sbp sign fromm [S(0), P] --> [S(0), B] + input = input.to_global(grad_sbp=input.sbp) + + logits = flow._C.matmul(input, w, transpose_b=True) + if self.bias is not None: + logits = logits + self.bias + return logits diff --git a/libai/layers/mlp.py b/libai/layers/mlp.py new file mode 100644 index 0000000000000000000000000000000000000000..03ae663de3b9016bc61dd18ab0f7e0fd6b69b8c9 --- /dev/null +++ b/libai/layers/mlp.py @@ -0,0 +1,114 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +from oneflow import nn + +from libai.layers import Linear, build_activation + + +class MLP(nn.Module): + """MLP + + MLP will take the input with h hidden state, project it to intermediate + hidden dimension, perform gelu transformation, and project the + state back into h hidden dimension. + + Arguments: + hidden_size: size of each input and output sample. + ffn_hidden_size: size of each intermediate sample. + output_dropout_prob: Output dropout probability. Defaults to 0.0. + init_method: method to initialize the first linear weight. + Defaults to :func:`nn.init.xavier_normal_`. + output_layer_init_method: method to initialize the second linear weight. If set to None, + it will use ``init_method`` instead. Defaults to None. + bias_gelu_fusion: If set to ``True``, it will fuse bias adding and elementwise + gelu activation. Defaults to ``False``. + bias_dropout_fusion: If set to ``True``, it will fuse bias adding and dropout. + Defaults to ``False``. + layer_idx: A layer_idx sign which determines the placement. It will be used in + pipeline parallelism. Defaults to 0. + """ + + def __init__( + self, + hidden_size, + ffn_hidden_size, + output_dropout_prob=0.0, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + *, + layer_idx=0, + ): + super().__init__() + self.output_dropout_prob = output_dropout_prob + self.bias_gelu_fusion = bias_gelu_fusion + self.bias_dropout_fusion = bias_dropout_fusion + + if output_layer_init_method is None: + output_layer_init_method = init_method + + self.dense_h_to_4h = Linear( + hidden_size, + ffn_hidden_size, + bias=True, + parallel="col", + skip_bias_add=bias_gelu_fusion, + init_method=init_method, + layer_idx=layer_idx, + ) + + if not bias_gelu_fusion: + self.activation_func = build_activation("gelu") + + self.dense_4h_to_h = Linear( + ffn_hidden_size, + hidden_size, + bias=True, + parallel="row", + skip_bias_add=bias_dropout_fusion, + init_method=output_layer_init_method, + layer_idx=layer_idx, + ) + + if not bias_dropout_fusion: + self.dropout = nn.Dropout(self.output_dropout_prob) + + def forward(self, hidden_states): + intermediate = self.dense_h_to_4h(hidden_states) + if self.bias_gelu_fusion: + intermediate, bias = intermediate + intermediate = flow._C.fused_bias_add_gelu( + intermediate, bias, axis=intermediate.ndim - 1 + ) + else: + intermediate = self.activation_func(intermediate) + + output = self.dense_4h_to_h(intermediate) + if self.bias_dropout_fusion: + output, bias = output + output = flow._C.fused_bias_add_dropout( + output, bias, p=self.output_dropout_prob, axis=output.ndim - 1 + ) + else: + output = self.dropout(output) + return output + + def extra_repr(self) -> str: + return "bias_gelu_fusion={}, bias_dropout_fusion={}, dropout={}".format( + self.bias_gelu_fusion, self.bias_dropout_fusion, self.output_dropout_prob + ) diff --git a/libai/layers/transformer_layer.py b/libai/layers/transformer_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..4372daf333aa9fb1400a8c4610f32bb1d6f4cf09 --- /dev/null +++ b/libai/layers/transformer_layer.py @@ -0,0 +1,248 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow.nn as nn + +from libai.utils import distributed as dist + +from .attention import AttnMaskType, MultiheadAttention +from .droppath import DropPath +from .layer_norm import LayerNorm +from .mlp import MLP + + +class TransformerLayer(nn.Module): + """A single transformer layer. + + Transformer layer takes input with size [bsz, seq_length, hidden size] and returns an + output of the same size. + The input and output has same sbp sign, (S(0), B). + + Arguments: + hidden_size: size of hidden state. + ffn_hidden_size: size of feed forword neural network. + num_attention_heads: number of attention heads. + is_decoder: used to specify whether this is transformer encoder layer or transformer + decoder layer. Default: ``False``. + attention_dropout_prob: dropout probability of attention weights. + output_dropout_prob: dropout probability of output. + layernorm_epsilon: epsilon used in layernorm layer. Default: `1e-5`. + init_method: method to initialize the input layer weights. + output_layer_init_method: method to initialize the output layer weights. + If None, use `init_method`. + bias_gelu_fusion: whether fuse add bias and gelu. Default: ``False``. + bias_dropout_fusion: whether fuse add bias and dropout. Default: ``False``. + scale_mask_softmax_fusion: whether to fuse scale, mask and softmax. Default: ``False``. + apply_query_key_layer_scaling: if `true`, scaling the attention score by layer index. + Default: ``False``. + apply_residual_post_layernorm: if ``true``, use original BERT residual + connection ordering. Otherwise, use Megatron BERT residual connection which + is more stable when scaling model size introduced in + https://arxiv.org/pdf/1909.08053.pdf. + Default: ``False``. + layer_idx: the layer index, which determines the placement. + """ + + def __init__( + self, + hidden_size, + ffn_hidden_size, + num_attention_heads, + is_decoder=False, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + drop_path_prob=0.0, + layernorm_epsilon=1e-5, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + apply_residual_post_layernorm=False, + attn_mask_type=AttnMaskType.padding, + *, + layer_idx=0 + ): + super().__init__() + self.hidden_size = hidden_size + self.ffn_hidden_size = ffn_hidden_size + self.num_attention_heads = num_attention_heads + self.attention_dropout_prob = attention_dropout_prob + self.output_dropout_prob = output_dropout_prob + self.layernorm_epsilon = layernorm_epsilon + self.attn_mask_type = attn_mask_type + + self.layer_idx = layer_idx + self.is_decoder = is_decoder + + self.bias_gelu_fusion = bias_gelu_fusion + self.bias_dropout_fusion = bias_dropout_fusion + self.scale_mask_softmax_fusion = scale_mask_softmax_fusion + self.apply_query_key_layer_scaling = apply_query_key_layer_scaling + self.apply_residual_post_layernorm = apply_residual_post_layernorm + + self.init_method = init_method + if output_layer_init_method is None: + output_layer_init_method = init_method + self.output_layer_init_method = output_layer_init_method + + self.drop_path = DropPath(drop_path_prob) if drop_path_prob > 0.0 else nn.Identity() + + self.input_layernorm = LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon, layer_idx=self.layer_idx + ) + + self.self_attention = self.build_attention(is_cross_attention=False) + self.post_attention_layernorm = LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon, layer_idx=self.layer_idx + ) + + if self.is_decoder: + self.cross_attention = self.build_attention(is_cross_attention=True) + self.post_cross_attention_layernorm = LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon, layer_idx=self.layer_idx + ) + + self.mlp = MLP( + self.hidden_size, + self.ffn_hidden_size, + self.output_dropout_prob, + self.init_method, + output_layer_init_method=self.output_layer_init_method, + bias_gelu_fusion=self.bias_gelu_fusion, + bias_dropout_fusion=self.bias_dropout_fusion, + layer_idx=self.layer_idx, + ) + + def forward( + self, + hidden_states, + attention_mask=None, + encoder_states=None, + encoder_attention_mask=None, + past_key_value=None, + use_cache=False, + ): + """ + Args: + hidden_states: shape is (batch_size, seq_length, hidden_size), + sbp signature is (S(0), B). + attention_mask: the combination of key padding mask and casual mask of hidden states + with shape (batch_size, 1, seq_length, seq_length) and the sbp + signature is (S(0), B), + encoder_states: encoder output with shape (batch_size, seq_length, hidden_size) + and the sbp signature is (S(0), B), which will be used in cross attention. + encoder_attention_mask: key padding mask of encoder states with shape + (batch_size, 1, seq_length, seq_length) and the sbp signature is (S(0), B). + past_key_value: tuple of key and value, each shape is + (seq_length, bsz, num_heads, head_size), For decoder layer, + the past_key_value contains the states both from self attention + and cross attention. + use_cache: it will be set to `True` when the model is in the inference phase and + used for incremental decoding. + """ + # Change placement for pipeline parallelsim + hidden_states = hidden_states.to_global(placement=dist.get_layer_placement(self.layer_idx)) + + # hidden_states shape: (batch_size, seq_length, hidden_size) + if attention_mask is not None: + attention_mask = attention_mask.to_global( + placement=dist.get_layer_placement(self.layer_idx) + ) + + if past_key_value is not None: + if self.is_decoder: + assert len(past_key_value) == 4 + self_attn_past_key_value = past_key_value[:2] + cross_attn_past_key_value = past_key_value[2:] + else: + self_attn_past_key_value = past_key_value + cross_attn_past_key_value = None + else: + self_attn_past_key_value, cross_attn_past_key_value = None, None + + layernorm_output = self.input_layernorm(hidden_states) + attention_output = self.self_attention( + layernorm_output, + attention_mask=attention_mask, + past_key_value=self_attn_past_key_value, + use_cache=use_cache, + ) + attention_output = self.drop_path(attention_output) + + if use_cache: + attention_output, presents = attention_output + + if self.apply_residual_post_layernorm: + residual = layernorm_output + else: + residual = hidden_states + + hidden_states = residual + attention_output + + layernorm_output = self.post_attention_layernorm(hidden_states) + + if self.is_decoder: + attention_output = self.cross_attention( + layernorm_output, + encoder_states, + attention_mask=encoder_attention_mask, + past_key_value=cross_attn_past_key_value, + use_cache=use_cache, + ) + + if use_cache: + attention_output, decoder_presents = attention_output + presents += decoder_presents + + attention_output = self.drop_path(attention_output) + if self.apply_residual_post_layernorm: + residual = layernorm_output + else: + residual = hidden_states + + hidden_states = residual + attention_output + layernorm_output = self.post_cross_attention_layernorm(hidden_states) + + mlp_output = self.mlp(layernorm_output) + mlp_output = self.drop_path(mlp_output) + + if self.apply_residual_post_layernorm: + residual = layernorm_output + else: + residual = hidden_states + + output = residual + mlp_output + + if use_cache: + output = (output, presents) + return output + + def build_attention(self, is_cross_attention=False): + return MultiheadAttention( + self.hidden_size, + self.num_attention_heads, + is_cross_attention=is_cross_attention, + attention_dropout_prob=self.attention_dropout_prob, + output_dropout_prob=self.output_dropout_prob, + init_method=self.init_method, + output_layer_init_method=self.output_layer_init_method, + bias_dropout_fusion=self.bias_dropout_fusion, + scale_mask_softmax_fusion=self.scale_mask_softmax_fusion, + apply_query_key_layer_scaling=self.apply_query_key_layer_scaling, + attn_mask_type=self.attn_mask_type, + layer_idx=self.layer_idx, + ) diff --git a/libai/models/__init__.py b/libai/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ef101bbf274b9e462af81862c645e3e1da0ef344 --- /dev/null +++ b/libai/models/__init__.py @@ -0,0 +1,44 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from .bert_model import BertForPreTraining, BertModel, BertForClassification +from .roberta_model import RobertaForPreTraining, RobertaForCausalLM, RobertaModel +from .build import build_graph, build_model +from .t5_model import T5ForPreTraining, T5Model +from .gpt_model import GPTForPreTraining, GPTModel +from .vision_transformer import VisionTransformer +from .swin_transformer import SwinTransformer +from .swin_transformer_v2 import SwinTransformerV2 +from .resmlp import ResMLP + +__all__ = [ + "build_model", + "build_graph", + "BertModel", + "BertForPreTraining", + "BertForClassification", + "RobertaModel", + "RobertaForCausalLM", + "RobertaForPreTraining", + "T5Model", + "T5ForPreTraining", + "GPTModel", + "GPTForPreTraining", + "VisionTransformer", + "SwinTransformer", + "SwinTransformerV2", + "ResMLP", +] diff --git a/libai/models/__pycache__/__init__.cpython-39.pyc b/libai/models/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d4f5791cc0acbcc1fd5529f1f77753fa622ba9d Binary files /dev/null and b/libai/models/__pycache__/__init__.cpython-39.pyc differ diff --git a/libai/models/__pycache__/bert_model.cpython-39.pyc b/libai/models/__pycache__/bert_model.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7d8790b81878b792dc13ecdbcfdf52984a9f9bc Binary files /dev/null and b/libai/models/__pycache__/bert_model.cpython-39.pyc differ diff --git a/libai/models/__pycache__/build.cpython-39.pyc b/libai/models/__pycache__/build.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fc60d35d5d16be2f6d0e86cb682896f5348d07a1 Binary files /dev/null and b/libai/models/__pycache__/build.cpython-39.pyc differ diff --git a/libai/models/__pycache__/gpt_model.cpython-39.pyc b/libai/models/__pycache__/gpt_model.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..584b2288fe9c4848ec8f30951365f9c3eb0aa897 Binary files /dev/null and b/libai/models/__pycache__/gpt_model.cpython-39.pyc differ diff --git a/libai/models/__pycache__/resmlp.cpython-39.pyc b/libai/models/__pycache__/resmlp.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ec4a8522c7991eb1dbbc2af0dacd66547556c833 Binary files /dev/null and b/libai/models/__pycache__/resmlp.cpython-39.pyc differ diff --git a/libai/models/__pycache__/roberta_model.cpython-39.pyc b/libai/models/__pycache__/roberta_model.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f4b8b8d3d152f7f3be7f810d1e3dd5065db5adfb Binary files /dev/null and b/libai/models/__pycache__/roberta_model.cpython-39.pyc differ diff --git a/libai/models/__pycache__/swin_transformer.cpython-39.pyc b/libai/models/__pycache__/swin_transformer.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d1fb74e95df9eedde8b03d2270d846b1859daa11 Binary files /dev/null and b/libai/models/__pycache__/swin_transformer.cpython-39.pyc differ diff --git a/libai/models/__pycache__/swin_transformer_v2.cpython-39.pyc b/libai/models/__pycache__/swin_transformer_v2.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..feb59b67b393b988c3a3589b0b48ba6729b4ad6f Binary files /dev/null and b/libai/models/__pycache__/swin_transformer_v2.cpython-39.pyc differ diff --git a/libai/models/__pycache__/t5_model.cpython-39.pyc b/libai/models/__pycache__/t5_model.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..69e9602b81ce73cee00163d50119ee9d2adf9609 Binary files /dev/null and b/libai/models/__pycache__/t5_model.cpython-39.pyc differ diff --git a/libai/models/__pycache__/vision_transformer.cpython-39.pyc b/libai/models/__pycache__/vision_transformer.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..63a20480a10524f7ba5fc23bd3e2e9f5b54587e3 Binary files /dev/null and b/libai/models/__pycache__/vision_transformer.cpython-39.pyc differ diff --git a/libai/models/bert_model.py b/libai/models/bert_model.py new file mode 100644 index 0000000000000000000000000000000000000000..bf477f6918f70a5f40455fa79b426a11eb07ac66 --- /dev/null +++ b/libai/models/bert_model.py @@ -0,0 +1,591 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +from oneflow import nn + +from libai.config import configurable +from libai.layers import ( + Embedding, + LayerNorm, + Linear, + LMLogits, + ParallelCrossEntropyLoss, + TransformerLayer, + VocabEmbedding, + build_activation, +) +from libai.layers.attention import AttnMaskType +from libai.utils import distributed as dist + +from .utils import init_method_normal, scaled_init_method_normal + + +class BertExtendedAttnMask(nn.Module): + def forward(self, attention_mask): + # We create a 3D attention mask from a 2D tensor mask. + # [b, 1, s] + attention_mask_b1s = attention_mask.unsqueeze(1) + # [b, s, 1] + attention_mask_bs1 = attention_mask.unsqueeze(2) + # [b, s, s] + attention_mask_bss = attention_mask_b1s * attention_mask_bs1 + # [b, 1, s, s] + extended_attention_mask = attention_mask_bss.unsqueeze(1) + + return extended_attention_mask + + +class BertEmbeddings(nn.Module): + def __init__( + self, + vocab_size, + hidden_size, + max_sequence_length, + embedding_dropout_prob, + num_tokentypes=0, + init_method=nn.init.xavier_normal_, + amp_enabled=False, + ): + super().__init__() + self.vocab_embeddings = VocabEmbedding( + vocab_size, hidden_size, init_method=init_method, amp_enabled=amp_enabled + ) + self.position_embeddings = Embedding( + max_sequence_length, hidden_size, init_method=init_method, amp_enabled=amp_enabled + ) + + # NOTE(l1aoxingyu): Set position_ids sbp sign to [B, B] initially, because position_ids is a + # 1D-tensor from 0 to seq_length, if set to [S(0), B] at first, then position_ids + # will split at the first dim of hierarchy. + self.position_ids = flow.arange( + max_sequence_length, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ).unsqueeze(0) + + if num_tokentypes > 0: + self.tokentype_embeddings = Embedding( + num_tokentypes, hidden_size, init_method=init_method, amp_enabled=amp_enabled + ) + self.tokentype_ids = flow.zeros( + self.position_ids.size(), + dtype=flow.long, + sbp=self.position_ids.sbp, + placement=self.position_ids.placement, + ) + else: + self.tokentype_embeddings = None + + self.embedding_dropout = nn.Dropout(embedding_dropout_prob) + + def forward(self, input_ids, tokentype_ids=None, position_ids=None): + seq_length = input_ids.size()[1] + + word_embeddings = self.vocab_embeddings(input_ids) + if position_ids is None: + # Change position_ids sbp sign: [B, B] -> [S(0), B] + position_ids = ( + self.position_ids[:, :seq_length].expand_as(input_ids).to_global(sbp=input_ids.sbp) + ) + position_embeddings = self.position_embeddings(position_ids) + embeddings = word_embeddings + position_embeddings + + if self.tokentype_embeddings is not None: + if tokentype_ids is None: + tokentype_ids = ( + self.tokentype_ids[:, :seq_length] + .expand_as(input_ids) + .to_global(sbp=input_ids.sbp) + ) + embeddings = embeddings + self.tokentype_embeddings(tokentype_ids) + + embeddings = self.embedding_dropout(embeddings) + return embeddings + + def word_embeddings(self): + return self.vocab_embeddings.weight + + +class BertLMPredictionHead(nn.Module): + def __init__(self, hidden_size, init_method): + super().__init__() + self.dense = Linear( + hidden_size, + hidden_size, + bias=True, + parallel="data", + init_method=init_method, + layer_idx=-1, + ) + self.activation_func = build_activation("gelu") + self.layernorm = LayerNorm((hidden_size,), layer_idx=-1) + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.activation_func(hidden_states) + hidden_states = hidden_states.to_global( + grad_sbp=dist.get_nd_sbp([flow.sbp.split(0), flow.sbp.split(2)]) + ) + + # NOTE(l1aoxingyu): hidden_states shape is [B, S, H] whose sbp sign: [S(0), S(2)] + # Change from [S(0), S(2)] -> [S(0), B] because layernorm cannot get inputs with sbp S(2) + hidden_states = hidden_states.to_global( + sbp=dist.get_nd_sbp([flow.sbp.split(0), flow.sbp.broadcast]) + ) + hidden_states = self.layernorm(hidden_states) + return hidden_states + + +class BertPooler(nn.Module): + """Pooler layer. + + Pool hidden states of the first token and + add a linear transformation followed by a tanh. + + Args: + hidden_size: hidden state feature dimension + """ + + def __init__(self, hidden_size, init_method): + super().__init__() + self.dense = Linear( + hidden_size, + hidden_size, + bias=True, + parallel="col", + init_method=init_method, + layer_idx=-1, + ) + self.activation_func = build_activation("tanh") + + def forward(self, hidden_states): + """Just "pool" the model by simply taking the [CLS] token corresponding + to the first token.""" + # hidden_states: [bsz, seq_len, hidden_size] + select_token_tensor = hidden_states[:, 0, :] + pooled_output = self.dense(select_token_tensor) + pooled_output = self.activation_func(pooled_output) + return pooled_output + + +class BertLoss(nn.Module): + def __init__(self, add_binary_head): + super().__init__() + self.add_binary_head = add_binary_head + self.lm_loss = ParallelCrossEntropyLoss() + + def forward(self, lm_output, lm_labels, loss_mask, binary_logits, ns_labels): + lm_labels = lm_labels.to_global(placement=lm_output.placement) + loss_mask = loss_mask.to_global(placement=lm_output.placement) + binary_logits = binary_logits.to_global(placement=lm_output.placement) + ns_labels = ns_labels.to_global(placement=lm_output.placement) + lm_loss = self.lm_loss(lm_output, lm_labels) + loss_mask = loss_mask.float() + # Change loss_mask.sum() sbp sign from [P, B] -> [B, B] + # because (lm_loss * loss_mask) / loss_mask.sum() cannot accept P / P + denominator = ( + loss_mask.sum().to_global(sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast])) + + 1e-7 + ) + masked_lm_loss = flow.sum(lm_loss.view(-1) * loss_mask.view(-1)) / denominator + # NOTE(l1aoxingyu): Change lm loss sbp sign [P, P] -> [P, B] to add with sop loss + # whose sbp sign: [P, B] + masked_lm_loss = masked_lm_loss.to_global( + sbp=dist.get_nd_sbp([flow.sbp.partial_sum, flow.sbp.broadcast]) + ) + + loss_dict = {"lm_loss": masked_lm_loss} + + if self.add_binary_head: + sop_loss = flow._C.cross_entropy( + binary_logits, ns_labels, ignore_index=-1, reduction="none" + ).mean() + loss_dict["sop_loss"] = sop_loss + return loss_dict + + +class BertModel(nn.Module): + """The bare Bert Model transformer outputting raw hidden-states without + any specific head on top. + + Args: + vocab_size (int): The size of vocabulary file. + hidden_size (int): The size of hidden states. + hidden_layers (int): The number of ``TransformerLayer`` in encoder. + num_attention_heads (int): + The number of attention heads for each attention layer of ``TransformerLayer``. + intermediate_size (int): + The size of intermediate layer in feed-forward network for each ``TransformerLayer``. + hidden_dropout_prob (float, optional): + The dropout ratio for the output for each TransformerLayer. Defaults to 0.0. + attention_probs_dropout_prob (float, optional): + The dropout ratio for the output of each attention layer in ``TransformerLayer``. + Defaults to 0.0. + max_position_embeddings (int): + Max sequence length of input, defines the shape of Position Embeddings + in ``BertEmbedding``. + num_tokentypes (int, optional): + Number of segment token indices. Defaults to 2. + add_pooling_layer (bool, optional): + Whether or not averaging or pooling the sequence of hidden-states for the + whole input sequence. Defaults to ``True``. + initializer_range (float, optional): + Sigma of the normal distribution in the initialization method. Defaults to 0.02. + layernorm_epsilon (float, optional): + The epsilon of LayerNorm layer. Defaults to 1e-5. + bias_gelu_fusion (bool, optional): + Whether or not to fuse the computing of bias and gelu. Defaults to ``False``. + bias_dropout_fusion (bool, optional): + Whether or not to fuse the computing of dropout and bias. Defaults to ``False``. + scale_mask_softmax_fusion (bool, optional): + Whether to fuse the computing of mask and softmax in attention layers. + Defaults to ``False``. + apply_query_key_layer_scaling (bool, optional): + Whether or not to use layer index related scaling in computing attention scores. + If ``True``, the scaling factor equals to sqrt(d) * (layer_index + 1). + Defaults to ``True``. + apply_residual_post_layernorm (bool, optional): + If set ``True``, use original BERT residual connection ordering otherwise use Megatron + BERT residual connection which is more stable when scaling model size introduced in + https://arxiv.org/pdf/1909.08053.pdf. + Default: ``False``. + amp_enabled (bool, optional): + Whether or not to set fp16 for embedding weight in T5 model. Defaults to ``False``. + """ + + @configurable + def __init__( + self, + vocab_size, + hidden_size, + hidden_layers, + num_attention_heads, + intermediate_size, + hidden_dropout_prob, + attention_probs_dropout_prob, + max_position_embeddings, + num_tokentypes=2, + add_pooling_layer=True, + initializer_range=0.02, + layernorm_eps=1e-12, + bias_gelu_fusion=True, + bias_dropout_fusion=True, + scale_mask_softmax_fusion=True, + apply_query_key_layer_scaling=True, + apply_residual_post_layernorm=False, + amp_enabled=False, + ): + super().__init__() + init_method = init_method_normal(initializer_range) + scaled_init_method = scaled_init_method_normal(initializer_range, hidden_layers) + + # Embeddings + self.embeddings = BertEmbeddings( + vocab_size, + hidden_size, + max_position_embeddings, + hidden_dropout_prob, + num_tokentypes, + init_method, + amp_enabled, + ) + + # Mask generation + self.extended_attn_mask = BertExtendedAttnMask() + + # Encoders + self.encoders = nn.ModuleList( + [ + TransformerLayer( + hidden_size, + intermediate_size, + num_attention_heads, + attention_dropout_prob=attention_probs_dropout_prob, + output_dropout_prob=hidden_dropout_prob, + layernorm_epsilon=layernorm_eps, + bias_gelu_fusion=bias_gelu_fusion, + bias_dropout_fusion=bias_dropout_fusion, + scale_mask_softmax_fusion=scale_mask_softmax_fusion, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + init_method=init_method, + output_layer_init_method=scaled_init_method, + apply_residual_post_layernorm=apply_residual_post_layernorm, + attn_mask_type=AttnMaskType.padding, # bert mask type + layer_idx=i, + ) + for i in range(hidden_layers) + ] + ) + self.final_layernorm = LayerNorm((hidden_size,), eps=layernorm_eps, layer_idx=-1) + + self.pooler = BertPooler(hidden_size, init_method) if add_pooling_layer else None + + @classmethod + def from_config(cls, cfg): + return { + "vocab_size": cfg.vocab_size, + "hidden_size": cfg.hidden_size, + "hidden_layers": cfg.hidden_layers, + "num_attention_heads": cfg.num_attention_heads, + "intermediate_size": cfg.intermediate_size, + "hidden_dropout_prob": cfg.hidden_dropout_prob, + "attention_probs_dropout_prob": cfg.attention_probs_dropout_prob, + "max_position_embeddings": cfg.max_position_embeddings, + "num_tokentypes": cfg.num_tokentypes, + "add_pooling_layer": cfg.add_pooling_layer, + "initializer_range": cfg.initializer_range, + "layernorm_eps": cfg.layernorm_eps, + "bias_gelu_fusion": cfg.bias_gelu_fusion, + "bias_dropout_fusion": cfg.bias_dropout_fusion, + "scale_mask_softmax_fusion": cfg.scale_mask_softmax_fusion, + "apply_query_key_layer_scaling": cfg.apply_query_key_layer_scaling, + "apply_residual_post_layernorm": cfg.apply_residual_post_layernorm, + "amp_enabled": cfg.amp_enabled, + } + + def forward(self, input_ids, attention_mask, tokentype_ids=None): + """ + + Args: + input_ids (flow.LongTensor): Indices of input sequence tokens in vocabulary. + attention_mask (flow.BoolTensor): Mask to avoid performing attention + on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + tokentype_ids (flow.LongTensor, optional): Segment token indices to indicate first and + second portions of the inputs. Indices are selected in `[0, 1]`. Defaults to None. + """ + extended_attention_mask = self.extended_attn_mask(attention_mask) + embedding_output = self.embeddings(input_ids, tokentype_ids) + + hidden_states = embedding_output + for layer in self.encoders: + hidden_states = layer(hidden_states, extended_attention_mask) + encoder_output = self.final_layernorm(hidden_states) + pooled_output = self.pooler(encoder_output) if self.pooler is not None else None + return encoder_output, pooled_output + + def word_embeddings_weight(self): + return self.embeddings.word_embeddings() + + +class BertPreTrainingHeads(nn.Module): + def __init__(self, vocab_size, hidden_size, init_method, add_binary_head=True): + super().__init__() + self.predictions = BertLMPredictionHead(hidden_size, init_method) + self.seq_relationship = Linear( + hidden_size, + 2, + bias=True, + parallel="data", + init_method=init_method, + layer_idx=-1, + ) + self.lm_logits = LMLogits(vocab_size, bias=True) + self.loss_func = BertLoss(add_binary_head) + + def forward( + self, + sequence_output, + pooled_output, + word_embeddings_weight, + ns_labels, + lm_labels, + loss_mask, + ): + prediction_scores = self.predictions(sequence_output) + seq_relationship_score = self.seq_relationship(pooled_output) + prediction_scores = self.lm_logits(prediction_scores, word_embeddings_weight) + + if lm_labels is not None: + return self.loss_func( + prediction_scores, lm_labels, loss_mask, seq_relationship_score, ns_labels + ) + return { + "prediction_scores": prediction_scores, + "seq_relationship_score": seq_relationship_score, + } + + +class BertForPreTraining(nn.Module): + """Bert Model with two heads on top as done during the pretraining: a + `masked language modeling` head and a `next sentence prediction (classification)` head. + """ + + def __init__(self, cfg): + super().__init__() + self.bert = BertModel(cfg) + self.cls_head = BertPreTrainingHeads( + cfg.vocab_size, + cfg.hidden_size, + init_method_normal(cfg.initializer_range), + cfg.add_binary_head, + ) + + def forward( + self, + input_ids, + attention_mask, + tokentype_ids=None, + ns_labels=None, + lm_labels=None, + loss_mask=None, + ): + """ + + Args: + input_ids (flow.LongTensor): Indices of input sequence tokens in vocabulary. + attention_mask (flow.BoolTensor): Mask to avoid performing attention on + padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + tokentype_ids (flow.LongTensor, optional): Segment token indices to indicate first + and second portions of the inputs. Indices are selected in `[0, 1]`. + Defaults to None. + ns_labels (flow.LongTensor, optional): Labels for computing the next sequence prediction + (classification) loss. Input should be a sequence pair (see `input_ids` docstring). + Indices should be in `[0, 1]`: + + - 0 indicates sequence B is a continuation of sequence A, + - 1 indicates sequence B is a random sequence. + + lm_labels (flow.LongTensor, optional): Labels for computing the masked + language modeling loss. Indices should be in `[-1, 0, ..., config.vocab_size]`. + loss_mask (flow.BoolTensor, optional): Mask to avoid performing loss computing + on ignored tokens. Tokens with indices set to `-1` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` + """ + + input_ids = input_ids.to_global(placement=dist.get_layer_placement(0)) + attention_mask = attention_mask.to_global(placement=dist.get_layer_placement(0)) + tokentype_ids = tokentype_ids.to_global(placement=dist.get_layer_placement(0)) + outputs = self.bert(input_ids, attention_mask, tokentype_ids) + sequence_output, pooled_output = outputs[:2] + + return self.cls_head( + sequence_output, + pooled_output, + self.bert.word_embeddings_weight(), + ns_labels, + lm_labels, + loss_mask, + ) + + @staticmethod + def set_pipeline_stage_id(model): + dist_utils = dist.get_dist_util() + + # Set pipeline parallelism stage_id + if hasattr(model.bert.final_layernorm, "config"): + # Old API in OneFlow 0.8 + for module_block in model.modules(): + # module.origin can get the original module + if isinstance(module_block.origin, BertEmbeddings): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, BertExtendedAttnMask): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, TransformerLayer): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.origin, BertPooler): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + elif isinstance(module_block.origin, BertPreTrainingHeads): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + # Set the last layernorm stage id + model.bert.final_layernorm.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + else: + for module_block in model.modules(): + if isinstance(module_block.to(nn.Module), BertEmbeddings): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), BertExtendedAttnMask): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.to(nn.Module), BertPooler): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + elif isinstance(module_block.to(nn.Module), BertPreTrainingHeads): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + # Set the last layernorm stage id + model.bert.final_layernorm.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + +class BertForClassification(nn.Module): + def __init__(self, cfg): + super().__init__() + self.cfg = cfg + self.num_labels = cfg.num_labels + + self.bert = BertModel(cfg) + self.classifier = Linear( + cfg.hidden_size, + cfg.num_labels, + bias=True, + parallel="row", + init_method=init_method_normal(cfg.initializer_range), + layer_idx=-1, + ) + classifier_dropout = ( + cfg.classifier_dropout + if cfg.classifier_dropout is not None + else cfg.hidden_dropout_prob + ) + self.dropout = nn.Dropout(classifier_dropout) + + def forward(self, input_ids, attention_mask, tokentype_ids=None, labels=None, **kwargs): + labels = labels if labels is not None else kwargs.get("ns_labels") + outputs = self.bert(input_ids, attention_mask, tokentype_ids) + pooled_output = outputs[1] + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + if labels is not None: + loss_fct = nn.CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + loss = loss.to_global(sbp=dist.get_nd_sbp([flow.sbp.partial_sum, flow.sbp.broadcast])) + return {"cls_loss": loss} + else: + return {"logits": logits} diff --git a/libai/models/build.py b/libai/models/build.py new file mode 100644 index 0000000000000000000000000000000000000000..019780eba9e3dcfbdb71a2ab677d97155f7a5804 --- /dev/null +++ b/libai/models/build.py @@ -0,0 +1,52 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libai.config import instantiate, try_get_key + + +def build_model(cfg): + """Build the whole model architecture, defined by ``cfg.model``. + Note that it does not load any weights from ``cfg``. + """ + model = instantiate(cfg) + return model + + +def build_graph(cfg, model, optimizer=None, lr_scheduler=None, is_train=False): + """Build the `nn.Graph`, defined by ``cfg.graph``.""" + auto_parallel_conf = try_get_key(cfg, "graph.auto_parallel", default=None) + if is_train: + # Set train graph + assert optimizer is not None, "optimizer must be set for train graph" + assert lr_scheduler is not None, "lr_scheduler must be set for train graph" + graph = cfg.graph.train_graph + graph.model = model + graph.optimizer = optimizer + graph.lr_scheduler = lr_scheduler + graph.fp16 = try_get_key(cfg, "train.amp.enabled", default=False) + graph.activation_checkpoint = try_get_key( + cfg, "train.activation_checkpoint.enabled", default=False + ) + graph.zero_optim = try_get_key(cfg, "train.zero_optimization.enabled", default=False) + graph.zero_stage = try_get_key(cfg, "train.zero_optimization.stage", default=1) + graph.grad_acc_steps = try_get_key(cfg, "train.num_accumulation_steps", default=1) + graph.auto_parallel_conf = auto_parallel_conf + return instantiate(graph) + else: + # Set eval graph + graph = cfg.graph.eval_graph + graph.model = model + graph.auto_parallel_conf = auto_parallel_conf + return instantiate(graph) diff --git a/libai/models/gpt_model.py b/libai/models/gpt_model.py new file mode 100644 index 0000000000000000000000000000000000000000..27f6bc8e9e4256f21f54115f8ce79e66fc31ec2d --- /dev/null +++ b/libai/models/gpt_model.py @@ -0,0 +1,399 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +from oneflow import nn +from oneflow.nn import init + +from libai.config import configurable +from libai.layers import ( + Embedding, + LayerNorm, + LMLogits, + ParallelCrossEntropyLoss, + TransformerLayer, + VocabEmbedding, +) +from libai.layers.attention import AttnMaskType +from libai.utils import distributed as dist + +from .utils import init_method_normal, scaled_init_method_normal + + +class CasualMask(nn.Module): + """ + Create a casual mask and combine it with the padding mask. + It will be used in gpt model and T5 decoder. + When in T5 decoder, the argument `layer_idx` should be set to first decoder layer index. + """ + + def __init__(self, max_positions=1024, *, layer_idx=0): + super().__init__() + self.mask = flow.tril( + flow.ones( + (max_positions, max_positions), + dtype=flow.int8, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + ) + + def forward(self, input_ids, past_length=0, attention_mask=None): + bsz, tgt_len = input_ids.size() + casual_mask = self.mask[:tgt_len, :tgt_len] + if past_length > 0: + # in case past_key_values are used, we need to add a prefix ones mask to casual mask + casual_mask = flow.cat( + [flow.ones(tgt_len, past_length, dtype=flow.int8), casual_mask], dim=-1 + ) + casual_mask = ( + casual_mask.unsqueeze(0).unsqueeze(1).expand(bsz, 1, tgt_len, tgt_len + past_length) + ) + casual_mask = casual_mask.to_global(sbp=input_ids.sbp) + if attention_mask is not None: + assert attention_mask.dim() == 4, "please extend the attention mask first" + casual_mask = casual_mask * attention_mask + return casual_mask + + +class GPTModel(nn.Module): + """GPT-2 language model. The output of the forward method is logits. + + Args: + hidden_layers (int): The number of ``TransformerLayer`` in the gpt model. + vocab_size (int): The size of vocabulary file. + hidden_size (int): The size of hidden states. + ffn_hidden_size (int): + The size of intermediate layer in feed-forward network for each ``TransformerLayer``. + num_attention_heads (int): + The number of attention heads for each attention layer of ``TransformerLayer``. + max_seq_length (int, optional): + Max sequence length of input, defines the shape of Position Embeddings in GPTEmebedding. + Defaults to 1024. + embedding_dropout_prob (float, optional): + The dropout ratio for the output of GPTEmbedding Layer. Defaults to 0.0. + attention_dropout_prob (float, optional): + The dropout ratio for the output of each attention layer in ``TransformerLayer``. + Defaults to 0.0. + output_dropout_prob (float, optional): + The dropout ratio for the output for each TransformerLayer. Defaults to 0.0. + layernorm_epsilon (float, optional): + The epsilon of LayerNorm layer. Defaults to 1e-5. + initializer_range (float, optional): + Sigma of the normal distribution in the initialization method. Defaults to 0.02. + use_scaled_init_for_output_weights (bool, optional): Defaults to ``True``. + bias_gelu_fusion (bool, optional): + Whether or not to fuse the computing of bias and gelu. Defaults to ``False``. + bias_dropout_fusion (bool, optional): + Whether or not to fuse the computing of dropout and bias. Defaults to ``False``. + scale_mask_softmax_fusion (bool, optional): + Whether to fuse the computing of mask and softmax in attention layers. + Defaults to ``False``. + apply_query_key_layer_scaling (bool, optional): + Whether or not to use layer index related scaling in computing attention scores. + If ``True``, the scaling factor equals to sqrt(d) * (layer_index + 1). + Defaults to ``False``. + apply_residual_post_layernorm (bool, optional): + If set ``True``, use original BERT residual connection ordering otherwise use Megatron + BERT residual connection which is more stable when scaling model size introduced in + https://arxiv.org/pdf/1909.08053.pdf. + Default: ``False``. + amp_enabled (bool, optional): + Whether or not to set fp16 for embedding weight in T5 model. Defaults to ``False``. + """ + + @configurable + def __init__( + self, + hidden_layers, + vocab_size, + hidden_size, + ffn_hidden_size, + num_attention_heads, + max_seq_length=1024, + embedding_dropout_prob=0.0, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + layernorm_epsilon=1e-5, + initializer_range=0.02, + use_scaled_init_for_output_weights=True, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + apply_residual_post_layernorm=False, + amp_enabled=False, + ): + super().__init__() + init_method = init_method_normal(sigma=initializer_range) + if use_scaled_init_for_output_weights: + output_layer_init_method = scaled_init_method_normal(initializer_range, hidden_layers) + else: + output_layer_init_method = init_method + + self.embeddings = GPTEmbedding( + vocab_size, + hidden_size, + max_seq_length, + init_method=init_method, + embedding_dropout_prob=embedding_dropout_prob, + amp_enabled=amp_enabled, + ) + + self.transformer = Transformer( + hidden_layers, + hidden_size, + ffn_hidden_size, + num_attention_heads, + attention_dropout_prob=attention_dropout_prob, + output_dropout_prob=output_dropout_prob, + layernorm_epsilon=layernorm_epsilon, + init_method=init_method, + output_layer_init_method=output_layer_init_method, + bias_gelu_fusion=bias_gelu_fusion, + bias_dropout_fusion=bias_dropout_fusion, + scale_mask_softmax_fusion=scale_mask_softmax_fusion, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + apply_residual_post_layernorm=apply_residual_post_layernorm, + ) + + self.lm_head = LMLogits(vocab_size, bias=False) + + @classmethod + def from_config(cls, cfg): + return { + "hidden_layers": cfg.hidden_layers, + "vocab_size": cfg.vocab_size, + "hidden_size": cfg.hidden_size, + "ffn_hidden_size": cfg.ffn_hidden_size, + "num_attention_heads": cfg.num_attention_heads, + "max_seq_length": cfg.max_seq_length, + "embedding_dropout_prob": cfg.embedding_dropout_prob, + "attention_dropout_prob": cfg.attention_dropout_prob, + "output_dropout_prob": cfg.output_dropout_prob, + "layernorm_epsilon": cfg.layernorm_epsilon, + "initializer_range": cfg.initializer_range, + "use_scaled_init_for_output_weights": cfg.use_scaled_init_for_output_weights, + "bias_gelu_fusion": cfg.bias_gelu_fusion, + "bias_dropout_fusion": cfg.bias_dropout_fusion, + "scale_mask_softmax_fusion": cfg.scale_mask_softmax_fusion, + "apply_query_key_layer_scaling": cfg.apply_query_key_layer_scaling, + "apply_residual_post_layernorm": cfg.apply_residual_post_layernorm, + "amp_enabled": cfg.amp_enabled, + } + + def forward(self, input_ids): + """ + + Args: + input_ids (flow.LongTensor): Indices of input sequence tokens in vocabulary. + + Returns: + flow.Tensor: logits + """ + + input_ids = input_ids.to_global(placement=dist.get_layer_placement(0)) + input_embeds = self.embeddings(input_ids, 0) + + transformer_output = self.transformer(input_embeds, attention_mask=None) + + output = self.lm_head(transformer_output, self.embeddings.token_embeddings.weight) + + return output + + +class GPTEmbedding(nn.Module): + def __init__( + self, + vocab_size, + hidden_size, + max_seq_length, + init_method=init.xavier_normal_, + embedding_dropout_prob=0.0, + amp_enabled=False, + ): + super().__init__() + self.token_embeddings = VocabEmbedding( + vocab_size, hidden_size, init_method=init_method, amp_enabled=amp_enabled + ) + self.position_embeddings = Embedding( + max_seq_length, hidden_size, init_method=init_method, amp_enabled=amp_enabled + ) + self.dropout = nn.Dropout(embedding_dropout_prob) + + self.position_ids = flow.arange( + max_seq_length, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ).unsqueeze(0) + + def forward(self, input_ids, past_length=0): + bsz, seq_length = input_ids.size() + + position_ids = self.position_ids[:, past_length : past_length + seq_length] + position_ids = position_ids.expand_as(input_ids).to_global(sbp=input_ids.sbp) + + token_embeds = self.token_embeddings(input_ids) + position_embeds = self.position_embeddings(position_ids) + input_embeds = token_embeds + position_embeds + input_embeds = self.dropout(input_embeds) + return input_embeds + + +class Transformer(nn.Module): + def __init__( + self, + hidden_layers, + hidden_size, + ffn_hidden_size, + num_attention_heads, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + layernorm_epsilon=1e-5, + init_method=init.xavier_normal_, + output_layer_init_method=None, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + apply_residual_post_layernorm=False, + ): + super().__init__() + self.hidden_layers = hidden_layers + + def build_layer(layer_number): + return TransformerLayer( + hidden_size, + ffn_hidden_size, + num_attention_heads, + attention_dropout_prob=attention_dropout_prob, + output_dropout_prob=output_dropout_prob, + layernorm_epsilon=layernorm_epsilon, + init_method=init_method, + output_layer_init_method=output_layer_init_method, + bias_gelu_fusion=bias_gelu_fusion, + bias_dropout_fusion=bias_dropout_fusion, + scale_mask_softmax_fusion=scale_mask_softmax_fusion, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + apply_residual_post_layernorm=apply_residual_post_layernorm, + attn_mask_type=AttnMaskType.causal, + layer_idx=layer_number, + ) + + self.layers = nn.ModuleList([build_layer(i) for i in range(self.hidden_layers)]) + self.layernorm_f = LayerNorm(hidden_size, eps=layernorm_epsilon, layer_idx=-1) + + def forward(self, hidden_states, attention_mask): + # hidden_states shape: (batch_size, seq_length, hidden_size) + # sbp: [S(0), B] + for i, layer in enumerate(self.layers): + hidden_states = layer(hidden_states, attention_mask) + + output = self.layernorm_f(hidden_states) + + return output + + +class GPTLoss(nn.Module): + def __init__(self) -> None: + super().__init__() + self.lm_loss = ParallelCrossEntropyLoss() + + def forward(self, logits, lm_labels): + lm_loss = self.lm_loss(logits, lm_labels) + lm_loss = lm_loss.mean() + return {"lm_loss": lm_loss} + + +class GPTForPreTraining(nn.Module): + """ + GPT Model with classification head on top. + """ + + def __init__(self, cfg) -> None: + super().__init__() + self.GPT_model = GPTModel(cfg) + self.loss_func = GPTLoss() + + def forward( + self, + input_ids, + labels=None, + ): + """ + + Args: + input_ids (flow.LongTensor): Indices of input sequence tokens in vocabulary. + labels (flow.LongTensor, optional): Labels for computing language modeling loss. + None for evaluating. Defaults to None. + + Returns: + dict: + A dict containing :code:`loss_value` or :code:`logits` + depending on training or evaluation. + :code:`{"masked_lm_loss": loss_value}` when training, + :code:`{"prediction_scores": logits}` when evaluating. + """ + logits = self.GPT_model(input_ids) + if labels is not None: + lm_loss = self.loss_func(logits, labels) + return lm_loss + else: + return {"prediction_scores": logits} + + @staticmethod + def set_pipeline_stage_id(model: nn.Module): + dist_utils = dist.get_dist_util() + + if hasattr(model.GPT_model.transformer.layernorm_f, "config"): + # Old API in OneFlow 0.8 + for module_block in model.modules(): + if isinstance(module_block.origin, (GPTEmbedding, CasualMask)): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, TransformerLayer): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.origin, (LMLogits, GPTLoss)): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + model.GPT_model.transformer.layernorm_f.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + else: + for module_block in model.modules(): + if isinstance(module_block.to(nn.Module), (GPTEmbedding, CasualMask)): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.to(nn.Module), (LMLogits, GPTLoss)): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + model.GPT_model.transformer.layernorm_f.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) diff --git a/libai/models/resmlp.py b/libai/models/resmlp.py new file mode 100644 index 0000000000000000000000000000000000000000..3143cd6d4a620cf1356967796afe220f4300237a --- /dev/null +++ b/libai/models/resmlp.py @@ -0,0 +1,295 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# -------------------------------------------------------- +# ResMLP Model +# References: +# resmlp: https://github.com/facebookresearch/deit/blob/main/resmlp_models.py +# -------------------------------------------------------- + +import oneflow as flow +import oneflow.nn as nn +from flowvision.layers.weight_init import trunc_normal_ + +import libai.utils.distributed as dist +from libai.config import configurable +from libai.layers import MLP, DropPath, LayerNorm, Linear, PatchEmbedding + + +class Affine(nn.Module): + def __init__(self, dim, *, layer_idx=0): + super().__init__() + self.alpha = nn.Parameter( + flow.ones( + dim, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + ) + self.beta = nn.Parameter( + flow.zeros( + dim, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ), + ) + + self.layer_idx = layer_idx + + def forward(self, x): + x = x.to_global(placement=dist.get_layer_placement(self.layer_idx)) + return self.alpha * x + self.beta + + +class layers_scale_mlp_blocks(nn.Module): + def __init__( + self, dim, drop=0.0, drop_path=0.0, init_values=1e-4, num_patches=196, *, layer_idx=0 + ): + super().__init__() + self.norm1 = Affine(dim, layer_idx=layer_idx) + self.attn = Linear(num_patches, num_patches, layer_idx=layer_idx) + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.norm2 = Affine(dim, layer_idx=layer_idx) + self.mlp = MLP(hidden_size=dim, ffn_hidden_size=int(4.0 * dim), layer_idx=layer_idx) + self.gamma_1 = nn.Parameter( + init_values + * flow.ones( + dim, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(layer_idx), + ), + requires_grad=True, + ) + self.gamma_2 = nn.Parameter( + init_values + * flow.ones( + dim, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(layer_idx), + ), + requires_grad=True, + ) + + self.layer_idx = layer_idx + + def forward(self, x): + x = x.to_global(placement=dist.get_layer_placement(self.layer_idx)) + x = x + self.drop_path( + self.gamma_1 * self.attn(self.norm1(x).transpose(1, 2)).transpose(1, 2) + ) + x = x + self.drop_path(self.gamma_2 * self.mlp(self.norm2(x))) + return x + + +class ResMLP(nn.Module): + """ResMLP in LiBai. + + LiBai's implementation of: + `ResMLP: Feedforward networks for image classification with data-efficient training + `_ + + Args: + img_size (int, tuple(int)): input image size + patch_size (int, tuple(int)): patch size + in_chans (int): number of input channels + embed_dim (int): embedding dimension + depth (int): depth of transformer + drop_rate (float): dropout rate + drop_path_rate (float): stochastic depth rate + init_scale (float): the layer scale ratio + num_classes (int): number of classes for classification head + loss_func (callable, optional): loss function for computing the total loss + between logits and labels + + """ + + @configurable + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + depth=12, + drop_rate=0.0, + drop_path_rate=0.0, + init_scale=1e-4, + num_classes=1000, + loss_func=None, + ): + super().__init__() + + self.num_classes = num_classes + self.num_features = self.embed_dim = embed_dim + + self.patch_embed = PatchEmbedding( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + ) + + num_patches = self.patch_embed.num_patches + dpr = [drop_path_rate for i in range(depth)] # stochastic depth decay rule + + self.blocks = nn.ModuleList( + [ + layers_scale_mlp_blocks( + dim=embed_dim, + drop=drop_rate, + drop_path=dpr[i], + init_values=init_scale, + num_patches=num_patches, + layer_idx=i, + ) + for i in range(depth) + ] + ) + + self.norm = Affine(embed_dim, layer_idx=-1) + self.head = ( + Linear(embed_dim, num_classes, layer_idx=-1) if num_classes > 0 else nn.Identity() + ) + + # loss func + self.loss_func = nn.CrossEntropyLoss() if loss_func is None else loss_func + + # weight init + self.apply(self._init_weights) + + @classmethod + def from_config(cls, cfg): + return { + "img_size": cfg.img_size, + "patch_size": cfg.patch_size, + "in_chans": cfg.in_chans, + "embed_dim": cfg.embed_dim, + "depth": cfg.depth, + "drop_rate": cfg.drop_rate, + "drop_path_rate": cfg.drop_path_rate, + "init_scale": cfg.init_scale, + "num_classes": cfg.num_classes, + "loss_func": cfg.loss_func, + } + + def _init_weights(self, m): + if isinstance(m, Linear): + trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def forward_features(self, x): + x = self.patch_embed(x) + + # layer scale mlp blocks + for i, blk in enumerate(self.blocks): + x = blk(x) + + return x + + def forward_head(self, x): + B = x.shape[0] + x = self.norm(x) + x = x.mean(dim=1).reshape(B, 1, -1) + return self.head(x[:, 0]) + + def forward(self, images, labels=None): + """ + + Args: + images (flow.Tensor): training samples. + labels (flow.LongTensor, optional): training targets + + Returns: + dict: + A dict containing :code:`loss_value` or :code:`logits` + depending on training or evaluation mode. + :code:`{"losses": loss_value}` when training, + :code:`{"prediction_scores": logits}` when evaluating. + """ + x = self.forward_features(images) + x = self.forward_head(x) + + if labels is not None and self.training: + losses = self.loss_func(x, labels) + return {"losses": losses} + else: + return {"prediction_scores": x} + + @staticmethod + def set_pipeline_stage_id(model): + dist_utils = dist.get_dist_util() + + # Set pipeline parallelism stage_id + if hasattr(model.loss_func, "config"): + # Old API in OneFlow 0.8 + for module_block in model.modules(): + # module.origin can get the original module + if isinstance(module_block.origin, PatchEmbedding): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, layers_scale_mlp_blocks): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + + # Set norm and head stage id + model.norm.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.head.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.loss_func.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + else: + for module_block in model.modules(): + if isinstance(module_block.to(nn.Module), PatchEmbedding): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), layers_scale_mlp_blocks): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + + # Set norm and head stage id + model.norm.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.head.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.loss_func.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + @staticmethod + def set_activation_checkpoint(model): + for module_block in model.modules(): + if hasattr(module_block, "origin"): + # Old API in OneFlow 0.8 + if isinstance(module_block.origin, layers_scale_mlp_blocks): + module_block.config.activation_checkpointing = True + else: + if isinstance(module_block.to(nn.Module), layers_scale_mlp_blocks): + module_block.to(nn.graph.GraphModule).activation_checkpointing = True diff --git a/libai/models/roberta_model.py b/libai/models/roberta_model.py new file mode 100644 index 0000000000000000000000000000000000000000..0e61a34755f5f1a22049a9ef7f6be9f0ce46b6d8 --- /dev/null +++ b/libai/models/roberta_model.py @@ -0,0 +1,512 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +from oneflow import nn + +from libai.config import configurable +from libai.layers import ( + Embedding, + LayerNorm, + Linear, + LMLogits, + ParallelCrossEntropyLoss, + TransformerLayer, + VocabEmbedding, + build_activation, +) +from libai.utils import distributed as dist + +from .bert_model import BertEmbeddings, BertExtendedAttnMask, BertModel, BertPooler +from .utils import init_method_normal + + +class RobertaExtendedAttnMask(BertExtendedAttnMask): + """ + Same as BertExtendedAttnMask. + """ + + +class RobertaEmbeddings(BertEmbeddings): + """ + Same as BertEmbeddings with a tiny tweak for vocab_embeddings and position_embeddings. + """ + + def __init__( + self, + vocab_size, + hidden_size, + max_sequence_length, + embedding_dropout_prob, + num_tokentypes=0, + pad_token_id=1, + init_method=nn.init.xavier_normal_, + amp_enabled=False, + ): + super().__init__( + vocab_size, + hidden_size, + max_sequence_length, + embedding_dropout_prob, + num_tokentypes=num_tokentypes, + init_method=init_method, + amp_enabled=amp_enabled, + ) + self.pad_token_id = pad_token_id + self.vocab_embeddings = VocabEmbedding( + vocab_size, + hidden_size, + init_method=init_method, + amp_enabled=amp_enabled, + padding_idx=pad_token_id, + ) + self.position_embeddings = Embedding( + max_sequence_length, + hidden_size, + init_method=init_method, + amp_enabled=amp_enabled, + padding_idx=pad_token_id, + ) + + if num_tokentypes > 0: + self.tokentype_embeddings = Embedding( + num_tokentypes, hidden_size, init_method=init_method, amp_enabled=amp_enabled + ) + self.tokentype_ids = flow.zeros( + 1, + max_sequence_length, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + else: + self.tokentype_embeddings = None + + def forward(self, input_ids, tokentype_ids=None, position_ids=None): + seq_length = input_ids.size()[1] + + word_embeddings = self.vocab_embeddings(input_ids) + + if position_ids is None: + position_ids = self.create_position_ids_from_input_ids(input_ids, self.pad_token_id) + position_embeddings = self.position_embeddings(position_ids) + embeddings = word_embeddings + position_embeddings + + if self.tokentype_embeddings is not None: + if tokentype_ids is None: + tokentype_ids = ( + self.tokentype_ids[:, :seq_length] + .expand_as(input_ids) + .to_global(sbp=input_ids.sbp) + ) + embeddings = embeddings + self.tokentype_embeddings(tokentype_ids) + embeddings = self.embedding_dropout(embeddings) + return embeddings + + def create_position_ids_from_input_ids(self, input_ids, pad_token_id): + mask = input_ids.ne(pad_token_id).int() + position_ids = (flow.cumsum(mask, dim=1).type_as(mask)) * mask + pad_token_id + position_ids = position_ids.to_global(sbp=input_ids.sbp, placement=input_ids.placement) + return position_ids + + +class RobertaPooler(BertPooler): + """ + Same as BertPooler. + """ + + +class RobertaLoss(nn.Module): + def __init__(self): + super().__init__() + self.lm_loss = ParallelCrossEntropyLoss() + + def forward(self, lm_output, lm_labels, loss_mask): + lm_labels = lm_labels.to_global(placement=lm_output.placement) + loss_mask = loss_mask.to_global(placement=lm_output.placement) + lm_loss = self.lm_loss(lm_output, lm_labels) + loss_mask = loss_mask.float() + # Change loss_mask.sum() sbp sign from [P, B] -> [B, B] + # because (lm_loss * loss_mask) / loss_mask.sum() cannot accept P / P + denominator = loss_mask.sum().to_global( + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]) + ) + masked_lm_loss = flow.sum(lm_loss.view(-1) * loss_mask.view(-1)) / denominator + masked_lm_loss = masked_lm_loss.to_global( + sbp=dist.get_nd_sbp([flow.sbp.partial_sum, flow.sbp.broadcast]) + ) + loss_dict = {"lm_loss": masked_lm_loss} + return loss_dict + + +class RobertaModel(BertModel): + """The bare Roberta Model transformer outputting raw hidden-states without + any specific head on top. + + Args: + vocab_size (int): + The size of vocabulary file. + hidden_size (int): + The size of hidden states. + hidden_layers (int): + The number of ``TransformerLayer`` in encoder. + num_attention_heads (int): + The number of attention heads for each attention layer of ``TransformerLayer``. + intermediate_size (int): + The size of intermediate layer in feed-forward network for each + ``TransformerLayer``. + hidden_dropout_prob (float, optional): + The dropout ratio for the output for each TransformerLayer. Defaults to 0.0. + attention_probs_dropout_prob (float, optional): + The dropout ratio for the output of each attention layer in ``TransformerLayer``. + Defaults to 0.0. + max_position_embeddings (int): + Max sequence length of input, defines the shape of Position Embeddings + in ``RobertaEmbeddings``. + type_vocab_size (int, optional): + Number of segment token indices. Defaults to 2. + add_pooling_layer (bool, optional): + Whether or not averaging or pooling the sequence of hidden-states for the + whole input sequence. Defaults to ``True``. + initializer_range (float, optional): + Sigma of the normal distribution in the initialization method. Defaults to 0.02. + layer_norm_eps (float, optional): + The epsilon of LayerNorm layer. Defaults to 1e-5. + pad_token_id (int, optional): + The token id used for padding. Defaults to 1. + bias_gelu_fusion (bool, optional): + Whether or not to fuse the computing of bias and gelu. Defaults to ``False``. + bias_dropout_fusion (bool, optional): + Whether or not to fuse the computing of dropout and bias. Defaults to ``False``. + scale_mask_softmax_fusion (bool, optional): + Whether to fuse the computing of mask and softmax in attention layers. + Defaults to ``False``. + apply_query_key_layer_scaling (bool, optional): + Whether or not to use layer index related scaling in computing attention scores. + If ``True``, the scaling factor equals to sqrt(d) * (layer_index + 1). + Defaults to ``True``. + apply_residual_post_layernorm (bool, optional): + If set ``True``, use original BERT(Roberta) residual connection ordering + otherwise use Megatron BERT residual connection which is more stable + when scaling model size introduced in https://arxiv.org/pdf/1909.08053.pdf. + Default: ``False``. + amp_enabled (bool, optional): + Whether or not to set fp16 for embedding weight in T5 model. Defaults to ``False``. + """ + + @configurable + def __init__( + self, + vocab_size, + hidden_size, + hidden_layers, + num_attention_heads, + intermediate_size, + hidden_dropout_prob, + attention_probs_dropout_prob, + max_position_embeddings, + num_tokentypes=2, + add_pooling_layer=True, + initializer_range=0.02, + layernorm_eps=1e-12, + pad_token_id=1, + bias_gelu_fusion=True, + bias_dropout_fusion=True, + scale_mask_softmax_fusion=True, + apply_query_key_layer_scaling=True, + apply_residual_post_layernorm=False, + amp_enabled=False, + ): + super().__init__( + vocab_size, + hidden_size, + hidden_layers, + num_attention_heads, + intermediate_size, + hidden_dropout_prob, + attention_probs_dropout_prob, + max_position_embeddings, + num_tokentypes=num_tokentypes, + add_pooling_layer=add_pooling_layer, + initializer_range=initializer_range, + layernorm_eps=layernorm_eps, + bias_gelu_fusion=bias_gelu_fusion, + bias_dropout_fusion=bias_dropout_fusion, + scale_mask_softmax_fusion=scale_mask_softmax_fusion, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + apply_residual_post_layernorm=apply_residual_post_layernorm, + amp_enabled=amp_enabled, + ) + + init_method = init_method_normal(initializer_range) + + # Embeddings + self.embeddings = RobertaEmbeddings( + vocab_size, + hidden_size, + max_position_embeddings, + hidden_dropout_prob, + num_tokentypes, + pad_token_id, + init_method, + amp_enabled, + ) + + # Mask generation + self.extended_attn_mask = RobertaExtendedAttnMask() + self.pooler = RobertaPooler(hidden_size, init_method) if add_pooling_layer else None + + @classmethod + def from_config(cls, cfg): + return { + "vocab_size": cfg.vocab_size, + "hidden_size": cfg.hidden_size, + "hidden_layers": cfg.hidden_layers, + "num_attention_heads": cfg.num_attention_heads, + "intermediate_size": cfg.intermediate_size, + "hidden_dropout_prob": cfg.hidden_dropout_prob, + "attention_probs_dropout_prob": cfg.attention_probs_dropout_prob, + "max_position_embeddings": cfg.max_position_embeddings, + "num_tokentypes": cfg.num_tokentypes, + "add_pooling_layer": cfg.add_pooling_layer, + "initializer_range": cfg.initializer_range, + "layernorm_eps": cfg.layernorm_eps, + "pad_token_id": cfg.pad_token_id, + "bias_gelu_fusion": cfg.bias_gelu_fusion, + "bias_dropout_fusion": cfg.bias_dropout_fusion, + "scale_mask_softmax_fusion": cfg.scale_mask_softmax_fusion, + "apply_query_key_layer_scaling": cfg.apply_query_key_layer_scaling, + "apply_residual_post_layernorm": cfg.apply_residual_post_layernorm, + "amp_enabled": cfg.amp_enabled, + } + + +class RobertaLMHead(nn.Module): + def __init__(self, vocab_size, hidden_size, init_method, layer_norm_eps): + super().__init__() + self.dense = Linear( + hidden_size, + hidden_size, + bias=True, + parallel="data", + init_method=init_method, + layer_idx=-1, + ) + self.activation_func = build_activation("gelu") + self.layernorm = LayerNorm((hidden_size,), eps=layer_norm_eps, layer_idx=-1) + + # NOTE(xzp): LMLogits as a decoder:nn.Linear(hidden_size, vocab_size), + # it shares the roberta.word_embeddings.weight + self.lm_logits = LMLogits(vocab_size, bias=True) + + def forward(self, hidden_states, word_embeddings_weight): + hidden_states = self.dense(hidden_states) + hidden_states = self.activation_func(hidden_states) + hidden_states = hidden_states.to_global( + sbp=dist.get_nd_sbp([flow.sbp.split(0), flow.sbp.broadcast]) + ) + hidden_states = self.layernorm(hidden_states) + hidden_states = self.lm_logits(hidden_states, word_embeddings_weight) + return hidden_states + + +class RobertaPreTrainedModel(nn.Module): + @staticmethod + def set_pipeline_stage_id(model): + dist_utils = dist.get_dist_util() + + # Set pipeline parallelism stage_id + if hasattr(model.roberta.final_layernorm, "config"): + # Old API in OneFlow 0.8 + for module_block in model.modules(): + # module.origin can get the original module + if isinstance(module_block.origin, RobertaEmbeddings): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, RobertaExtendedAttnMask): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, TransformerLayer): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + # `add_pooling_layer` in RobertaForMaskedLM and RobertaForCausalLM. + # default to False. + elif isinstance(module_block.origin, RobertaPooler): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + elif isinstance(module_block.origin, RobertaLMHead): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + # Set the last layernorm stage id + model.roberta.final_layernorm.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + else: + for module_block in model.modules(): + # module.origin can get the original module + if isinstance(module_block.to(nn.Module), RobertaEmbeddings): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), RobertaExtendedAttnMask): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + # `add_pooling_layer` in RobertaForMaskedLM and RobertaForCausalLM. + # default to False. + elif isinstance(module_block.to(nn.Module), RobertaPooler): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + elif isinstance(module_block.to(nn.Module), RobertaLMHead): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + # Set the last layernorm stage id + model.roberta.final_layernorm.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + +class RobertaForPreTraining(RobertaPreTrainedModel): + def __init__(self, cfg): + super().__init__() + + cfg.add_pooling_layer = False + self.roberta = RobertaModel(cfg) + self.lm_head = RobertaLMHead( + cfg.vocab_size, + cfg.hidden_size, + init_method_normal(cfg.initializer_range), + cfg.layernorm_eps, + ) + self.loss_fc = RobertaLoss() + + def forward( + self, + input_ids, + attention_mask, + tokentype_ids=None, + lm_labels=None, + loss_mask=None, + ): + """ + + Args: + input_ids (flow.LongTensor): Indices of input sequence tokens in vocabulary. + attention_mask (flow.BoolTensor): Mask to avoid performing attention on + padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + tokentype_ids (flow.LongTensor, optional): Segment token indices to indicate first + and second portions of the inputs. Indices are selected in `[0, 1]`. + Defaults to None. + labels (flow.LongTensor, optional): Labels for computing the masked + language modeling loss. Indices should be in `[-1, 0, ..., config.vocab_size]`. + Defaults to None. + loss_mask (flow.BoolTensor, optional): Mask to avoid performing loss computing + on ignored tokens. Tokens with indices set to `-1` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + Defaults to None. + """ + input_ids = input_ids.to_global(placement=dist.get_layer_placement(0)) + attention_mask = attention_mask.to_global(placement=dist.get_layer_placement(0)) + tokentype_ids = tokentype_ids.to_global(placement=dist.get_layer_placement(0)) + + outputs = self.roberta(input_ids, attention_mask, tokentype_ids=tokentype_ids) + sequence_output = outputs[0] + prediction_scores = self.lm_head(sequence_output, self.roberta.word_embeddings_weight()) + + if lm_labels is not None: + return self.loss_fc(prediction_scores, lm_labels, loss_mask) + + return {"prediction_scores": prediction_scores} + + +class RobertaForCausalLM(RobertaPreTrainedModel): + def __init__(self, cfg): + super().__init__() + + cfg.add_pooling_layer = False + self.roberta = RobertaModel(cfg) + self.lm_head = RobertaLMHead( + cfg.vocab_size, + cfg.hidden_size, + init_method_normal(cfg.initializer_range), + cfg.layernorm_eps, + ) + self.loss_fc = RobertaLoss() + + def forward( + self, + input_ids, + attention_mask, + tokentype_ids=None, + position_ids=None, + labels=None, + loss_mask=None, + ): + """ + + Args: + input_ids (flow.LongTensor): Indices of input sequence tokens in vocabulary. + attention_mask (flow.BoolTensor): Mask to avoid performing attention on + padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + tokentype_ids (flow.LongTensor, optional): Segment token indices to indicate first + and second portions of the inputs. Indices are selected in `[0, 1]`. + Defaults to None. + position_ids (flow.LongTensor, optional): Indices of positions of each input sequence + tokens in the position embeddings. Defaults to None. + labels (flow.LongTensor, optional): Labels for computing the masked + language modeling loss. Indices should be in `[-1, 0, ..., config.vocab_size]`. + Defaults to None. + loss_mask (flow.BoolTensor, optional): Mask to avoid performing loss computing + on ignored tokens. Tokens with indices set to `-1` are ignored (masked), the + loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + Defaults to None. + """ + outputs = self.roberta(input_ids, attention_mask, position_ids, tokentype_ids) + sequence_output = outputs[0] + prediction_scores = self.lm_head(sequence_output, self.roberta.word_embeddings_weight()) + + if labels is not None: + # next-token prediction task, shift prediction_scores and labels by one. + shifted_prediction_scores = prediction_scores[:, :-1, :].contiguous() + shifted_prediction_scores = shifted_prediction_scores.to_global( + sbp=prediction_scores.sbp + ) + shifted_labels = labels[:, 1:].contiguous() + shifted_labels = shifted_labels.to_global(sbp=shifted_labels.sbp) + lm_loss = self.loss_fc(shifted_prediction_scores, shifted_labels, loss_mask) + return {"lm_loss": lm_loss} + + return {"prediction_scores": prediction_scores} diff --git a/libai/models/swin_transformer.py b/libai/models/swin_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..91fbce78bb66cc9dcd78432cc61f2f94312533a5 --- /dev/null +++ b/libai/models/swin_transformer.py @@ -0,0 +1,772 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +import oneflow.nn as nn +from flowvision.layers import trunc_normal_ +from flowvision.models import to_2tuple + +from libai.config.config import configurable +from libai.layers import MLP, DropPath, LayerNorm, Linear +from libai.utils import distributed as dist + + +def window_partition(x, window_size): + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class WindowAttention(nn.Module): + """Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query,key,value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__( + self, + dim, + window_size, + num_heads, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + fused_bias_add_dropout=False, + layer_idx=0, + ): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim ** -0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + flow.zeros( + (2 * window_size[0] - 1) * (2 * window_size[1] - 1), + num_heads, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + ) # 2*Wh-1 * 2*Ww-1, nH + trunc_normal_(self.relative_position_bias_table, std=0.02) + + # get pair-wise relative position index for each token inside the window + coords_h = flow.arange(self.window_size[0]) + coords_w = flow.arange(self.window_size[1]) + coords = flow.stack(flow.meshgrid(*[coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = flow.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] = ( + relative_coords[:, :, 0] + self.window_size[0] - 1 + ) # shift to start from 0 + relative_coords[:, :, 1] = relative_coords[:, :, 1] + self.window_size[1] - 1 + relative_coords[:, :, 0] = relative_coords[:, :, 0] * (2 * self.window_size[1] - 1) + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer( + "relative_position_index", + relative_position_index.to_global( + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ), + ) + + self.qkv = Linear(dim, dim * 3, bias=qkv_bias, layer_idx=layer_idx) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = Linear(dim, dim, layer_idx=layer_idx) + self.proj_drop = nn.Dropout(proj_drop) + self.softmax = nn.Softmax(dim=-1) + self.fused_bias_add_dropout = fused_bias_add_dropout + self.p = proj_drop + + def forward(self, x, mask): + """ + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = ( + self.qkv(x) + .reshape(B_, N, 3, self.num_heads, C // self.num_heads) + .permute(2, 0, 3, 1, 4) + ) + q, k, v = qkv[0], qkv[1], qkv[2] + + q = q * self.scale + # attn = flow.matmul(q, k.transpose(-2, -1)) + attn = flow.matmul(q, k, transpose_b=True) + + relative_position_bias = self.relative_position_bias_table[ + self.relative_position_index.view(-1) + ].view( + self.window_size[0] * self.window_size[1], + self.window_size[0] * self.window_size[1], + -1, + ) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1 + ).contiguous() # nH, Wh*Ww, Wh*Ww + unsqueeze_relative_position_bias = relative_position_bias.unsqueeze(0) + attn = attn + unsqueeze_relative_position_bias + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = flow.matmul(attn, v).transpose(1, 2).reshape(B_, N, C) + if self.fused_bias_add_dropout: + x = flow._C.matmul(x, self.proj.weight, transpose_a=False, transpose_b=True) + x = flow._C.fused_bias_add_dropout(x, self.proj.bias, p=self.p, axis=2) + else: + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class SwinTransformerBlock(nn.Module): + """Swin Transformer Block. + Args: + dim (int): Number of input channels. + input_resolution (tuple[int]): Input resulotion. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: libai.layers.LayerNorm + """ + + def __init__( + self, + dim, + input_resolution, + num_heads, + window_size=7, + shift_size=0, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + act_layer=nn.GELU, + norm_layer=LayerNorm, + layer_idx=0, + ): + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + self.layer_idx = layer_idx + if min(self.input_resolution) <= self.window_size: + # if window size is larger than input resolution, we don't partition windows + self.shift_size = 0 + self.window_size = min(self.input_resolution) + assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size" + + self.norm1 = norm_layer(dim, layer_idx=layer_idx) + self.attn = WindowAttention( + dim, + window_size=to_2tuple(self.window_size), + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + fused_bias_add_dropout=True, + layer_idx=layer_idx, + ) + + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.norm2 = norm_layer(dim, layer_idx=layer_idx) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = MLP( + hidden_size=dim, + ffn_hidden_size=mlp_hidden_dim, + output_dropout_prob=drop, + bias_gelu_fusion=True, + bias_dropout_fusion=True, + layer_idx=layer_idx, + ) + + if self.shift_size > 0: + # calculate attention mask for SW-MSA + H, W = self.input_resolution + img_mask = flow.zeros((1, H, W, 1)) # 1 H W 1 + h_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + w_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt = cnt + 1 + + mask_windows = window_partition( + img_mask, self.window_size + ) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill( + attn_mask == 0, float(0.0) + ) + attn_mask = attn_mask.to_global( + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + else: + attn_mask = None + + self.register_buffer("attn_mask", attn_mask) + + def forward(self, x): + H, W = self.input_resolution + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # cyclic shift + if self.shift_size > 0: + shifted_x = flow.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + else: + shifted_x = x + + # partition windows + x_windows = window_partition( + shifted_x, self.window_size + ) # nW*B, window_size, window_size, C + x_windows = x_windows.view( + -1, self.window_size * self.window_size, C + ) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn(x_windows, self.attn_mask) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, H, W) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = flow.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2)) + else: + x = shifted_x + x = x.view(B, H * W, C) + + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x + + +class PatchMerging(nn.Module): + """Patch Merging Layer. + Args: + input_resolution (tuple[int]): Resolution of input feature. + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: libai.layers.LayerNorm + """ + + def __init__(self, input_resolution, dim, norm_layer=LayerNorm, layer_idx=0): + super().__init__() + self.input_resolution = input_resolution + self.dim = dim + self.reduction = Linear(4 * dim, 2 * dim, bias=False, layer_idx=layer_idx) + self.norm = norm_layer(4 * dim, layer_idx=layer_idx) + self.layer_idx = layer_idx + + def forward(self, x): + """ + x: B, H*W, C + """ + H, W = self.input_resolution + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + assert H % 2 == 0 and W % 2 == 0, f"x size ({H}*{W}) are not even." + + x = x.view(B, H, W, C) + + x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C + x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C + x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C + x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C + x = flow.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C + x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C + + x = self.norm(x) + x = self.reduction(x) + + return x + + +class PatchEmbed(nn.Module): + """Image to Patch Embedding + Args: + img_size (int): Image size. Default: 224. + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__( + self, img_size=224, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None, layer_idx=0 + ): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + patches_resolution = [ + img_size[0] // patch_size[0], + img_size[1] // patch_size[1], + ] + self.img_size = img_size + self.patch_size = patch_size + self.patches_resolution = patches_resolution + self.num_patches = patches_resolution[0] * patches_resolution[1] + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.proj = nn.Conv2d( + in_chans, embed_dim, kernel_size=patch_size, stride=patch_size + ).to_global( + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + if norm_layer is not None: + self.norm = norm_layer(embed_dim, layer_idx=layer_idx) + else: + self.norm = None + + def forward(self, x): + B, C, H, W = x.shape + # FIXME look at relaxing size constraints + assert ( + H == self.img_size[0] and W == self.img_size[1] + ), f"Input image size ({H}*{W}) doesn't match model({self.img_size[0]}*{self.img_size[1]})." + x = self.proj(x).flatten(2).transpose(1, 2) # B Ph*Pw C + if self.norm is not None: + x = self.norm(x) + return x + + +class BasicLayer(nn.Module): + """A basic Swin Transformer layer for one stage. + Args: + dim (int): Number of input channels. + input_resolution (tuple[int]): Input resolution. + depth (int): Number of blocks. + num_heads (int): Number of attention heads. + window_size (int): Local window size. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: libai.layers.LayerNorm + downsample (nn.Module | None, optional): Downsample at the end of the layer. Default: None + """ + + def __init__( + self, + dim, + input_resolution, + depth, + num_heads, + window_size, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + norm_layer=LayerNorm, + downsample=None, + layer_id_offset=0, + ): + + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.depth = depth + self.layer_id_offset = layer_id_offset + # build blocks + self.blocks = nn.ModuleList( + [ + SwinTransformerBlock( + dim=dim, + input_resolution=input_resolution, + num_heads=num_heads, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, + layer_idx=layer_id_offset + i, + ) + for i in range(depth) + ] + ) + + # patch merging layer + if downsample is not None: + self.downsample = downsample( + input_resolution, + dim=dim, + norm_layer=norm_layer, + layer_idx=layer_id_offset + depth - 1, + ) + else: + self.downsample = None + + def forward(self, x): + layer_idx = self.layer_id_offset + for i in range(len(self.blocks)): + x = x.to_global(placement=dist.get_layer_placement(layer_idx)) + x = self.blocks[i](x) + layer_idx += 1 + if self.downsample is not None: + x = self.downsample(x) + return x + + +class SwinTransformer(nn.Module): + """Swin Transformer in LiBai. + + LiBai implement of: + `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows + `_ + + Args: + img_size (int, tuple(int)): Input image size. Default 224 + patch_size (int, tuple(int)): Patch size. Default: 4 + in_chans (int): Number of input image channels. Default: 3 + num_classes (int): Number of classes for classification head. Default: 1000 + embed_dim (int): Patch embedding dimension. Default: 96 + depths (tuple(int)): Depth of each Swin Transformer layer. + num_heads (tuple(int)): Number of attention heads in different layers. + window_size (int): Window size. Default: 7 + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4 + qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. Default: None + drop_rate (float): Dropout rate. Default: 0 + attn_drop_rate (float): Attention dropout rate. Default: 0 + drop_path_rate (float): Stochastic depth rate. Default: 0.1 + norm_layer (nn.Module): Normalization layer. Default: libai.layers.LayerNorm. + ape (bool): If True, add absolute position embedding to the patch embedding. Default: False + patch_norm (bool): If True, add normalization after patch embedding. Default: True + loss_func (callable, optional): Loss function for computing the total loss + between logits and labels + """ + + @configurable + def __init__( + self, + img_size=224, + patch_size=4, + in_chans=3, + num_classes=1000, + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.1, + norm_layer=LayerNorm, + ape=False, + patch_norm=True, + loss_func=None, + **kwargs, + ): + super().__init__() + + self.num_classes = num_classes + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.ape = ape + self.patch_norm = patch_norm + self.num_features = int(embed_dim * 2 ** (self.num_layers - 1)) + self.mlp_ratio = mlp_ratio + + # split image into non-overlapping patches + self.patch_embed = PatchEmbed( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None, + layer_idx=0, + ) + num_patches = self.patch_embed.num_patches + patches_resolution = self.patch_embed.patches_resolution + self.patches_resolution = patches_resolution + + # absolute position embedding + if self.ape: + self.absolute_pos_embed = nn.Parameter( + flow.zeros(1, num_patches, embed_dim), + placement=dist.get_layer_placement(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + trunc_normal_(self.absolute_pos_embed, std=0.02) + + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth + dpr = [ + x.item() for x in flow.linspace(0, drop_path_rate, sum(depths)) + ] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + layer_id_offset = 0 + for i_layer in range(self.num_layers): + layer = BasicLayer( + dim=int(embed_dim * 2 ** i_layer), + input_resolution=( + patches_resolution[0] // (2 ** i_layer), + patches_resolution[1] // (2 ** i_layer), + ), + depth=depths[i_layer], + num_heads=num_heads[i_layer], + window_size=window_size, + mlp_ratio=self.mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]) : sum(depths[: i_layer + 1])], + norm_layer=norm_layer, + downsample=PatchMerging if (i_layer < self.num_layers - 1) else None, + layer_id_offset=layer_id_offset, + ) + layer_id_offset += depths[i_layer] + self.layers.append(layer) + + self.norm = norm_layer(self.num_features, layer_idx=-1) + self.avgpool = nn.AdaptiveAvgPool1d(1) + self.head = ( + Linear(self.num_features, num_classes, layer_idx=-1) + if num_classes > 0 + else nn.Identity() + ) + + # Loss func + self.loss_func = nn.CrossEntropyLoss() if loss_func is None else loss_func + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, Linear): + trunc_normal_(m.weight, std=0.02) + if isinstance(m, Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @classmethod + def from_config(cls, cfg): + return { + "img_size": cfg.img_size, + "patch_size": cfg.patch_size, + "in_chans": cfg.in_chans, + "num_classes": cfg.num_classes, + "embed_dim": cfg.embed_dim, + "depths": cfg.depths, + "num_heads": cfg.num_heads, + "window_size": cfg.window_size, + "mlp_ratio": cfg.mlp_ratio, + "qkv_bias": cfg.qkv_bias, + "qk_scale": cfg.qk_scale, + "drop_rate": cfg.drop_rate, + "drop_path_rate": cfg.drop_path_rate, + "ape": cfg.ape, + "patch_norm": cfg.patch_norm, + "loss_func": cfg.loss_func, + } + + def forward_features(self, x): + x = self.patch_embed(x) + if self.ape: + x = x + self.absolute_pos_embed + x = self.pos_drop(x) + + for layer in self.layers: + x = layer(x) + + x = self.norm(x) # B L C + x = self.avgpool(x.transpose(1, 2)) # B C 1 + x = flow.flatten(x, 1) + return x + + def forward(self, images, labels=None): + """ + + Args: + images (flow.Tensor): training samples. + labels (flow.LongTensor, optional): training targets + + Returns: + dict: + A dict containing :code:`loss_value` or :code:`logits` + depending on training or evaluation mode. + :code:`{"losses": loss_value}` when training, + :code:`{"prediction_scores": logits}` when evaluating. + """ + x = self.forward_features(images) + x = self.head(x) + + if labels is not None and self.training: + losses = self.loss_func(x, labels) + return {"losses": losses} + else: + return {"prediction_scores": x} + + @staticmethod + def set_pipeline_stage_id(model): + dist_utils = dist.get_dist_util() + + # Set pipeline parallelism stage_id + if hasattr(model.patch_embed, "config"): + # Old API in OneFlow 0.8 + model.patch_embed.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + model.pos_drop.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + + for module_block in model.modules(): + if isinstance(module_block.origin, SwinTransformerBlock): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.origin, PatchMerging): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + + model.norm.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.head.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.avgpool.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.loss_func.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + else: + model.patch_embed.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + model.pos_drop.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + + for module_block in model.modules(): + if isinstance(module_block.to(nn.Module), SwinTransformerBlock): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.to(nn.Module), PatchMerging): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + + model.norm.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.head.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.avgpool.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.loss_func.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + @staticmethod + def set_activation_checkpoint(model): + for module_block in model.modules(): + if hasattr(module_block, "origin"): + # Old API in OneFlow 0.8 + if isinstance(module_block.origin, SwinTransformerBlock): + module_block.config.activation_checkpointing = True + else: + if isinstance(module_block.to(nn.Module), SwinTransformerBlock): + module_block.to(flow.nn.graph.GraphModule).activation_checkpointing = True diff --git a/libai/models/swin_transformer_v2.py b/libai/models/swin_transformer_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..f41e6b02cf411b38f3adfb9f090e0e9604306db2 --- /dev/null +++ b/libai/models/swin_transformer_v2.py @@ -0,0 +1,900 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import oneflow as flow +import oneflow.nn as nn +import oneflow.nn.functional as F +from flowvision.layers import trunc_normal_ +from flowvision.models import to_2tuple + +from libai.config.config import configurable +from libai.layers import MLP, DropPath, LayerNorm, Linear +from libai.utils import distributed as dist + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class WindowAttention(nn.Module): + r"""Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. + Default: True + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + pretrained_window_size (tuple[int]): The height and width of the window in pre-training. + """ + + def __init__( + self, + dim, + window_size, + num_heads, + qkv_bias=True, + attn_drop=0.0, + proj_drop=0.0, + pretrained_window_size=[0, 0], + fused_bias_add_dropout=False, + layer_idx=0, + ): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.pretrained_window_size = pretrained_window_size + self.fused_bias_add_dropout = fused_bias_add_dropout + self.num_heads = num_heads + self.layer_idx = layer_idx + self.p = proj_drop + self.logit_scale = nn.Parameter( + flow.log( + 10 + * flow.ones( + 1, + num_heads, + 1, + 1, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + ), + requires_grad=True, + ) + + # NOTE: generate meta network, using mlp to generate continuous relative position bias + self.cpb_mlp = nn.Sequential( + Linear(2, 512, bias=True, layer_idx=layer_idx), + nn.ReLU(inplace=True), + Linear(512, num_heads, bias=False, layer_idx=layer_idx), + ).to_global( + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + + # NOTE: get relative_coords_table + relative_coords_h = flow.arange( + -(self.window_size[0] - 1), self.window_size[0], dtype=flow.float32 + ) + relative_coords_w = flow.arange( + -(self.window_size[1] - 1), self.window_size[1], dtype=flow.float32 + ) + relative_coords_table = ( + flow.stack(flow.meshgrid(*[relative_coords_h, relative_coords_w])) + .permute(1, 2, 0) + .contiguous() + .unsqueeze(0) + ) # 1, 2*Wh-1, 2*Ww-1, 2 + + # NOTE: For any relative coordinate, constrain it to -8~8 (window size) + if pretrained_window_size[0] > 0: + relative_coords_table[:, :, :, 0] = relative_coords_table[:, :, :, 0] / ( + pretrained_window_size[0] - 1 + ) + relative_coords_table[:, :, :, 1] = relative_coords_table[:, :, :, 1] / ( + pretrained_window_size[1] - 1 + ) + else: + relative_coords_table[:, :, :, 0] = relative_coords_table[:, :, :, 0] / ( + self.window_size[0] - 1 + ) + relative_coords_table[:, :, :, 1] = relative_coords_table[:, :, :, 1] / ( + self.window_size[1] - 1 + ) + relative_coords_table = relative_coords_table * 8 + # NOTE: y=sign(x)*log(|x|+1) + relative_coords_table = ( + flow.sign(relative_coords_table) + * flow.log2(flow.abs(relative_coords_table) + 1.0) + / math.log2(8.0) + ) + self.register_buffer( + "relative_coords_table", + relative_coords_table.to_global( + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ), + ) + + # NOTE: get pair-wise relative position index for each token inside the window + coords_h = flow.arange(self.window_size[0]) + coords_w = flow.arange(self.window_size[1]) + coords = flow.stack(flow.meshgrid(*[coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = flow.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] = ( + relative_coords[:, :, 0] + self.window_size[0] - 1 + ) # shift to start from 0 + relative_coords[:, :, 1] = relative_coords[:, :, 1] + self.window_size[1] - 1 + relative_coords[:, :, 0] = relative_coords[:, :, 0] * (2 * self.window_size[1] - 1) + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer( + "relative_position_index", + relative_position_index.to_global( + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ), + ) + + self.qkv = Linear(dim, dim * 3, bias=False, layer_idx=layer_idx) + if qkv_bias: + self.q_bias = nn.Parameter( + flow.zeros( + dim, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + ) + self.v_bias = nn.Parameter( + flow.zeros( + dim, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + ) + else: + self.q_bias = None + self.v_bias = None + self.attn_drop = nn.Dropout(attn_drop) + self.proj = Linear(dim, dim, layer_idx=layer_idx) + self.proj_drop = nn.Dropout(proj_drop) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """ + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + + qkv_bias = None + if self.q_bias is not None: + qkv_bias = flow.concat( + [ + self.q_bias, + flow.zeros( + self.v_bias.shape, + requires_grad=False, + placement=dist.get_layer_placement( + self.layer_idx, device_type=self.v_bias.placement.type + ), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ), + self.v_bias, + ], + dim=0, + ) + qkv = self.qkv(x) + qkv_bias.unsqueeze(0).unsqueeze(0) + qkv = qkv.reshape(B_, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] + + # NOTE: cosine attention + attn = F.normalize(q, dim=-1) @ F.normalize(k, dim=-1).transpose(-2, -1) + + # NOTE: a learnable scalar + logit_scale = flow.clamp(self.logit_scale, min=-1e6, max=math.log(1.0 / 0.01)).exp() + attn = attn * logit_scale + + # NOTE: use relative_coords_table and meta network to generate relative_position_bias + relative_position_bias_table = self.cpb_mlp(self.relative_coords_table).view( + -1, self.num_heads + ) + relative_position_bias = relative_position_bias_table[ + self.relative_position_index.view(-1) + ].view( + self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1 + ) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1 + ).contiguous() # nH, Wh*Ww, Wh*Ww + + # NOTE: constrained to a range of -16~16 + relative_position_bias = 16 * flow.sigmoid(relative_position_bias).unsqueeze(0) + attn = attn + relative_position_bias + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + if self.fused_bias_add_dropout: + x = flow._C.matmul(x, self.proj.weight, transpose_a=False, transpose_b=True) + x = flow._C.fused_bias_add_dropout(x, self.proj.bias, p=self.p, axis=2) + else: + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class SwinTransformerBlock(nn.Module): + r"""Swin Transformer Block. + + Args: + dim (int): Number of input channels. + input_resolution (tuple[int]): Input resulotion. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + pretrained_window_size (int): Window size in pre-training. + """ + + def __init__( + self, + dim, + input_resolution, + num_heads, + window_size=7, + shift_size=0, + mlp_ratio=4.0, + qkv_bias=True, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + act_layer=nn.GELU, + norm_layer=LayerNorm, + pretrained_window_size=0, + layer_idx=0, + ): + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + self.layer_idx = layer_idx + if min(self.input_resolution) <= self.window_size: + # if window size is larger than input resolution, we don't partition windows + self.shift_size = 0 + self.window_size = min(self.input_resolution) + assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size" + + self.norm1 = norm_layer(dim, layer_idx=layer_idx) + + self.attn = WindowAttention( + dim, + window_size=to_2tuple(self.window_size), + num_heads=num_heads, + qkv_bias=qkv_bias, + attn_drop=attn_drop, + proj_drop=drop, + pretrained_window_size=to_2tuple(pretrained_window_size), + fused_bias_add_dropout=True, + layer_idx=layer_idx, + ) + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + self.norm2 = norm_layer(dim, layer_idx=layer_idx) + + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = MLP( + hidden_size=dim, + ffn_hidden_size=mlp_hidden_dim, + output_dropout_prob=drop, + bias_gelu_fusion=True, + bias_dropout_fusion=True, + layer_idx=layer_idx, + ) + + if self.shift_size > 0: + # calculate attention mask for SW-MSA + H, W = self.input_resolution + img_mask = flow.zeros((1, H, W, 1)) # 1 H W 1 + h_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + w_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt = cnt + 1 + + mask_windows = window_partition( + img_mask, self.window_size + ) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = ( + attn_mask.masked_fill(attn_mask != 0, float(-100.0)) + .masked_fill(attn_mask == 0, float(0.0)) + .to_global( + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + ) + else: + attn_mask = None + + self.register_buffer("attn_mask", attn_mask) + + def forward(self, x): + H, W = self.input_resolution + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + + shortcut = x + x = x.view(B, H, W, C) + + # cyclic shift + if self.shift_size > 0: + shifted_x = flow.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + else: + shifted_x = x + + # partition windows + x_windows = window_partition( + shifted_x, self.window_size + ) # nW*B, window_size, window_size, C + x_windows = x_windows.view( + -1, self.window_size * self.window_size, C + ) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn(x_windows, mask=self.attn_mask) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, H, W) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = flow.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2)) + else: + x = shifted_x + x = x.view(B, H * W, C) + # NOTE: res-post-norm + x = shortcut + self.drop_path(self.norm1(x)) + + # NOTE: res-post-norm + x = x + self.drop_path(self.norm2(self.mlp(x))) + return x + + +class PatchMerging(nn.Module): + """Patch Merging Layer. + Args: + input_resolution (tuple[int]): Resolution of input feature. + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: libai.layers.LayerNorm + """ + + def __init__(self, input_resolution, dim, norm_layer=LayerNorm, layer_idx=0): + super().__init__() + self.input_resolution = input_resolution + self.dim = dim + self.reduction = Linear(4 * dim, 2 * dim, bias=False, layer_idx=layer_idx) + + # NOTE: swinv2-> 2*dim, swin-> 4*dim + self.norm = norm_layer(2 * dim, layer_idx=layer_idx) + self.layer_idx = layer_idx + + def forward(self, x): + """ + x: B, H*W, C + """ + H, W = self.input_resolution + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + assert H % 2 == 0 and W % 2 == 0, f"x size ({H}*{W}) are not even." + + x = x.view(B, H, W, C) + + x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C + x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C + x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C + x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C + x = flow.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C + x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C + + # NOTE: post-res-norm, a change that swin-v2 compared to swin + x = self.reduction(x) + x = self.norm(x) + + return x + + +class BasicLayer(nn.Module): + """A basic Swin Transformer layer for one stage. + Args: + dim (int): Number of input channels. + input_resolution (tuple[int]): Input resolution. + depth (int): Number of blocks. + num_heads (int): Number of attention heads. + window_size (int): Local window size. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. + Default: None + pretrained_window_size (int): Local window size in pre-training. + """ + + def __init__( + self, + dim, + input_resolution, + depth, + num_heads, + window_size, + mlp_ratio=4.0, + qkv_bias=True, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + norm_layer=LayerNorm, + downsample=None, + pretrained_window_size=0, + layer_id_offset=0, + ): + + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.depth = depth + self.layer_id_offset = layer_id_offset + # build blocks + self.blocks = nn.ModuleList( + [ + SwinTransformerBlock( + dim=dim, + input_resolution=input_resolution, + num_heads=num_heads, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, + pretrained_window_size=pretrained_window_size, + layer_idx=layer_id_offset + i, + ) + for i in range(depth) + ] + ) + + # patch merging layer + if downsample is not None: + self.downsample = downsample( + input_resolution, + dim=dim, + norm_layer=norm_layer, + layer_idx=layer_id_offset + depth - 1, + ) + else: + self.downsample = None + + def forward(self, x): + layer_idx = self.layer_id_offset + for blk in self.blocks: + x = x.to_global( + placement=dist.get_layer_placement(layer_idx, device_type=x.placement.type) + ) + x = blk(x) + layer_idx += 1 + if self.downsample is not None: + x = self.downsample(x) + return x + + def _init_respostnorm(self): + for blk in self.blocks: + nn.init.constant_(blk.norm1.bias, 0) + nn.init.constant_(blk.norm1.weight, 0) + nn.init.constant_(blk.norm2.bias, 0) + nn.init.constant_(blk.norm2.weight, 0) + + +class PatchEmbed(nn.Module): + r"""Image to Patch Embedding + Args: + img_size (int): Image size. Default: 224. + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__( + self, img_size=224, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None, layer_idx=0 + ): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + patches_resolution = [img_size[0] // patch_size[0], img_size[1] // patch_size[1]] + self.img_size = img_size + self.patch_size = patch_size + self.patches_resolution = patches_resolution + self.num_patches = patches_resolution[0] * patches_resolution[1] + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.proj = nn.Conv2d( + in_chans, embed_dim, kernel_size=patch_size, stride=patch_size + ).to_global( + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + B, C, H, W = x.shape + # FIXME look at relaxing size constraints + assert ( + H == self.img_size[0] and W == self.img_size[1] + ), f"Input image size ({H}*{W}) doesn't match model \ + ({self.img_size[0]}*{self.img_size[1]})." + x = self.proj(x).flatten(2).transpose(1, 2) # B Ph*Pw C + if self.norm is not None: + x = self.norm(x) + return x + + +class SwinTransformerV2(nn.Module): + r"""Swin Transformer + Args: + img_size (int | tuple(int)): Input image size. Default 224 + patch_size (int | tuple(int)): Patch size. Default: 4 + in_chans (int): Number of input image channels. Default: 3 + num_classes (int): Number of classes for classification head. Default: 1000 + embed_dim (int): Patch embedding dimension. Default: 96 + depths (tuple(int)): Depth of each Swin Transformer layer. + num_heads (tuple(int)): Number of attention heads in different layers. + window_size (int): Window size. Default: 7 + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4 + qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True + drop_rate (float): Dropout rate. Default: 0 + attn_drop_rate (float): Attention dropout rate. Default: 0 + drop_path_rate (float): Stochastic depth rate. Default: 0.1 + norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. + ape (bool): If True, add absolute position embedding to the patch embedding. Default: False + patch_norm (bool): If True, add normalization after patch embedding. Default: True + pretrained_window_sizes (tuple(int)): Pretrained window sizes of each layer. + """ + + @configurable + def __init__( + self, + img_size=224, + patch_size=4, + in_chans=3, + num_classes=1000, + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + mlp_ratio=4.0, + qkv_bias=True, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.1, + norm_layer=LayerNorm, + ape=False, + patch_norm=True, + pretrained_window_sizes=[0, 0, 0, 0], + loss_func=None, + ): + super().__init__() + + self.num_classes = num_classes + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.ape = ape + self.patch_norm = patch_norm + self.num_features = int(embed_dim * 2 ** (self.num_layers - 1)) + self.mlp_ratio = mlp_ratio + + # split image into non-overlapping patches + self.patch_embed = PatchEmbed( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None, + layer_idx=0, + ) + num_patches = self.patch_embed.num_patches + patches_resolution = self.patch_embed.patches_resolution + self.patches_resolution = patches_resolution + + # absolute position embedding + if self.ape: + self.absolute_pos_embed = nn.Parameter( + flow.zeros( + 1, + num_patches, + embed_dim, + placement=dist.get_layer_placement(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + ) + trunc_normal_(self.absolute_pos_embed, std=0.02) + + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth + dpr = [ + x.item() for x in flow.linspace(0, drop_path_rate, sum(depths)) + ] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + layer_id_offset = 0 + for i_layer in range(self.num_layers): + layer = BasicLayer( + dim=int(embed_dim * 2 ** i_layer), + input_resolution=( + patches_resolution[0] // (2 ** i_layer), + patches_resolution[1] // (2 ** i_layer), + ), + depth=depths[i_layer], + num_heads=num_heads[i_layer], + window_size=window_size, + mlp_ratio=self.mlp_ratio, + qkv_bias=qkv_bias, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]) : sum(depths[: i_layer + 1])], + norm_layer=norm_layer, + downsample=PatchMerging if (i_layer < self.num_layers - 1) else None, + pretrained_window_size=pretrained_window_sizes[i_layer], + layer_id_offset=layer_id_offset, + ) + layer_id_offset += depths[i_layer] + self.layers.append(layer) + + self.norm = norm_layer(self.num_features, layer_idx=-1) + self.avgpool = nn.AdaptiveAvgPool1d(1) + self.head = ( + Linear(self.num_features, num_classes, layer_idx=-1) + if num_classes > 0 + else nn.Identity() + ) + self.loss_func = nn.CrossEntropyLoss() if loss_func is None else loss_func + + self.apply(self._init_weights) + for bly in self.layers: + bly._init_respostnorm() + + def _init_weights(self, m): + if isinstance(m, Linear): + trunc_normal_(m.weight, std=0.02) + if isinstance(m, Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @classmethod + def from_config(cls, cfg): + return { + "img_size": cfg.img_size, + "patch_size": cfg.patch_size, + "in_chans": cfg.in_chans, + "num_classes": cfg.num_classes, + "embed_dim": cfg.embed_dim, + "depths": cfg.depths, + "num_heads": cfg.num_heads, + "window_size": cfg.window_size, + "mlp_ratio": cfg.mlp_ratio, + "qkv_bias": cfg.qkv_bias, + "drop_rate": cfg.drop_rate, + "drop_path_rate": cfg.drop_path_rate, + "ape": cfg.ape, + "patch_norm": cfg.patch_norm, + "pretrained_window_sizes": cfg.pretrained_window_sizes, + "loss_func": cfg.loss_func, + } + + def forward_features(self, x): + + x = self.patch_embed(x) + if self.ape: + x = x + self.absolute_pos_embed + x = self.pos_drop(x) + + for layer in self.layers: + x = layer(x) + + x = self.norm(x) # B L C + x = self.avgpool(x.transpose(1, 2)) # B C 1 + x = flow.flatten(x, 1) + return x + + def forward(self, images, labels=None): + """ + + Args: + images (flow.Tensor): training samples. + labels (flow.LongTensor, optional): training targets + + Returns: + dict: + A dict containing :code:`loss_value` or :code:`logits` + depending on training or evaluation mode. + :code:`{"losses": loss_value}` when training, + :code:`{"prediction_scores": logits}` when evaluating. + """ + x = self.forward_features(images) + x = self.head(x) + + if labels is not None and self.training: + losses = self.loss_func(x, labels) + return {"losses": losses} + else: + return {"prediction_scores": x} + + @staticmethod + def set_pipeline_stage_id(model): + dist_utils = dist.get_dist_util() + + # Set pipeline parallelism stage_id + if hasattr(model.patch_embed, "config"): + # Old API in OneFlow 0.8 + model.patch_embed.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + model.pos_drop.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + + for module_block in model.modules(): + if isinstance(module_block.origin, SwinTransformerBlock): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.origin, PatchMerging): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + + model.norm.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.head.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.avgpool.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.loss_func.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + else: + model.patch_embed.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + model.pos_drop.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + + for module_block in model.modules(): + if isinstance(module_block.to(nn.Module), SwinTransformerBlock): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.to(nn.Module), PatchMerging): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + + model.norm.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.head.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.avgpool.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.loss_func.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + @staticmethod + def set_activation_checkpoint(model): + for module_block in model.modules(): + if hasattr(module_block, "origin"): + # Old API in OneFlow 0.8 + if isinstance(module_block.origin, SwinTransformerBlock): + module_block.config.activation_checkpointing = True + else: + if isinstance(module_block.to(nn.Module), SwinTransformerBlock): + module_block.to(flow.nn.graph.GraphModule).activation_checkpointing = True diff --git a/libai/models/t5_model.py b/libai/models/t5_model.py new file mode 100644 index 0000000000000000000000000000000000000000..66b5c7eece3bcb30d8c05e7d86c4f579fc7e7c02 --- /dev/null +++ b/libai/models/t5_model.py @@ -0,0 +1,518 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +import oneflow.nn as nn + +from libai.config import configurable +from libai.layers import ( + Embedding, + LayerNorm, + LMLogits, + ParallelCrossEntropyLoss, + TransformerLayer, + VocabEmbedding, +) +from libai.layers.attention import AttnMaskType +from libai.models.utils import init_method_normal, scaled_init_method_normal +from libai.utils import distributed as dist + + +class ExtendedMask(flow.nn.Module): + def forward(self, attention_mask): + return attention_mask.unsqueeze(1) + + +class T5Embedding(flow.nn.Module): + def __init__( + self, + hidden_size, + vocab_size, + max_sequence_length, + embedding_dropout_prob, + init_method=flow.nn.init.xavier_normal_, + amp_enabled=False, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.vocab_size = vocab_size + + self.word_embeddings = VocabEmbedding( + num_embeddings=vocab_size, + embedding_dim=hidden_size, + init_method=init_method, + amp_enabled=amp_enabled, + ) + self.position_embeddings = Embedding( + num_embeddings=max_sequence_length, + embedding_dim=hidden_size, + init_method=init_method, + amp_enabled=amp_enabled, + ) + self.position_ids = flow.arange( + max_sequence_length, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ).unsqueeze(0) + + self.embedding_dropout = flow.nn.Dropout(embedding_dropout_prob) + + def forward(self, input_ids, past_length=0): + seq_length = input_ids.size()[1] + + position_ids = self.position_ids[:, past_length : past_length + seq_length] + position_ids = position_ids.expand_as(input_ids).to_global(sbp=input_ids.sbp) + + word_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + embeddings = word_embeddings + position_embeddings + embeddings = self.embedding_dropout(embeddings) + return embeddings + + +class T5Model(flow.nn.Module): + """T5 Model that outputs logits. + + Args: + vocab_size (int): The size of vocabulary file. + hidden_size (int): The size of hidden states. + hidden_layers (int): The number of ``TransformerLayer`` in the encoder and decoder. + num_attention_heads (int): + The number of attention heads for each attention layer of ``TransformerLayer``. + intermediate_size (int): + The size of intermediate layer in feed-forward network for each ``TransformerLayer``. + embedding_dropout_prob (float): The dropout ratio for the output of T5Embedding Layer. + hidden_dropout_prob (float): The dropout ratio for the output for each ``TransformerLayer``. + attention_probs_dropout_prob (float): + The dropout ratio for the output of each attention layer in ``TransformerLayer``. + max_position_embeddings (int): + Max sequence length of input, defines the shape of Position Embeddings + in ``T5Emebedding``. + initializer_range (float, optional): + Sigma of the normal distribution in the initialization method. Defaults to 0.02. + layernorm_eps (float, optional): The epsilon of LayerNorm layer. Defaults to 1e-12. + bias_gelu_fusion (bool, optional): + Whether or not to fuse the computing of bias and gelu. Defaults to ``False``. + bias_dropout_fusion (bool, optional): + Whether or not to fuse the computing of dropout and bias. Defaults to ``False``. + scale_mask_softmax_fusion (bool, optional): + Whether to fuse the computing of mask and softmax in attention layers. + Defaults to ``False``. + apply_query_key_layer_scaling (bool, optional): + Whether or not to use layer index related scaling in computing attention scores. + If ``True``, the scaling factor equals to sqrt(d) * (layer_index + 1). + Defaults to ``True``. + apply_residual_post_layernorm (bool, optional): + If set ``True``, use original BERT residual connection ordering otherwise use Megatron + BERT residual connection which is more stable when scaling model size introduced in + https://arxiv.org/pdf/1909.08053.pdf. + Default: ``False``. + amp_enabled (bool, optional): + Whether or not to set fp16 for embedding weight in T5 model. Defaults to ``False``. + """ + + @configurable + def __init__( + self, + vocab_size, + hidden_size, + hidden_layers, + num_attention_heads, + intermediate_size, + embedding_dropout_prob, + hidden_dropout_prob, + attention_probs_dropout_prob, + max_position_embeddings, + initializer_range=0.02, + layernorm_eps=1e-12, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=True, + apply_residual_post_layernorm=False, + amp_enabled=False, + ) -> None: + super().__init__() + init_method = init_method_normal(initializer_range) + scaled_init_method = scaled_init_method_normal(initializer_range, hidden_layers) + self.embedding = T5Embedding( + hidden_size=hidden_size, + vocab_size=vocab_size, + max_sequence_length=max_position_embeddings, + embedding_dropout_prob=embedding_dropout_prob, + init_method=init_method, + amp_enabled=amp_enabled, + ) + self.extended_attn_mask = ExtendedMask() + + encoder_layers = flow.nn.ModuleList( + [ + TransformerLayer( + hidden_size=hidden_size, + ffn_hidden_size=intermediate_size, + num_attention_heads=num_attention_heads, + is_decoder=False, + attention_dropout_prob=attention_probs_dropout_prob, + output_dropout_prob=hidden_dropout_prob, + layernorm_epsilon=layernorm_eps, + init_method=init_method, + output_layer_init_method=scaled_init_method, + bias_gelu_fusion=bias_gelu_fusion, + bias_dropout_fusion=bias_dropout_fusion, + scale_mask_softmax_fusion=scale_mask_softmax_fusion, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + apply_residual_post_layernorm=apply_residual_post_layernorm, + attn_mask_type=AttnMaskType.padding, + layer_idx=i, + ) + for i in range(hidden_layers) + ] + ) + + encoder_final_layernorm = LayerNorm( + (hidden_size,), + eps=layernorm_eps, + layer_idx=hidden_layers - 1, + ) + + self.encoder = flow.nn.Sequential() + self.encoder.add_module("layers", encoder_layers) + self.encoder.add_module("final_layernorm", encoder_final_layernorm) + + decoder_layers = flow.nn.ModuleList( + [ + TransformerLayer( + hidden_size=hidden_size, + ffn_hidden_size=intermediate_size, + num_attention_heads=num_attention_heads, + is_decoder=True, + attention_dropout_prob=attention_probs_dropout_prob, + output_dropout_prob=hidden_dropout_prob, + layernorm_epsilon=layernorm_eps, + init_method=init_method, + output_layer_init_method=scaled_init_method, + bias_gelu_fusion=bias_gelu_fusion, + bias_dropout_fusion=bias_dropout_fusion, + scale_mask_softmax_fusion=scale_mask_softmax_fusion, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + attn_mask_type=AttnMaskType.padding, + layer_idx=i, + ) + for i in range(hidden_layers, 2 * hidden_layers) + ] + ) + + decoder_final_layernorm = LayerNorm( + (hidden_size,), + eps=layernorm_eps, + layer_idx=2 * hidden_layers - 1, + ) + + self.decoder = flow.nn.Sequential() + self.decoder.add_module("layers", decoder_layers) + self.decoder.add_module("final_layernorm", decoder_final_layernorm) + self.past_key_values = [None] * len(self.decoder.layers) + self.encoder_states = None + self.past_length = 0 + + self.lm_head = LMLogits(vocab_size, bias=True) + + @classmethod + def from_config(cls, cfg): + return { + "vocab_size": cfg.vocab_size, + "hidden_size": cfg.hidden_size, + "hidden_layers": cfg.hidden_layers, + "num_attention_heads": cfg.num_attention_heads, + "intermediate_size": cfg.intermediate_size, + "embedding_dropout_prob": cfg.embedding_dropout_prob, + "hidden_dropout_prob": cfg.hidden_dropout_prob, + "attention_probs_dropout_prob": cfg.attention_probs_dropout_prob, + "max_position_embeddings": cfg.max_position_embeddings, + "initializer_range": cfg.initializer_range, + "layernorm_eps": cfg.layernorm_eps, + "bias_gelu_fusion": cfg.bias_gelu_fusion, + "bias_dropout_fusion": cfg.bias_dropout_fusion, + "scale_mask_softmax_fusion": cfg.scale_mask_softmax_fusion, + "apply_query_key_layer_scaling": cfg.apply_query_key_layer_scaling, + "apply_residual_post_layernorm": cfg.apply_residual_post_layernorm, + "amp_enabled": cfg.amp_enabled, + } + + def forward( + self, + encoder_input_ids, + decoder_input_ids, + encoder_attn_mask, + decoder_attn_mask, + encoder_decoder_attn_mask, + use_cache=False, + ): + """ + + Args: + encoder_input_ids (flow.LongTensor): + Indices of input sequence tokens in vocabulary for encoder. + decoder_input_ids (flow.LongTensor): + Indices of input sequence tokens in vocabulary for decoder. + encoder_attn_mask (flow.BoolTensor): + Mask for encoder to avoid performing attention on + padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + decoder_attn_mask (flow.BoolTensor): + Mask for decoder to avoid performing attention on subsequent token indices. + Mask values have the same meaning as encoder_attn_mask. + encoder_decoder_attn_mask (flow.BoolTensor): + Mask for decoder to avoid performing attention on encoder padded token indices. + Mask values have the same meaning as encoder_attn_mask. + use_cache (bool, optional): + It will be set to True, when the model is in the inference + phase and used for incremental decoding. Defaults to False. + + Returns: + flow.Tensor: logits + """ + + encoder_input_ids = encoder_input_ids.to_global(placement=dist.get_layer_placement(0)) + decoder_input_ids = decoder_input_ids.to_global(placement=dist.get_layer_placement(0)) + encoder_attn_mask = encoder_attn_mask.to_global(placement=dist.get_layer_placement(0)) + decoder_attn_mask = decoder_attn_mask.to_global(placement=dist.get_layer_placement(0)) + encoder_decoder_attn_mask = encoder_decoder_attn_mask.to_global( + placement=dist.get_layer_placement(0) + ) + if use_cache and self.encoder_states is not None: + encoder_states = self.encoder_states + else: + self.set_cache(encoder_states=None, past_key_values=None) + encoder_attn_mask = self.extended_attn_mask(encoder_attn_mask) + enc_embedding_output = self.embedding(encoder_input_ids) + enc_hidden_states = enc_embedding_output + for layer in self.encoder.layers: + enc_hidden_states = layer(enc_hidden_states, encoder_attn_mask) + encoder_states = self.encoder.final_layernorm(enc_hidden_states) + + decoder_attn_mask = self.extended_attn_mask(decoder_attn_mask) + encoder_decoder_attn_mask = self.extended_attn_mask(encoder_decoder_attn_mask) + dec_embedding_output = self.embedding(decoder_input_ids, self.past_length) + dec_hidden_states = dec_embedding_output + if use_cache: + presents = [] + for layer, past_key_value in zip(self.decoder.layers, self.past_key_values): + dec_hidden_states = layer( + dec_hidden_states, + decoder_attn_mask, + encoder_states, + encoder_decoder_attn_mask, + past_key_value=past_key_value, + use_cache=use_cache, + ) + if use_cache: + dec_hidden_states, present = dec_hidden_states + presents.append(present) + if use_cache: + self.set_cache(encoder_states, past_key_values=presents) + + decoder_states = self.decoder.final_layernorm(dec_hidden_states) + logits = self.lm_head(decoder_states, self.embedding.word_embeddings.weight) + return logits + + def set_cache(self, encoder_states, past_key_values): + self.encoder_states = encoder_states + self.past_length = 0 if past_key_values is None else past_key_values[0][0].shape[2] + + if past_key_values is None: + past_key_values = [None] * len(self.decoder.layers) + assert len(past_key_values) == len(self.decoder.layers), ( + f"past_key_values's length {len(past_key_values)} doesn't match " + f"decoder num_layers' length {self.decoder.layers}" + ) + self.past_key_values = past_key_values + + +class T5Loss(flow.nn.Module): + def __init__(self) -> None: + super().__init__() + self.lm_loss = ParallelCrossEntropyLoss() + + def forward(self, logits, lm_labels, loss_mask): + lm_loss = self.lm_loss(logits, lm_labels) + loss_mask = loss_mask.to_global(placement=lm_loss.placement) + loss_mask = loss_mask.float() + denominator = loss_mask.sum().to_global( + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]) + ) + lm_loss = flow._C.amp_white_identity(lm_loss) + lm_loss = flow._C.amp_black_identity(lm_loss) + masked_lm_loss = flow.sum(lm_loss.view(-1) * loss_mask.view(-1)) / denominator + masked_lm_loss = masked_lm_loss.to_global( + sbp=dist.get_nd_sbp([flow.sbp.partial_sum, flow.sbp.broadcast]) + ) + return {"masked_lm_loss": masked_lm_loss} + + +class T5ForPreTraining(flow.nn.Module): + """ + T5 Model with classification head on top. + """ + + def __init__(self, cfg) -> None: + super().__init__() + self.t5_model = T5Model(cfg) + self.loss_func = T5Loss() + + def set_cache(self, encoder_states, past_key_values): + self.t5_model.set_cache(encoder_states, past_key_values) + + def forward( + self, + encoder_input_ids, + decoder_input_ids, + encoder_attn_mask, + decoder_attn_mask, + encoder_decoder_attn_mask, + lm_labels=None, + loss_mask=None, + use_cache=False, + ): + """ + + Args: + encoder_input_ids (flow.LongTensor): + Indices of input sequence tokens in vocabulary for encoder. + decoder_input_ids (flow.LongTensor): + Indices of input sequence tokens in vocabulary for decoder. + encoder_attn_mask (flow.BoolTensor): + Mask for encoder to avoid performing attention on + padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + decoder_attn_mask (flow.BoolTensor): + Mask for decoder to avoid performing attention on subsequent token indices. + Mask values have the same meaning as encoder_attn_mask. + encoder_decoder_attn_mask (flow.BoolTensor): + Mask for decoder to avoid performing attention on encoder padded token indices. + Mask values have the same meaning as encoder_attn_mask. + lm_labels (flow.LongTensor, optional): Labels for computing the masked + language modeling loss. Indices should be in `[-1, 0, ..., config.vocab_size]`. + None for evaluating. + loss_mask (flow.BoolTensor, optional): + Mask to avoid performing loss computing on ignored tokens. + Tokens with indices set to `-1` are ignored (masked), the loss is only computed + for the tokens with labels in `[0, ..., config.vocab_size]`. + None for evaluating. + use_cache (bool, optional): + It will be set to True, when the model is in the inference + phase and used for incremental decoding. Defaults to False. + + Returns: + dict: + A dict containing :code:`loss_value` or :code:`logits` + depending on training or evaluation mode. + :code:`{"masked_lm_loss": loss_value}` when training, + :code:`{"prediction_scores": logits}` when evaluating. + """ + logits = self.t5_model( + encoder_input_ids, + decoder_input_ids, + encoder_attn_mask, + decoder_attn_mask, + encoder_decoder_attn_mask, + use_cache=use_cache, + ) + + if lm_labels is not None: + lm_loss = self.loss_func(logits, lm_labels, loss_mask) + return lm_loss + else: + return { + "prediction_scores": logits, + } + + @staticmethod + def set_pipeline_stage_id(model): + dist_utils = dist.get_dist_util() + + # Set pipeline parallelism stage_id + if hasattr(model.t5_model.encoder.final_layernorm, "config"): + # Old API in OneFlow 0.8 + for module_block in model.modules(): + if isinstance(module_block.origin, T5Embedding): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, ExtendedMask): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, TransformerLayer): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.origin, LMLogits): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + elif isinstance(module_block.origin, T5Loss): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + model.t5_model.encoder.final_layernorm.config.set_stage( + dist_utils.get_layer_stage_id(model.t5_model.encoder.final_layernorm.layer_idx), + dist.get_layer_placement(model.t5_model.encoder.final_layernorm.layer_idx), + ) + model.t5_model.decoder.final_layernorm.config.set_stage( + dist_utils.get_layer_stage_id(model.t5_model.decoder.final_layernorm.layer_idx), + dist.get_layer_placement(model.t5_model.decoder.final_layernorm.layer_idx), + ) + else: + for module_block in model.modules(): + if isinstance(module_block.to(nn.Module), T5Embedding): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), ExtendedMask): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.to(nn.Module), LMLogits): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + elif isinstance(module_block.to(nn.Module), T5Loss): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + model.t5_model.encoder.final_layernorm.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(model.t5_model.encoder.final_layernorm.layer_idx), + dist.get_layer_placement(model.t5_model.encoder.final_layernorm.layer_idx), + ) + model.t5_model.decoder.final_layernorm.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(model.t5_model.decoder.final_layernorm.layer_idx), + dist.get_layer_placement(model.t5_model.decoder.final_layernorm.layer_idx), + ) diff --git a/libai/models/utils/__init__.py b/libai/models/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d960c137ec0d49f7570a716fb3b0dec0825602e0 --- /dev/null +++ b/libai/models/utils/__init__.py @@ -0,0 +1,24 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .graph_base import GraphBase +from .weight_init import init_method_normal, scaled_init_method_normal +from .model_utils.base_loader import ModelLoaderHuggerFace, ModelLoaderLiBai +from .model_utils.bert_loader import BertLoaderHuggerFace, BertLoaderLiBai +from .model_utils.roberta_loader import RobertaLoaderHuggerFace, RobertaLoaderLiBai +from .model_utils.gpt_loader import GPT2LoaderHuggerFace, GPT2LoaderLiBai +from .model_utils.swin_loader import SwinLoaderHuggerFace, SwinLoaderLiBai +from .model_utils.swinv2_loader import SwinV2LoaderHuggerFace, SwinV2LoaderLiBai +from .model_utils.vit_loader import ViTLoaderHuggerFace, ViTLoaderLiBai diff --git a/libai/models/utils/__pycache__/__init__.cpython-39.pyc b/libai/models/utils/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ad8f6fb3ed2a2de6c98911b7f5c4cfbd952b2dc9 Binary files /dev/null and b/libai/models/utils/__pycache__/__init__.cpython-39.pyc differ diff --git a/libai/models/utils/__pycache__/graph_base.cpython-39.pyc b/libai/models/utils/__pycache__/graph_base.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1f66509c851cd08dfaec0c6750f07cb4ced7d059 Binary files /dev/null and b/libai/models/utils/__pycache__/graph_base.cpython-39.pyc differ diff --git a/libai/models/utils/__pycache__/weight_init.cpython-39.pyc b/libai/models/utils/__pycache__/weight_init.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..37af784ead22bdba0df8e009bc8811d484ee3fd9 Binary files /dev/null and b/libai/models/utils/__pycache__/weight_init.cpython-39.pyc differ diff --git a/libai/models/utils/graph_base.py b/libai/models/utils/graph_base.py new file mode 100644 index 0000000000000000000000000000000000000000..96e68009be5d96c8864c6ed81c7eb7a22a568fe6 --- /dev/null +++ b/libai/models/utils/graph_base.py @@ -0,0 +1,138 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import oneflow as flow +from oneflow import nn + +from libai.layers import TransformerLayer +from libai.utils import distributed as dist + +logger = logging.getLogger(__name__) + + +class GraphBase(nn.Graph): + def __init__( + self, + model: nn.Module, + optimizer: flow.optim.Optimizer = None, + lr_scheduler: flow.optim.lr_scheduler = None, + fp16=False, + activation_checkpoint=False, + grad_acc_steps=1, + zero_optim=False, + zero_stage=0, + is_train=True, + auto_parallel_conf=None, + ): + super().__init__() + + self.model = model + self.is_train = is_train + + if is_train: + self.add_optimizer(optimizer, lr_sch=lr_scheduler) + if fp16: + self.config.enable_amp(True) + grad_scaler = flow.amp.GradScaler( + init_scale=65536.0 * dist.get_data_parallel_size(), + growth_factor=2.0, + backoff_factor=0.5, + growth_interval=2000, + ) + self.set_grad_scaler(grad_scaler) + + if grad_acc_steps > 1: + self.config.set_gradient_accumulation_steps(grad_acc_steps) + + if activation_checkpoint: + self.set_activation_checkpoint() + + if zero_optim: + self.config.enable_zero(True, stage=zero_stage) + + self.set_pipeline_stage_id() + + self.config.allow_fuse_add_to_output(True) + self.config.allow_fuse_model_update_ops(True) + self.config.allow_fuse_cast_scale(True) + + # Enable cuda stream for computation and communication as the same stream. + # This will reduce memory when using model parallelism. + dist_util = dist.get_dist_util() + if dist_util.is_tensor_model_parallel() or dist_util.is_pipeline_model_parallel(): + flow.boxing.nccl.enable_use_compute_stream(True) + + # auto_parallel + if auto_parallel_conf is not None and auto_parallel_conf.enabled: + try: + self.config.enable_auto_parallel(True) + self.config.enable_auto_parallel_ignore_user_sbp_config( + auto_parallel_conf.enable_auto_parallel_ignore_user_sbp_config + ) + self.config.set_auto_parallel_computation_cost_ratio(0.05) + self.config.set_auto_parallel_wait_time(1.65e4) + self.config.enable_auto_parallel_trunk_algo(auto_parallel_conf.trunk_algo) + self.config.enable_auto_parallel_sbp_collector(auto_parallel_conf.sbp_collector) + except RuntimeWarning: + import warnings + + warnings.warn( + "The version of oneflow don't support auto_parallel.\n" + "Please reinstall the oneflow nightly:\n" + "python3 -m pip install --pre oneflow -f https://staging.oneflow.info/branch/master/[PLATFORM]" # noqa + ) + + def build(self, **kwargs): + if self.is_train: + logger.info( + "Start compling the train graph which may take some time. " + "Please wait for a moment ..." + ) + loss_dict = self.model(**kwargs) + losses = sum(v for k, v in loss_dict.items() if "loss" in k) + losses.backward() + return loss_dict + else: + logger.info( + "Start compling the eval graph which may take some time. " + "Please wait for a moment ..." + ) + return self.model(**kwargs) + + def set_activation_checkpoint(self): + if hasattr(self.model, "origin"): + if hasattr(type(self.model.origin), "set_activation_checkpoint"): + type(self.model.origin).set_activation_checkpoint(self.model) + else: + for module_block in self.model.modules(): + if isinstance(module_block.origin, TransformerLayer): + module_block.config.activation_checkpointing = True + else: + if hasattr(type(self.model.to(nn.Module)), "set_activation_checkpoint"): + type(self.model.to(nn.Module)).set_activation_checkpoint(self.model) + else: + for module_block in self.model.modules(): + if isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(nn.graph.GraphModule).activation_checkpointing = True + + def set_pipeline_stage_id(self): + if hasattr(self.model, "origin"): + if hasattr(type(self.model.origin), "set_pipeline_stage_id"): + type(self.model.origin).set_pipeline_stage_id(self.model) + else: + if hasattr(type(self.model.to(nn.Module)), "set_pipeline_stage_id"): + type(self.model.to(nn.Module)).set_pipeline_stage_id(self.model) diff --git a/libai/models/utils/model_utils/README.md b/libai/models/utils/model_utils/README.md new file mode 100644 index 0000000000000000000000000000000000000000..86a7a494f088425d71ca63bb11d24da86fb06d61 --- /dev/null +++ b/libai/models/utils/model_utils/README.md @@ -0,0 +1,39 @@ +## Introduction +Here are the Weight Loaders currently supported in LiBai. You can use them to load the models in LiBai and the models stored on the huggingface. + + +## Weight Loader On LiBai +- [BERT Loader](./bert_loader.py) +- [RoBERTa Loader](./roberta_loader.py) +- [GPT2 Loader](./gpt_loader.py) +- [MT5 Loader](../../../../projects/MT5/utils/mt5_loader.py) +- [SWIN Loader](./swin_loader.py) +- [SWIN2 Loader](./swinv2_loader.py) +- [VIT Loader](./vit_loader.py) + +## How To Use +We can easily load pretrained BERT as following: +```python +import libai +from libai.models.utils import BertLoaderHuggerFace, BertLoaderLiBai +from configs.common.models.bert import cfg + +# load huggingface weight +loader = BertLoaderHuggerFace( + model=libai.models.BertModel, + libai_cfg=cfg, + pretrained_model_path="path/to/huggingface_pretrained_model_directory", + hidden_dropout_prob=0, + apply_residual_post_layernorm=True +) +bert = loader.load() + +# load libai weight +loader = BertLoaderLiBai( + model=libai.models.BertModel, + libai_cfg=cfg, + pretrained_model_path='path/to/libai_pretrained_model_directory', + hidden_dropout_prob=0, +) +bert = loader.load() +``` diff --git a/libai/models/utils/model_utils/__init__.py b/libai/models/utils/model_utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/libai/models/utils/model_utils/__pycache__/__init__.cpython-39.pyc b/libai/models/utils/model_utils/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..819196ecc8eeb5502ac24e757310d4b9f4b656ef Binary files /dev/null and b/libai/models/utils/model_utils/__pycache__/__init__.cpython-39.pyc differ diff --git a/libai/models/utils/model_utils/__pycache__/base_loader.cpython-39.pyc b/libai/models/utils/model_utils/__pycache__/base_loader.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a61a6b9653169c0ce769983de103de1b223d5c60 Binary files /dev/null and b/libai/models/utils/model_utils/__pycache__/base_loader.cpython-39.pyc differ diff --git a/libai/models/utils/model_utils/__pycache__/bert_loader.cpython-39.pyc b/libai/models/utils/model_utils/__pycache__/bert_loader.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..96fcc0fa11f40f747df93e1721fb604133daabcc Binary files /dev/null and b/libai/models/utils/model_utils/__pycache__/bert_loader.cpython-39.pyc differ diff --git a/libai/models/utils/model_utils/__pycache__/gpt_loader.cpython-39.pyc b/libai/models/utils/model_utils/__pycache__/gpt_loader.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f53159300b504fa096bd4e6c6a9e9d438823891 Binary files /dev/null and b/libai/models/utils/model_utils/__pycache__/gpt_loader.cpython-39.pyc differ diff --git a/libai/models/utils/model_utils/__pycache__/roberta_loader.cpython-39.pyc b/libai/models/utils/model_utils/__pycache__/roberta_loader.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f16660f28d947bbdb1dd2d9b648fff4b6478e618 Binary files /dev/null and b/libai/models/utils/model_utils/__pycache__/roberta_loader.cpython-39.pyc differ diff --git a/libai/models/utils/model_utils/__pycache__/swin_loader.cpython-39.pyc b/libai/models/utils/model_utils/__pycache__/swin_loader.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c2899bfd6e6f9f945232119bfea22dc98e1fdbea Binary files /dev/null and b/libai/models/utils/model_utils/__pycache__/swin_loader.cpython-39.pyc differ diff --git a/libai/models/utils/model_utils/__pycache__/swinv2_loader.cpython-39.pyc b/libai/models/utils/model_utils/__pycache__/swinv2_loader.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3cc1f4401fff985133e2a6fa89d3b78640b3981e Binary files /dev/null and b/libai/models/utils/model_utils/__pycache__/swinv2_loader.cpython-39.pyc differ diff --git a/libai/models/utils/model_utils/__pycache__/vit_loader.cpython-39.pyc b/libai/models/utils/model_utils/__pycache__/vit_loader.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f1e39b9edead8fb1bb3fcb4e5bf61540f9d3510a Binary files /dev/null and b/libai/models/utils/model_utils/__pycache__/vit_loader.cpython-39.pyc differ diff --git a/libai/models/utils/model_utils/base_loader.py b/libai/models/utils/model_utils/base_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..c753a3d69685017ae5d87454490b7dc982b1bb4d --- /dev/null +++ b/libai/models/utils/model_utils/base_loader.py @@ -0,0 +1,603 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import copy +import logging +import os + +import omegaconf +import oneflow as flow +from termcolor import colored + +import libai.utils.distributed as dist +from libai.config import LazyCall +from libai.models.build import build_model + +logger = logging.getLogger(__name__) + + +WEIGHTS_NAME_PT = "pytorch_model.bin" +CONFIG_NAME = "config.json" + + +def _load_state_dict_into_model(model_to_load, state_dict, start_prefix): + """load state dict into model + + Args: + model_to_load (nn.Module): Model to be loaded. + state_dict (OrderedDict): State dict of pretrained model. + start_prefix (str): Start prefix. + + Returns: + list: error message about loading. + """ + metadata = getattr(state_dict, "_metadata", None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + error_msgs = [] + + def load(module, prefix=""): + local_metadata = {} if metadata is None else metadata.get(prefix[:-1], {}) + args = (state_dict, prefix, local_metadata, True, [], [], error_msgs) + module._load_from_state_dict(*args) + + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + ".") + + load(model_to_load, prefix=start_prefix) + + return error_msgs + + +class ModelLoader(object): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + """Class used to load the [`transformers`](https://huggingface.co/models) pretrained model + or `OneFlow` pretrained model. + + Args: + model (libai.models): Model to be loaded in Libai. + libai_cfg (dict): The config of model in LiBai, you can import it from + `libai.config.configs.common.models`. + pretrained_model_path (str): The directory path of pretrained model, + which contains model weights file and config file. + output_loading_info (`bool`, *optional*, defaults to `False`): + Whether to return a dictionary containing missing keys, unexpected keys + and error messages. + """ + self.model = model + self.libai_cfg = libai_cfg + self.pretrained_model_path = pretrained_model_path + self.kwargs = kwargs + self.output_loading_info = kwargs.pop("output_loading_info", False) + + def _state_dict_to_global(self, flow_state_dict=None, mode="libai"): + """Tensor in OneFlow state dict to global according to model's sbp and placement. + + Args: + flow_state_dict (OrderedDict): State dict of OneFlow's pretrained model. + """ + assert mode in ["libai", "pytorch"], f"not support for mode {mode}" + if mode == "libai" or dist.is_main_process(): + prefix = self.base_model_prefix_2 + + # Checkpoint + has_prefix_module = any( + s.startswith(self.base_model_prefix_2) for s in flow_state_dict.keys() + ) + # Module + expects_prefix_module = any( + s.startswith(prefix) for s in self.model.state_dict().keys() + ) + + start_prefix = "" if has_prefix_module else prefix + "." + loaded_keys = [start_prefix + key for key in flow_state_dict.keys()] + else: + prefix, has_prefix_module, expects_prefix_module, loaded_keys = [None] * 4 + flow_state_dict = collections.OrderedDict() + + prefix = dist.broadcast_py_object(prefix, src=0) + has_prefix_module = dist.broadcast_py_object(has_prefix_module, src=0) + expects_prefix_module = dist.broadcast_py_object(expects_prefix_module, src=0) + loaded_keys = dist.broadcast_py_object(loaded_keys, src=0) + + # to global + for key, value in self.model.state_dict().items(): + if not expects_prefix_module: + key = prefix + "." + key + if key in loaded_keys: + if not has_prefix_module: + key = ".".join(key.split(".")[1:]) + + if mode == "pytorch": + flow_state_dict[key] = flow.to_global( + flow_state_dict[key] if dist.is_main_process() else flow.Tensor(None), + sbp=flow.sbp.broadcast, + placement=flow.placement("cpu", ranks=[0]), + ) + + flow_state_dict[key] = flow.to_global( + flow_state_dict[key], + sbp=value.sbp, + placement=flow.placement("cpu", ranks=list(value.placement.ranks)), + ) + return flow_state_dict + + def _load_pretrained_model( + self, + model, + state_dict, + pretrained_model_path, + ignore_mismatched_sizes=False, + ): + """Load pretrained model. + + Args: + model (libai.models): The model to be loaded. + state_dict (OrderedDict): state dict. + loaded_keys (list): keys of state dict. + pretrained_model_path (str): pretrained modelE path. + ignore_mismatched_sizes (bool): + Whether or not to raise an error if some of the weights + from the checkpoint do not have the same size as the + weights of the model, defaults to `False`. + """ + model_state_dict = model.state_dict() + expected_keys = list(model_state_dict.keys()) + prefix = self.base_model_prefix_2 + + loaded_keys = state_dict.keys() + if len(prefix) > 0: + has_prefix_module = any(s.startswith(prefix) for s in loaded_keys) + expects_prefix_module = any(s.startswith(prefix) for s in expected_keys) + else: + has_prefix_module = False + expects_prefix_module = False + + remove_prefix_from_model = not has_prefix_module and expects_prefix_module + add_prefix_to_model = has_prefix_module and not expects_prefix_module + + if remove_prefix_from_model: + expected_keys_not_prefixed = [s for s in expected_keys if not s.startswith(prefix)] + expected_keys = [ + ".".join(s.split(".")[1:]) if s.startswith(prefix) else s for s in expected_keys + ] + elif add_prefix_to_model: + expected_keys = [".".join([prefix, s]) for s in expected_keys] + + missing_keys = list(set(expected_keys) - set(loaded_keys)) + unexpected_keys = list(set(loaded_keys) - set(expected_keys)) + + start_prefix = "" + model_to_load = model + if ( + len(self.base_model_prefix_2) > 0 + and not hasattr(model, self.base_model_prefix_2) + and has_prefix_module + ): + start_prefix = self.base_model_prefix_2 + "." + if ( + len(self.base_model_prefix_2) > 0 + and hasattr(model, self.base_model_prefix_2) + and not has_prefix_module + ): + model_to_load = getattr(model, self.base_model_prefix_2) + if any(key in expected_keys_not_prefixed for key in loaded_keys): + raise ValueError("The state dict of the model you are loading is corrupted.") + + def _find_mismatched_keys( + state_dict, + model_state_dict, + loaded_keys, + add_prefix_to_model, + remove_prefix_from_model, + ignore_mismatched_sizes, + ): + mismatched_keys = [] + if ignore_mismatched_sizes: + for checkpoint_key in loaded_keys: + model_key = checkpoint_key + if remove_prefix_from_model: + model_key = f"{prefix}.{checkpoint_key}" + elif add_prefix_to_model: + model_key = ".".join(checkpoint_key.split(".")[1:]) + + if ( + model_key in model_state_dict + and state_dict[checkpoint_key].shape != model_state_dict[model_key].shape + ): + mismatched_keys.append( + ( + checkpoint_key, + state_dict[checkpoint_key].shape, + model_state_dict[model_key].shape, + ) + ) + del state_dict[checkpoint_key] + return mismatched_keys + + if state_dict is not None: + mismatched_keys = _find_mismatched_keys( + state_dict, + model_state_dict, + loaded_keys, + add_prefix_to_model, + remove_prefix_from_model, + ignore_mismatched_sizes, + ) + error_msgs = _load_state_dict_into_model(model_to_load, state_dict, start_prefix) + + if dist.get_local_rank() == 0: + if len(error_msgs) > 0: + error_msg = "\n\t".join(error_msgs) + raise RuntimeError( + f"Error(s) in loading state_dict for {model.__class__.__name__}:\n\t{error_msg}" + ) + if len(unexpected_keys) > 0: + logger.warning( + f"Some weights of the model checkpoint at {pretrained_model_path} " + "were not used when " + f"initializing {model.__class__.__name__}:\n {unexpected_keys}\n" + ) + else: + logger.info( + f"All model checkpoint weights were used when initializing " + f"{model.__class__.__name__}.\n" + ) + if len(missing_keys) > 0: + logger.warning( + f"Some weights of {model.__class__.__name__} were not initialized " + f"from the model checkpoint at {pretrained_model_path}:\n " + f"{missing_keys} \n" + ) + elif len(mismatched_keys) == 0: + logger.info( + f"All the weights of {model.__class__.__name__} were initialized " + f"from the model checkpoint at {pretrained_model_path}.\n" + ) + if len(mismatched_keys) > 0: + mismatched_warning = "\n".join( + [ + f"- {key}: found shape {shape1} in the checkpoint and {shape2}" + "in the model instantiated" + for key, shape1, shape2 in mismatched_keys + ] + ) + logger.warning( + f"Some weights of {model.__class__.__name__} were not initialized" + f"from the model checkpoint at {pretrained_model_path} " + f"and are newly initialized because the shapes did not" + f"match:\n{mismatched_warning}\n" + ) + + return model, missing_keys, unexpected_keys, mismatched_keys, error_msgs + + +class ModelLoaderLiBai(ModelLoader): + """Class used to load `OneFlow` pretrained model. + + Args: + model (libai.models): Model to be loaded in Libai. + libai_cfg (dict): The config of model in LiBai, you can import it from + `libai.config.configs.common.models`. + pretrained_model_path (str): The directory path of pretrained model, + which contains model weights file and config file. + output_loading_info (`bool`, *optional*, defaults to `False`): + Whether to return a dictionary containing missing keys, unexpected keys + and error messages. + """ + + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + self.base_model_prefix_2 = None # prefix in LiBai + + def _load_flow_state_dict(self, state_dict_file): + # load oneflow_model + state_dict = flow.load(state_dict_file, global_src_rank=0) + return state_dict + + def load(self): + """Load model. + + # For example: + + # .. code-block:: python + + >>> import libai + >>> from libai.config.configs.common.models.bert import cfg + >>> from model_utils import BertLoaderLiBai + + >>> loder = BertLoaderLiBai( + libai.models.BertModel, + cfg, + 'path/bert-base-chinese' + ) + >>> bert = loder.load() + + """ + + if dist.is_main_process(): + assert os.path.isdir( + self.pretrained_model_path + ), f"{self.pretrained_model_path} must be a directory" + + flow_state_dict = self._load_flow_state_dict(self.pretrained_model_path) + + # Instance model + if isinstance(self.model, omegaconf.dictconfig.DictConfig): + self.model.cfg = self.libai_cfg + self.model = build_model(self.model) + else: + self.model = build_model(LazyCall(self.model)(cfg=self.libai_cfg)) + + # State_dict to global + self._state_dict_to_global(flow_state_dict, mode="libai") + + # Load + ( + model, + missing_keys, + unexpected_keys, + mismatched_keys, + error_msgs, + ) = self._load_pretrained_model(self.model, flow_state_dict, self.pretrained_model_path) + + if self.output_loading_info: + loading_info = { + "missing_keys": missing_keys, + "unexpected_keys": unexpected_keys, + "mismatched_keys": mismatched_keys, + "error_msgs": error_msgs, + } + return model, loading_info + return model + + +class ModelLoaderHuggerFace(ModelLoader): + """Class used to load the [`transformers`](https://huggingface.co/models) + pretrained model. + """ + + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + self.base_model_prefix_1 = None # prefix in Transformers + self.base_model_prefix_2 = None # prefix in LiBai + self.origin_libai_cfg = copy.deepcopy(self.libai_cfg) + self.changed_keys = set() # Store the changed configuration + + def _convert_tensor(self, tensor): + """Convert PyTorch tensor to OneFlow tensor. + + Args: + tensor (torch.Tensor): The source tensor. + + Returns: + flow.Tensor: The target tensor. + """ + tensor = tensor.float() + return flow.Tensor(tensor.detach().cpu().numpy()) + + def _convert_tensors(self, torch_state_dict): + + for k, v in torch_state_dict.items(): + torch_state_dict[k] = self._convert_tensor(v) + + return torch_state_dict + + def _fix_key(self, state_dict): + """Fix the key in state dict: Convert "gamma" to "weight" and "beta" to "bias". + + Args: + state_dict (OrderedDict): state dict of pretrained model. + + Returns: + OrderedDict: State dict after fix key. + """ + old_keys = [] + new_keys = [] + for key in state_dict.keys(): + new_key = None + if "gamma" in key: + new_key = key.replace("gamma", "weight") + if "beta" in key: + new_key = key.replace("beta", "bias") + if new_key: + old_keys.append(key) + new_keys.append(new_key) + for old_key, new_key in zip(old_keys, new_keys): + state_dict[new_key] = state_dict.pop(old_key) + return state_dict + + def _fix_qkv_ordering( + self, qkv, head_size, num_heads, hidden_size=None, checkpoint_version=0.0 + ): + # TODO(xzp): Different versions checkpoint + + hidden_size = (head_size * num_heads) if hidden_size is None else hidden_size + num_of_qkv = qkv.shape[0] // (head_size * num_heads) + mode = "weight" if qkv.ndim > 1 else "bias" + if mode == "weight": + qkv = qkv.view([num_of_qkv, num_heads, head_size, hidden_size]) + qkv = ( + qkv.permute(1, 0, 2, 3) + .contiguous() + .view(num_of_qkv * head_size * num_heads, hidden_size) + ) + elif mode == "bias": + qkv = qkv.view(num_of_qkv, num_heads, head_size) + qkv = qkv.permute(1, 0, 2).contiguous().view(-1) + return qkv + + def _convert_state_dict(self, flow_state_dict, cfg): + """A function used to convert the checkpoint file of Huggingface to LiBai. + + Args: + torch_state_dict (OrderedDict): torch state dict. + cfg (dict): model's default config dict in LiBai. + + Returns: + OrderedDict: flow state dict. + """ + raise NotImplementedError("_convert_state_dict not implemented") + + def _load_config_from_json(self, config_file): + """load config from `config.json`, and update default config. + + Args: + config_file (str): Path of config file. + """ + + raise NotImplementedError("_load_config_from_json not implemented") + + def _load_torch_state_dict(self, state_dict_file): + try: + import torch + except ImportError: + raise ImportError("Load torch state dict need torch.") + + # load pytorch_model.bin + state_dict = torch.load(state_dict_file, map_location="cpu") + return state_dict + + def _update_cfg(self, keys_libai, value_target): + """Update the libai_cfg according to target_cfg. + + Args: + keys_libai (str): The key of libai_cfg. + value_target (int | float): The value of target_cfg. + """ + if keys_libai not in self.libai_cfg.keys(): + return + if self.libai_cfg[keys_libai] != value_target: + self.libai_cfg[keys_libai] = value_target + + def _update_cfg_log(self): + if dist.get_local_rank() == 0: + for key in sorted(self.libai_cfg): + if self.origin_libai_cfg[key] == self.libai_cfg[key]: + continue + self.changed_keys.add(key) + temp_key = colored(key, "yellow") + logger.info( + f"changed libai model cfg {temp_key} : " + f"{self.origin_libai_cfg[key]} -> {self.libai_cfg[key]} " + ) + logger.warning( + "The following model configurations has been modified according " + "to `config.json` or kwargs: \n" + f"{self.changed_keys} \n" + ) + + if dist.get_pipeline_parallel_size() > 1: + logger.warning( + colored( + "If you use pipeline parallel, please " + "confirm the setting of `train.dist.pipeline_num_layers` \n", + "red", + ) + ) + + def load(self): + """Load model. + + # For example: + + # .. code-block:: python + + >>> import libai + >>> from configs.common.models.bert import cfg + >>> from libai.models.utils import BertLoaderHugger + + >>> loader = BertLoaderHugger( + libai.models.BertModel, + cfg, + 'path/bert-base-chinese' + ) + >>> bert = loader.load() + + """ + if dist.is_main_process(): + if os.path.isdir(self.pretrained_model_path): + # state_dict file pytorch + if os.path.isfile(os.path.join(self.pretrained_model_path, WEIGHTS_NAME_PT)): + model_file = os.path.join(self.pretrained_model_path, WEIGHTS_NAME_PT) + else: + raise EnvironmentError( + f"Error no file named {WEIGHTS_NAME_PT} found" + f"in directory {self.pretrained_model_path}." + ) + + # config file + if os.path.isfile(os.path.join(self.pretrained_model_path, CONFIG_NAME)): + config_file = os.path.join(self.pretrained_model_path, CONFIG_NAME) + + # Load config and update config. + self._load_config_from_json(config_file) + else: + import warnings + + warnings.warn( + f"Error no file named {CONFIG_NAME} found in directory" + f"{self.pretrained_model_path}", + RuntimeWarning, + ) + else: + raise EnvironmentError(f"{self.pretrained_model_path} is not a directory.") + + logger.info("loading torch model...") + torch_state_dict = self._load_torch_state_dict(model_file) + torch_state_dict = self._fix_key(torch_state_dict) + logger.info("transfering torch model into oneflow model...") + flow_state_dict = self._convert_tensors(torch_state_dict) + flow_state_dict = self._convert_state_dict(torch_state_dict, self.libai_cfg) + else: + flow_state_dict = None + + self.libai_cfg = dist.broadcast_py_object(self.libai_cfg, src=0) + + # Instance model + logger.info("building LiBai model...") + if isinstance(self.model, omegaconf.dictconfig.DictConfig): + self.model.cfg = self.libai_cfg + self.model = build_model(self.model) + else: + self.model = build_model(LazyCall(self.model)(cfg=self.libai_cfg)) + + # State_dict to global + logger.info("transfering state_dict local to global...") + flow_state_dict = self._state_dict_to_global(flow_state_dict, mode="pytorch") + + logger.info("loading model weights into LiBai...") + # Load + ( + model, + missing_keys, + unexpected_keys, + mismatched_keys, + error_msgs, + ) = self._load_pretrained_model(self.model, flow_state_dict, self.pretrained_model_path) + + if self.output_loading_info: + loading_info = { + "missing_keys": missing_keys, + "unexpected_keys": unexpected_keys, + "mismatched_keys": mismatched_keys, + "error_msgs": error_msgs, + } + return model, loading_info + return model diff --git a/libai/models/utils/model_utils/bert_loader.py b/libai/models/utils/model_utils/bert_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..8ae5fc2d2012769f81ac5ea450856970f6276678 --- /dev/null +++ b/libai/models/utils/model_utils/bert_loader.py @@ -0,0 +1,263 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import oneflow as flow + +from .base_loader import ModelLoaderHuggerFace, ModelLoaderLiBai + + +class BertLoaderHuggerFace(ModelLoaderHuggerFace): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + + """NOTE: base_model_prefix_1 is BERT's prefix in Transformers. + base_model_prefix_2 is BERT's prefix in LiBai.""" + self.base_model_prefix_1 = "bert" + self.base_model_prefix_2 = "bert" + + def _convert_state_dict(self, flow_state_dict, cfg): + """Convert state_dict's keys to match model. + + Args: + flow_state_dict (OrderedDict): model state dict. + cfg (dict): model's default config dict in LiBai. + + Returns: + OrderedDict: flow state dict. + """ + # The converted checkpoint. + oneflow_state_dict = flow_state_dict.copy() + + # Get configs + num_heads = cfg.get("num_attention_heads") + hidden_size = cfg.get("hidden_size") + layers = cfg.get("hidden_layers") + head_size = int(hidden_size / num_heads) + + # prefix + has_prefix = any(s.startswith(self.base_model_prefix_1) for s in oneflow_state_dict) + + prefix = "bert." if has_prefix else "" + index_idx = 3 if has_prefix else 2 + qkv_idx = 6 if has_prefix else 5 + + old_keys = oneflow_state_dict.keys() + + for key in list(old_keys): + + # Convert bert's embedding layers + if "embeddings" in key: + if "word_embeddings" in key: + new_key = key.replace("word_embeddings", "vocab_embeddings") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "token_type_embeddings" in key: + new_key = key.replace("token_type_embeddings", "tokentype_embeddings") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "LayerNorm.weight" in key: + new_key = prefix + "encoders.0.input_layernorm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "LayerNorm.bias" in key: + new_key = prefix + "encoders.0.input_layernorm.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + else: + oneflow_state_dict[key] = oneflow_state_dict[key] + + # Convert bert's attention layers + elif "attention" in key: + if "self" in key: + index = key.split(".")[index_idx] + if ( + prefix + "encoders." + index + ".self_attention.query_key_value.weight" + in oneflow_state_dict.keys() + ): + continue + q_w = key.replace(key.split(".")[qkv_idx], "query").replace( + key.split(".")[qkv_idx + 1], "weight" + ) + k_w = q_w.replace("query", "key") + v_w = q_w.replace("query", "value") + q_b = q_w.replace("weight", "bias") + k_b = k_w.replace("weight", "bias") + v_b = v_w.replace("weight", "bias") + + qkv_w = flow.cat( + ( + oneflow_state_dict.pop(q_w), + oneflow_state_dict.pop(k_w), + oneflow_state_dict.pop(v_w), + ), + dim=0, + ) + qkv_b = flow.cat( + ( + oneflow_state_dict.pop(q_b), + oneflow_state_dict.pop(k_b), + oneflow_state_dict.pop(v_b), + ), + dim=-1, + ) + + qkv_w = self._fix_qkv_ordering(qkv_w, head_size, num_heads) + qkv_b = self._fix_qkv_ordering(qkv_b, head_size, num_heads) + + new_key = ( + prefix + "encoders." + index + ".self_attention.query_key_value.weight" + ) + oneflow_state_dict[new_key] = qkv_w + + new_key = prefix + "encoders." + index + ".self_attention.query_key_value.bias" + oneflow_state_dict[new_key] = qkv_b + elif "output" in key: + index = key.split(".")[index_idx] + if "dense" in key: + if "weight" in key: + new_key = prefix + "encoders." + index + ".self_attention.dense.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = prefix + "encoders." + index + ".self_attention.dense.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "LayerNorm" in key: + if "weight" in key: + new_key = ( + prefix + "encoders." + index + ".post_attention_layernorm.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = ( + prefix + "encoders." + index + ".post_attention_layernorm.bias" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + # Convert bert's intermediate layers + elif "intermediate" in key: + index = key.split(".")[index_idx] + if ( + prefix + "encoders." + index + ".mlp.dense_h_to_4h.weight" + in oneflow_state_dict.keys() + ): + continue + if "weight" in key: + w = key + b = key.replace("weight", "bias") + new_key = prefix + "encoders." + index + ".mlp.dense_h_to_4h.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + + # Convert bert's output layers + elif "output" in key: + index = key.split(".")[index_idx] + if "dense.weight" in key: + if ( + prefix + "encoders." + index + ".mlp.dense_4h_to_h.weight" + in oneflow_state_dict.keys() + ): + continue + w = key + b = w.replace("weight", "bias") + new_key = prefix + "encoders." + index + ".mlp.dense_4h_to_h.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + elif "LayerNorm.weight" in key: + if ( + prefix + "encoders." + str(int(index) + 1) + ".input_layernorm.weight" + in oneflow_state_dict.keys() + ): + continue + w = key + b = w.replace("weight", "bias") + if index == str(layers - 1): + new_key = prefix + "final_layernorm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + continue + new_key = prefix + "encoders." + str(int(index) + 1) + ".input_layernorm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + + # Convert bert's pooler layers + elif "pooler" in key: + if "weight" in key: + new_key = prefix + "pooler.dense.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = prefix + "pooler.dense.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + # Convert cls_head layers + elif "cls" in key: + if "predictions.bias" in key: + new_key = "cls_head.lm_logits.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "dense.weight" in key: + new_key = "cls_head.predictions.dense.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "dense.bias" in key: + new_key = "cls_head.predictions.dense.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "LayerNorm.weight" in key: + new_key = "cls_head.predictions.layernorm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "LayerNorm.bias" in key: + new_key = "cls_head.predictions.layernorm.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "seq_relationship" in key: + new_key = key.replace("cls", "cls_head") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + else: + oneflow_state_dict[key] = oneflow_state_dict.pop(key) + return oneflow_state_dict + + def _load_config_from_json(self, config_file): + """load config from `config.json`, and update default config. + + Args: + config_file (str): Path of config file. + """ + with open(config_file, mode="r", encoding="utf-8") as f: + cfg_dict = json.load(f) + + # update libai_cfg by config.json + self._update_cfg("vocab_size", cfg_dict["vocab_size"]) + self._update_cfg("hidden_size", cfg_dict["hidden_size"]) + self._update_cfg("hidden_layers", cfg_dict["num_hidden_layers"]) + self._update_cfg("num_attention_heads", cfg_dict["num_attention_heads"]) + self._update_cfg("intermediate_size", cfg_dict["intermediate_size"]) + self._update_cfg("hidden_dropout_prob", cfg_dict["hidden_dropout_prob"]) + self._update_cfg("attention_probs_dropout_prob", cfg_dict["attention_probs_dropout_prob"]) + self._update_cfg("max_position_embeddings", cfg_dict["max_position_embeddings"]) + self._update_cfg("num_tokentypes", cfg_dict["type_vocab_size"]) + self._update_cfg("initializer_range", cfg_dict["initializer_range"]) + self._update_cfg("layernorm_eps", cfg_dict["layer_norm_eps"]) + + # update libai_cfg by kwargs + for k, v in self.kwargs.items(): + self._update_cfg(k, v) + + # use original BERT residual connection ordering + self.libai_cfg.apply_residual_post_layernorm = True + + self._update_cfg_log() + + +class BertLoaderLiBai(ModelLoaderLiBai): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + self.base_model_prefix_2 = "bert" diff --git a/libai/models/utils/model_utils/gpt_loader.py b/libai/models/utils/model_utils/gpt_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..85ae5224c1f9ccc01e0931673ad0a2feb0fe28f9 --- /dev/null +++ b/libai/models/utils/model_utils/gpt_loader.py @@ -0,0 +1,174 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from .base_loader import ModelLoaderHuggerFace, ModelLoaderLiBai + + +class GPT2LoaderHuggerFace(ModelLoaderHuggerFace): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + + """NOTE: base_model_prefix_1 is GPT's prefix in Transformers. + base_model_prefix_2 is GPT's prefix in LiBai.""" + self.base_model_prefix_1 = "transformer" + self.base_model_prefix_2 = "GPT_model" + + def _convert_state_dict(self, flow_state_dict, cfg): + """Convert state_dict's keys to match model. + + Args: + flow_state_dict (OrderedDict): model state dict. + cfg (dict): model's default config dict in LiBai. + + Returns: + OrderedDict: flow state dict. + """ + # The converted checkpoint. + oneflow_state_dict = flow_state_dict.copy() + old_keys = list(oneflow_state_dict.keys()) + + # Get configs + num_heads = cfg.get("num_attention_heads") + hidden_size = cfg.get("hidden_size") + head_size = int(hidden_size / num_heads) + + # prefix + has_prefix = any(s.startswith(self.base_model_prefix_1) for s in oneflow_state_dict) + prefix1 = self.base_model_prefix_1 + "." if has_prefix else "" + prefix2 = "GPT_model.transformer." + layer_idx = 2 if has_prefix else 1 + + # Convert Embedding layers. + new_key = "GPT_model.embeddings.token_embeddings.weight" + old_keys.remove(prefix1 + "wte.weight") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(prefix1 + "wte.weight") + + new_key = "GPT_model.embeddings.position_embeddings.weight" + old_keys.remove(prefix1 + "wpe.weight") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(prefix1 + "wpe.weight") + + for key in old_keys: + keys = key.split(".") + if layer_idx >= len(keys): + continue + layer = keys[layer_idx] + # Convert transformer layers. + if "h." in key: + if "ln_1" in key: + if "weight" in key: + new_key = prefix2 + "layers." + layer + ".input_layernorm.weight" + else: + new_key = prefix2 + "layers." + layer + ".input_layernorm.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "ln_2" in key: + if "weight" in key: + new_key = prefix2 + "layers." + layer + ".post_attention_layernorm.weight" + else: + new_key = prefix2 + "layers." + layer + ".post_attention_layernorm.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "attn" in key: + if "c_attn" in key: + if "weight" in key: + new_key = ( + prefix2 + + "layers." + + layer + + ".self_attention.query_key_value.weight" + ) + else: + new_key = ( + prefix2 + "layers." + layer + ".self_attention.query_key_value.bias" + ) + qkv = oneflow_state_dict.pop(key) + if qkv.ndim > 1: + qkv = qkv.transpose(1, 0) + qkv = self._fix_qkv_ordering(qkv, head_size, num_heads) + oneflow_state_dict[new_key] = qkv + elif "c_proj" in key: + if "weight" in key: + new_key = prefix2 + "layers." + layer + ".self_attention.dense.weight" + elif "bias" in key: + new_key = prefix2 + "layers." + layer + ".self_attention.dense.bias" + value = oneflow_state_dict.pop(key) + if value.ndim > 1: + value = value.transpose(1, 0) + oneflow_state_dict[new_key] = value + elif "mlp" in key: + if "c_fc" in key: + if "weight" in key: + new_key = prefix2 + "layers." + layer + ".mlp.dense_h_to_4h.weight" + elif "bias" in key: + new_key = prefix2 + "layers." + layer + ".mlp.dense_h_to_4h.bias" + value = oneflow_state_dict.pop(key) + if value.ndim > 1: + value = value.transpose(1, 0) + oneflow_state_dict[new_key] = value + elif "c_proj" in key: + if "weight" in key: + new_key = prefix2 + "layers." + layer + ".mlp.dense_4h_to_h.weight" + elif "bias" in key: + new_key = prefix2 + "layers." + layer + ".mlp.dense_4h_to_h.bias" + value = oneflow_state_dict.pop(key) + if value.ndim > 1: + value = value.transpose(1, 0) + oneflow_state_dict[new_key] = value + elif "ln_f" in key: + if "weight" in key: + new_key = prefix2 + "layernorm_f.weight" + elif "bias" in key: + new_key = prefix2 + "layernorm_f.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + return oneflow_state_dict + + def _load_config_from_json(self, config_file): + """load config from `config.json`, and update default config. + + Args: + config_file (str): Path of config file. + """ + with open(config_file, mode="r", encoding="utf-8") as f: + cfg_dict = json.load(f) + + # update libai_cfg by config.json + self._update_cfg("hidden_layers", cfg_dict["n_layer"]) + self._update_cfg("hidden_size", cfg_dict["n_embd"]) + self._update_cfg("num_attention_heads", cfg_dict["n_head"]) + self._update_cfg("max_seq_length", cfg_dict["n_positions"]) + self._update_cfg("embedding_dropout_prob", cfg_dict["embd_pdrop"]) + self._update_cfg("attention_dropout_prob", cfg_dict["attn_pdrop"]) + self._update_cfg("output_dropout_prob", cfg_dict["resid_pdrop"]) + self._update_cfg("layernorm_epsilon", cfg_dict["layer_norm_epsilon"]) + self._update_cfg("vocab_size", cfg_dict["vocab_size"]) + self._update_cfg("initializer_range", cfg_dict["initializer_range"]) + self._update_cfg( + "ffn_hidden_size", + cfg_dict.get("n_inner") + if cfg_dict.get("n_inner") is not None + else 4 * self.libai_cfg["hidden_size"], + ) + + # update libai_cfg by kwargs + for k, v in self.kwargs.items(): + self._update_cfg(k, v) + + self._update_cfg_log() + + +class GPT2LoaderLiBai(ModelLoaderLiBai): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + self.base_model_prefix_2 = "GPT_model" diff --git a/libai/models/utils/model_utils/roberta_loader.py b/libai/models/utils/model_utils/roberta_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..791149f0b97779abfe120f3f55bf7e26697e30fd --- /dev/null +++ b/libai/models/utils/model_utils/roberta_loader.py @@ -0,0 +1,226 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow + +from .bert_loader import BertLoaderHuggerFace, BertLoaderLiBai + + +class RobertaLoaderHuggerFace(BertLoaderHuggerFace): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + + """NOTE: base_model_prefix_1 is RoBERTa's prefix in Transformers, + base_model_prefix_2 is RoBERTa's prefix in LiBai.""" + self.base_model_prefix_1 = "roberta" + self.base_model_prefix_2 = "roberta" + + def _convert_state_dict(self, flow_state_dict, cfg): + """Convert state_dict's keys to match model. + + Args: + flow_state_dict (OrderedDict): model state dict. + cfg (dict): model's default config dict in LiBai. + + Returns: + OrderedDict: flow state dict. + """ + # The converted checkpoint. + oneflow_state_dict = flow_state_dict.copy() + + # Get configs + num_heads = cfg.get("num_attention_heads") + hidden_size = cfg.get("hidden_size") + layers = cfg.get("hidden_layers") + head_size = int(hidden_size / num_heads) + + # prefix + has_prefix = any(s.startswith(self.base_model_prefix_1) for s in oneflow_state_dict) + + prefix = "roberta." if has_prefix else "" + index_idx = 3 if has_prefix else 2 + qkv_idx = 6 if has_prefix else 5 + + old_keys = oneflow_state_dict.keys() + + for key in list(old_keys): + + # Convert roberta's embedding layers + if "embeddings" in key: + if "word_embeddings" in key: + new_key = key.replace("word_embeddings", "vocab_embeddings") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "token_type_embeddings" in key: + new_key = key.replace("token_type_embeddings", "tokentype_embeddings") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "LayerNorm.weight" in key: + new_key = prefix + "encoders.0.input_layernorm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "LayerNorm.bias" in key: + new_key = prefix + "encoders.0.input_layernorm.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + else: + oneflow_state_dict[key] = oneflow_state_dict[key] + + # Convert roberta's attention layers + elif "attention" in key: + if "self" in key: + index = key.split(".")[index_idx] + if ( + prefix + "encoders." + index + ".self_attention.query_key_value.weight" + in oneflow_state_dict.keys() + ): + continue + q_w = key.replace(key.split(".")[qkv_idx], "query").replace( + key.split(".")[qkv_idx + 1], "weight" + ) + k_w = q_w.replace("query", "key") + v_w = q_w.replace("query", "value") + q_b = q_w.replace("weight", "bias") + k_b = k_w.replace("weight", "bias") + v_b = v_w.replace("weight", "bias") + + qkv_w = flow.cat( + ( + oneflow_state_dict.pop(q_w), + oneflow_state_dict.pop(k_w), + oneflow_state_dict.pop(v_w), + ), + dim=0, + ) + qkv_b = flow.cat( + ( + oneflow_state_dict.pop(q_b), + oneflow_state_dict.pop(k_b), + oneflow_state_dict.pop(v_b), + ), + dim=-1, + ) + + qkv_w = self._fix_qkv_ordering(qkv_w, head_size, num_heads) + qkv_b = self._fix_qkv_ordering(qkv_b, head_size, num_heads) + + new_key = ( + prefix + "encoders." + index + ".self_attention.query_key_value.weight" + ) + oneflow_state_dict[new_key] = qkv_w + + new_key = prefix + "encoders." + index + ".self_attention.query_key_value.bias" + oneflow_state_dict[new_key] = qkv_b + elif "output" in key: + index = key.split(".")[index_idx] + if "dense" in key: + if "weight" in key: + new_key = prefix + "encoders." + index + ".self_attention.dense.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = prefix + "encoders." + index + ".self_attention.dense.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "LayerNorm" in key: + if "weight" in key: + new_key = ( + prefix + "encoders." + index + ".post_attention_layernorm.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = ( + prefix + "encoders." + index + ".post_attention_layernorm.bias" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + # Convert roberta's intermediate layers + elif "intermediate" in key: + index = key.split(".")[index_idx] + if ( + prefix + "encoders." + index + ".mlp.dense_h_to_4h.weight" + in oneflow_state_dict.keys() + ): + continue + if "weight" in key: + w = key + b = key.replace("weight", "bias") + new_key = prefix + "encoders." + index + ".mlp.dense_h_to_4h.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + + # Convert roberta's output layers + elif "output" in key: + index = key.split(".")[index_idx] + if "dense.weight" in key: + if ( + prefix + "encoders." + index + ".mlp.dense_4h_to_h.weight" + in oneflow_state_dict.keys() + ): + continue + w = key + b = w.replace("weight", "bias") + new_key = prefix + "encoders." + index + ".mlp.dense_4h_to_h.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + elif "LayerNorm.weight" in key: + if ( + prefix + "encoders." + str(int(index) + 1) + ".input_layernorm.weight" + in oneflow_state_dict.keys() + ): + continue + w = key + b = w.replace("weight", "bias") + if index == str(layers - 1): + new_key = prefix + "final_layernorm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + continue + new_key = prefix + "encoders." + str(int(index) + 1) + ".input_layernorm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + + # Convert roberta's pooler layers + elif "pooler" in key: + if "weight" in key: + new_key = prefix + "pooler.dense.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = prefix + "pooler.dense.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + # Convert lm_head layers + elif "lm_head" in key: + if "layer_norm.weight" in key: + new_key = "lm_head.layernorm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "layer_norm.bias" in key: + new_key = "lm_head.layernorm.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "seq_relationship" in key: + new_key = key.replace("cls", "cls_head") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "lm_head.bias" in key: + new_key = "lm_head.lm_logits.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + else: + oneflow_state_dict[key] = oneflow_state_dict.pop(key) + else: + oneflow_state_dict[key] = oneflow_state_dict.pop(key) + return oneflow_state_dict + + +class RobertaLoaderLiBai(BertLoaderLiBai): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + self.base_model_prefix_2 = "roberta" diff --git a/libai/models/utils/model_utils/swin_loader.py b/libai/models/utils/model_utils/swin_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..b4a3b3caee025d14e2245626cfb5cb65faa39a4d --- /dev/null +++ b/libai/models/utils/model_utils/swin_loader.py @@ -0,0 +1,298 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import oneflow as flow + +from .base_loader import ModelLoaderHuggerFace, ModelLoaderLiBai + + +class SwinLoaderHuggerFace(ModelLoaderHuggerFace): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + + """NOTE: base_model_prefix_1 is SWIN's prefix in Transformers. + base_model_prefix_2 is SWIN's prefix in LiBai.""" + + self.base_model_prefix_1 = "swin" + self.base_model_prefix_2 = "" + + def _convert_state_dict(self, flow_state_dict, cfg=None): + """Convert state_dict's keys to match model. + + Args: + flow_state_dict (OrderedDict): model state dict. + cfg (dict): model's default config dict. + + Returns: + OrderedDict: flow state dict. + """ + # The converted checkpoint. + oneflow_state_dict = flow_state_dict.copy() + + # prefix + has_prefix = any(s.startswith(self.base_model_prefix_1) for s in oneflow_state_dict) + + index_idx_1 = 3 if has_prefix else 2 + index_idx_2 = 5 if has_prefix else 4 + + old_keys = oneflow_state_dict.keys() + + for key in list(old_keys): + + # Convert swin's embedding layers + if "embeddings" in key: + if "patch_embeddings.projection" in key: + if "weight" in key: + new_key = "patch_embed.proj.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = "patch_embed.proj.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "norm" in key: + if "weight" in key: + new_key = "patch_embed.norm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = "patch_embed.norm.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + # Convert swin's layernorm layers + elif "layernorm_before" in key: + index_layer = key.split(".")[index_idx_1] + index_block = key.split(".")[index_idx_2] + if "weight" in key: + new_key = "layers." + index_layer + ".blocks." + index_block + ".norm1.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = "layers." + index_layer + ".blocks." + index_block + ".norm1.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + elif "layernorm_after" in key: + index_layer = key.split(".")[index_idx_1] + index_block = key.split(".")[index_idx_2] + if "weight" in key: + new_key = "layers." + index_layer + ".blocks." + index_block + ".norm2.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = "layers." + index_layer + ".blocks." + index_block + ".norm2.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + # Convert swin's attention layers + elif "attention" in key: + index_layer = key.split(".")[index_idx_1] + index_block = key.split(".")[index_idx_2] + if "self" in key: + if ( + "relative_position_bias_table" in key + ): # convert relative_position_bias_table but not index + new_key = ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".attn.relative_position_bias_table" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "relative_position_index" in key: + new_key = ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".attn.relative_position_index" + ) + oneflow_state_dict.pop(key) + else: + if ( + "layers." + index_layer + ".blocks." + index_block + ".attn.qkv.weight" + in oneflow_state_dict.keys() + ): + continue + q_w = key + k_w = q_w.replace("query", "key") + v_w = q_w.replace("query", "value") + q_b = q_w.replace("weight", "bias") + k_b = k_w.replace("weight", "bias") + v_b = v_w.replace("weight", "bias") + + qkv_w = flow.cat( + ( + oneflow_state_dict.pop(q_w), + oneflow_state_dict.pop(k_w), + oneflow_state_dict.pop(v_w), + ), + dim=0, + ) + qkv_b = flow.cat( + ( + oneflow_state_dict.pop(q_b), + oneflow_state_dict.pop(k_b), + oneflow_state_dict.pop(v_b), + ), + dim=-1, + ) + + new_key = ( + "layers." + index_layer + ".blocks." + index_block + ".attn.qkv.weight" + ) + oneflow_state_dict[new_key] = qkv_w + + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = qkv_b + + elif "output" in key: + if "dense" in key: + if "weight" in key: + new_key = ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".attn.proj.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + if "bias" in key: + new_key = ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".attn.proj.bias" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + elif "intermediate" in key: + index_layer = key.split(".")[index_idx_1] + index_block = key.split(".")[index_idx_2] + if "weight" in key: + if ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".mlp.dense_h_to_4h.weight" + in oneflow_state_dict.keys() + ): + continue + w = key + b = key.replace("weight", "bias") + new_key = ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".mlp.dense_h_to_4h.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + + elif "output" in key: + index_layer = key.split(".")[index_idx_1] + index_block = key.split(".")[index_idx_2] + if "dense.weight" in key: + if ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".mlp.dense_4h_to_h.weight" + in oneflow_state_dict.keys() + ): + continue + w = key + b = w.replace("weight", "bias") + new_key = ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".mlp.dense_4h_to_h.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + + elif "downsample" in key: + index_layer = key.split(".")[index_idx_1] + if "reduction.weight" in key: + new_key = "layers." + index_layer + ".downsample.reduction.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "norm" in key: + if ( + "layers." + index_layer + ".downsample.norm.weight" + in oneflow_state_dict.keys() + ): + continue + w = key + b = w.replace("weight", "bias") + new_key = "layers." + index_layer + ".downsample.norm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + + elif "layernorm" in key: + if "weight" in key: + new_key = "norm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = "norm.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + elif "classifier" in key: + if "weight" in key: + new_key = "head.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = "head.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + else: + oneflow_state_dict[key] = oneflow_state_dict.pop(key) + + return oneflow_state_dict + + def _load_config_from_json(self, config_file): + """load config from `config.json`, and update default config. + + Args: + config_file (str): Path of config file. + """ + with open(config_file, mode="r", encoding="utf-8") as f: + cfg_dict = json.load(f) + + # update libai_cfg by config.json + self._update_cfg("img_size", cfg_dict["image_size"]) + self._update_cfg("patch_size", cfg_dict["patch_size"]) + self._update_cfg("embed_dim", cfg_dict["embed_dim"]) + self._update_cfg("depths", cfg_dict["depths"]) + self._update_cfg("num_heads", cfg_dict["num_heads"]) + self._update_cfg("window_size", cfg_dict["window_size"]) + self._update_cfg("mlp_ratio", cfg_dict["mlp_ratio"]) + self._update_cfg("qkv_bias", cfg_dict["qkv_bias"]) + self._update_cfg("drop_path_rate", cfg_dict["drop_path_rate"]) + + # update libai_cfg by kwargs + for k, v in self.kwargs.items(): + self._update_cfg(k, v) + + self._update_cfg_log() + + +class SwinLoaderLiBai(ModelLoaderLiBai): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + self.base_model_prefix_2 = "" diff --git a/libai/models/utils/model_utils/swinv2_loader.py b/libai/models/utils/model_utils/swinv2_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..04a08a0e1f22c7650d47739a2b891719c26e342b --- /dev/null +++ b/libai/models/utils/model_utils/swinv2_loader.py @@ -0,0 +1,316 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import oneflow as flow + +from .base_loader import ModelLoaderHuggerFace, ModelLoaderLiBai + + +class SwinV2LoaderHuggerFace(ModelLoaderHuggerFace): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + + """NOTE: base_model_prefix_1 is SWINV2's prefix in Transformers. + base_model_prefix_2 is SWINV2's prefix in LiBai.""" + + self.base_model_prefix_1 = "swinv2" + self.base_model_prefix_2 = "" + + def _convert_state_dict(self, flow_state_dict, cfg=None): + """Convert state_dict's keys to match model. + + Args: + flow_state_dict (OrderedDict): model state dict. + cfg (dict): model's default config dict. + + Returns: + OrderedDict: flow state dict. + """ + # The converted checkpoint. + oneflow_state_dict = flow_state_dict.copy() + + # prefix + has_prefix = any(s.startswith(self.base_model_prefix_1) for s in oneflow_state_dict) + index_idx_1 = 3 if has_prefix else 2 + index_idx_2 = 5 if has_prefix else 4 + + old_keys = oneflow_state_dict.keys() + + for key in list(old_keys): + # Convert swinv2's embedding layers + if "embeddings" in key: + if "patch_embeddings.projection" in key: + if "weight" in key: + new_key = "patch_embed.proj.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + if "bias" in key: + new_key = "patch_embed.proj.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "norm" in key: + if "weight" in key: + new_key = "patch_embed.norm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + if "bias" in key: + new_key = "patch_embed.norm.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + # Convert swinv2's layernorm layers + elif "layernorm_before" in key: + index_layer = key.split(".")[index_idx_1] + index_block = key.split(".")[index_idx_2] + if "weight" in key: + new_key = "layers." + index_layer + ".blocks." + index_block + ".norm1.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = "layers." + index_layer + ".blocks." + index_block + ".norm1.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + elif "layernorm_after" in key: + index_layer = key.split(".")[index_idx_1] + index_block = key.split(".")[index_idx_2] + if "weight" in key: + new_key = "layers." + index_layer + ".blocks." + index_block + ".norm2.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = "layers." + index_layer + ".blocks." + index_block + ".norm2.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + # Convert swinv2's attention layers + elif "attention" in key: + index_layer = key.split(".")[index_idx_1] + index_block = key.split(".")[index_idx_2] + if "self" in key: + if ( + "relative_position_bias_table" in key + ): # convert relative_position_bias_table but not index + new_key = ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".attn.relative_position_bias_table" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "relative_position_index" in key: + new_key = ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".attn.relative_position_index" + ) + oneflow_state_dict.pop(key) + elif "continuous_position_bias_mlp" in key: + if ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".attn.cpb_mlp" + + ".0.weight" + ) in oneflow_state_dict.keys(): + continue + new_key = ( + "layers." + index_layer + ".blocks." + index_block + ".attn.cpb_mlp" + ) + m_1_w = key + m_1_b = key.replace(".0.weight", ".0.bias") + m_2_w = key.replace(".0.weight", ".2.weight") + oneflow_state_dict[new_key + ".0.weight"] = oneflow_state_dict.pop(m_1_w) + oneflow_state_dict[new_key + ".0.bias"] = oneflow_state_dict.pop(m_1_b) + oneflow_state_dict[new_key + ".2.weight"] = oneflow_state_dict.pop(m_2_w) + elif "logit_scale" in key: + new_key = ( + "layers." + index_layer + ".blocks." + index_block + ".attn.logit_scale" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key)[None, ...] + else: + if ( + "layers." + index_layer + ".blocks." + index_block + ".attn.qkv.weight" + in oneflow_state_dict.keys() + ): + continue + q_w = key + k_w = q_w.replace("query", "key") + v_w = q_w.replace("query", "value") + q_b = q_w.replace("weight", "bias") + v_b = v_w.replace("weight", "bias") + + qkv_w = flow.cat( + ( + oneflow_state_dict.pop(q_w), + oneflow_state_dict.pop(k_w), + oneflow_state_dict.pop(v_w), + ), + dim=0, + ) + + new_key = ( + "layers." + index_layer + ".blocks." + index_block + ".attn.qkv.weight" + ) + oneflow_state_dict[new_key] = qkv_w + + new_key = ( + "layers." + index_layer + ".blocks." + index_block + ".attn.q_bias" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(q_b) + new_key = new_key.replace("q_bias", "v_bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(v_b) + + elif "output" in key: + if "dense" in key: + if "weight" in key: + new_key = ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".attn.proj.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + if "bias" in key: + new_key = ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".attn.proj.bias" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + elif "intermediate" in key: + index_layer = key.split(".")[index_idx_1] + index_block = key.split(".")[index_idx_2] + if "weight" in key: + if ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".mlp.dense_h_to_4h.weight" + in oneflow_state_dict.keys() + ): + continue + w = key + b = key.replace("weight", "bias") + new_key = ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".mlp.dense_h_to_4h.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + + elif "output" in key: + index_layer = key.split(".")[index_idx_1] + index_block = key.split(".")[index_idx_2] + if "dense.weight" in key: + if ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".mlp.dense_4h_to_h.weight" + in oneflow_state_dict.keys() + ): + continue + w = key + b = w.replace("weight", "bias") + new_key = ( + "layers." + + index_layer + + ".blocks." + + index_block + + ".mlp.dense_4h_to_h.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + + elif "downsample" in key: + index_layer = key.split(".")[index_idx_1] + if "reduction.weight" in key: + new_key = "layers." + index_layer + ".downsample.reduction.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "norm" in key: + if ( + "layers." + index_layer + ".downsample.norm.weight" + in oneflow_state_dict.keys() + ): + continue + w = key + b = w.replace("weight", "bias") + new_key = "layers." + index_layer + ".downsample.norm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + + elif "layernorm" in key: + if "weight" in key: + new_key = "norm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = "norm.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + elif "classifier" in key: + if "weight" in key: + new_key = "head.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = "head.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + else: + oneflow_state_dict[key] = oneflow_state_dict.pop(key) + + return oneflow_state_dict + + def _load_config_from_json(self, config_file): + """load config from `config.json`, and update default config. + + Args: + config_file (str): Path of config file. + """ + with open(config_file, mode="r", encoding="utf-8") as f: + cfg_dict = json.load(f) + + # update libai_cfg by config.json + self._update_cfg("img_size", cfg_dict["image_size"]) + self._update_cfg("patch_size", cfg_dict["patch_size"]) + self._update_cfg("embed_dim", cfg_dict["embed_dim"]) + self._update_cfg("depths", cfg_dict["depths"]) + self._update_cfg("num_heads", cfg_dict["num_heads"]) + self._update_cfg("window_size", cfg_dict["window_size"]) + self._update_cfg("mlp_ratio", cfg_dict["mlp_ratio"]) + self._update_cfg("qkv_bias", cfg_dict["qkv_bias"]) + self._update_cfg("drop_path_rate", cfg_dict["drop_path_rate"]) + self._update_cfg("pretrained_window_sizes", cfg_dict["pretrained_window_sizes"]) + + # update libai_cfg by kwargs + for k, v in self.kwargs.items(): + self._update_cfg(k, v) + + self._update_cfg_log() + + +class SwinV2LoaderLiBai(ModelLoaderLiBai): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + self.base_model_prefix_2 = "" diff --git a/libai/models/utils/model_utils/vit_loader.py b/libai/models/utils/model_utils/vit_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..7a35ad52833b4b2addf229cc0196eaed4c351da9 --- /dev/null +++ b/libai/models/utils/model_utils/vit_loader.py @@ -0,0 +1,225 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import oneflow as flow + +from .base_loader import ModelLoaderHuggerFace, ModelLoaderLiBai + + +class ViTLoaderHuggerFace(ModelLoaderHuggerFace): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + + """NOTE: base_model_prefix_1 is ViT's prefix in Transformers. + base_model_prefix_2 is ViT's prefix in LiBai.""" + + self.base_model_prefix_1 = "vit" + self.base_model_prefix_2 = "" + + def _convert_state_dict(self, flow_state_dict, cfg=None): + """Convert state_dict's keys to match model. + + Args: + flow_state_dict (OrderedDict): model state dict. + cfg (dict): model's default config dict. + + Returns: + OrderedDict: flow state dict. + """ + # The converted checkpoint. + oneflow_state_dict = flow_state_dict.copy() + + # Get configs + num_heads = cfg.get("num_heads") + hidden_size = cfg.get("embed_dim") + head_size = int(hidden_size / num_heads) + + # prefix + has_prefix = any(s.startswith(self.base_model_prefix_1) for s in oneflow_state_dict) + + index_idx = 3 if has_prefix else 2 + + old_keys = oneflow_state_dict.keys() + + for key in list(old_keys): + + # Convert vit's embedding layers + if "embeddings" in key: + if "cls_token" in key: + new_key = "cls_token" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "position_embeddings" in key: + new_key = "pos_embed" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "patch_embeddings.projection" in key: + if "weight" in key: + new_key = "patch_embed.proj.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = "patch_embed.proj.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + # Convert vit's layernorm layers + elif "layernorm_before" in key: + index_block = key.split(".")[index_idx] + if "weight" in key: + new_key = "blocks." + index_block + ".input_layernorm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = "blocks." + index_block + ".input_layernorm.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + elif "layernorm_after" in key: + index_block = key.split(".")[index_idx] + if "weight" in key: + new_key = "blocks." + index_block + ".post_attention_layernorm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = "blocks." + index_block + ".post_attention_layernorm.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + # Convert vit's attention layers + elif "attention" in key: + index_block = key.split(".")[index_idx] + if "attention.attention" in key: + if ( + "blocks." + index_block + ".self_attention.query_key_value.weight" + in oneflow_state_dict.keys() + ): + continue + q_w = key + k_w = q_w.replace("query", "key") + v_w = q_w.replace("query", "value") + q_b = q_w.replace("weight", "bias") + k_b = k_w.replace("weight", "bias") + v_b = v_w.replace("weight", "bias") + + qkv_w = flow.cat( + ( + oneflow_state_dict.pop(q_w), + oneflow_state_dict.pop(k_w), + oneflow_state_dict.pop(v_w), + ), + dim=0, + ) + qkv_b = flow.cat( + ( + oneflow_state_dict.pop(q_b), + oneflow_state_dict.pop(k_b), + oneflow_state_dict.pop(v_b), + ), + dim=-1, + ) + + qkv_w = self._fix_qkv_ordering(qkv_w, head_size, num_heads) + qkv_b = self._fix_qkv_ordering(qkv_b, head_size, num_heads) + + new_key = "blocks." + index_block + ".self_attention.query_key_value.weight" + oneflow_state_dict[new_key] = qkv_w + + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = qkv_b + + elif "output" in key: + if "dense" in key: + if "weight" in key: + new_key = "blocks." + index_block + ".self_attention.dense.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + if "bias" in key: + new_key = "blocks." + index_block + ".self_attention.dense.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + elif "intermediate" in key: + index_block = key.split(".")[index_idx] + if "weight" in key: + if ( + "blocks." + index_block + ".mlp.dense_h_to_4h.weight" + in oneflow_state_dict.keys() + ): + continue + w = key + b = key.replace("weight", "bias") + new_key = "blocks." + index_block + ".mlp.dense_h_to_4h.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + + elif "output" in key: + index_block = key.split(".")[index_idx] + if "dense.weight" in key: + if ( + "blocks." + index_block + ".mlp.dense_4h_to_h.weight" + in oneflow_state_dict.keys() + ): + continue + w = key + b = w.replace("weight", "bias") + new_key = "blocks." + index_block + ".mlp.dense_4h_to_h.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(w) + new_key = new_key.replace("weight", "bias") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(b) + + elif "layernorm" in key: + if "weight" in key: + new_key = "norm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = "norm.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + elif "classifier" in key: + if "weight" in key: + new_key = "head.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif "bias" in key: + new_key = "head.bias" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + else: + oneflow_state_dict[key] = oneflow_state_dict.pop(key) + + return oneflow_state_dict + + def _load_config_from_json(self, config_file): + """load config from `config.json`, and update default config. + + Args: + config_file (str): Path of config file. + """ + with open(config_file, mode="r", encoding="utf-8") as f: + cfg_dict = json.load(f) + + # update libai_cfg by config.json + self._update_cfg("img_size", cfg_dict["image_size"]) + self._update_cfg("patch_size", cfg_dict["patch_size"]) + self._update_cfg("in_chans", cfg_dict["num_channels"]) + self._update_cfg("embed_dim", cfg_dict["hidden_size"]) + self._update_cfg("depth", cfg_dict["num_hidden_layers"]) + self._update_cfg("num_heads", cfg_dict["num_attention_heads"]) + self._update_cfg("attn_drop_rate", cfg_dict["attention_probs_dropout_prob"]) + self._update_cfg("drop_rate", cfg_dict["hidden_dropout_prob"]) + + # update libai_cfg by kwargs + for k, v in self.kwargs.items(): + self._update_cfg(k, v) + + self._update_cfg_log() + + +class ViTLoaderLiBai(ModelLoaderLiBai): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + self.base_model_prefix_2 = "" diff --git a/libai/models/utils/weight_init.py b/libai/models/utils/weight_init.py new file mode 100644 index 0000000000000000000000000000000000000000..6bb1ccd6564b0c5a8e8112ba43f3e7764127dc03 --- /dev/null +++ b/libai/models/utils/weight_init.py @@ -0,0 +1,37 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import oneflow.nn as nn + + +def init_method_normal(sigma, mean=0.0): + """Init method based on N(0, sigma).""" + + def init_(tensor): + return nn.init.normal_(tensor, mean=mean, std=sigma) + + return init_ + + +def scaled_init_method_normal(sigma, num_layers, mean=0.0): + """Init method based on N(0, sigma/sqrt(2*num_layers).""" + std = sigma / math.sqrt(2.0 * num_layers) + + def init_(tensor): + return nn.init.normal_(tensor, mean=mean, std=std) + + return init_ diff --git a/libai/models/vision_transformer.py b/libai/models/vision_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..aa6b920955dbc0e3fbee0cd740883223cfd244cb --- /dev/null +++ b/libai/models/vision_transformer.py @@ -0,0 +1,267 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +import oneflow.nn as nn +from flowvision.layers.weight_init import trunc_normal_ + +import libai.utils.distributed as dist +from libai.config.config import configurable +from libai.layers import LayerNorm, Linear, PatchEmbedding, TransformerLayer + + +class VisionTransformer(nn.Module): + """Vision Transformer in LiBai. + + LiBai's implementation of: + `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale + `_ + + Args: + img_size (int, tuple(int)): input image size + patch_size (int, tuple(int)): patch size + in_chans (int): number of input channels + embed_dim (int): embedding dimension + depth (int): depth of transformer + num_heads (int): number of attention heads + mlp_ratio (int): ratio of mlp hidden dim to embedding dim + drop_rate (float): dropout rate + attn_drop_rate (float): attention dropout rate + drop_path_rate (float): stochastic depth rate + num_classes (int): number of classes for classification head + loss_func (callable, optional): loss function for computing the total loss + between logits and labels + """ + + @configurable + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=192, + depth=12, + num_heads=3, + mlp_ratio=4.0, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.0, + num_classes=1000, + loss_func=None, + ): + super().__init__() + self.img_size = img_size + self.num_classes = num_classes + self.patch_embed = PatchEmbedding( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + ) + ffn_size = int(embed_dim * mlp_ratio) + num_patches = self.patch_embed.num_patches + self.cls_token = nn.Parameter( + flow.zeros( + 1, + 1, + embed_dim, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + ) + self.pos_embed = nn.Parameter( + flow.zeros( + 1, + num_patches + 1, + embed_dim, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + ) + + self.pos_drop = nn.Dropout(p=drop_rate) + dpr = [ + x.item() for x in flow.linspace(0, drop_path_rate, depth) + ] # stochastic depth decay rule + self.blocks = nn.Sequential( + *[ + TransformerLayer( + hidden_size=embed_dim, + ffn_hidden_size=ffn_size, + num_attention_heads=num_heads, + attention_dropout_prob=attn_drop_rate, + output_dropout_prob=drop_rate, + drop_path_prob=dpr[i], + layer_idx=i, + ) + for i in range(depth) + ] + ) + self.norm = LayerNorm(embed_dim, layer_idx=-1) + self.head = Linear(embed_dim, num_classes, layer_idx=-1) + + # loss func + self.loss_func = nn.CrossEntropyLoss() if loss_func is None else loss_func + + # weight init + trunc_normal_(self.pos_embed, std=0.02) + trunc_normal_(self.cls_token, std=0.02) + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, Linear): + trunc_normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def no_weight_decay(self): + return {"pos_embed", "cls_token"} + + @classmethod + def from_config(cls, cfg): + return { + "img_size": cfg.img_size, + "patch_size": cfg.patch_size, + "in_chans": cfg.in_chans, + "embed_dim": cfg.embed_dim, + "depth": cfg.depth, + "num_heads": cfg.num_heads, + "mlp_ratio": cfg.mlp_ratio, + "drop_rate": cfg.drop_rate, + "attn_drop_rate": cfg.attn_drop_rate, + "drop_path_rate": cfg.drop_path_rate, + "num_classes": cfg.num_classes, + "loss_func": cfg.loss_func, + } + + def forward_features(self, x): + # patch embedding + x = self.patch_embed(x) + + cls_token = self.cls_token.expand( + x.shape[0], -1, -1 + ) # stole cls_tokens impl from Phil Wang, thanks + cls_token = cls_token.to_global(sbp=x.sbp, placement=cls_token.placement) + x = flow.cat((cls_token, x), dim=1) + + # position embedding + pos_embed = self.pos_embed.expand(x.shape[0], -1, -1) + pos_embed = pos_embed.to_global(sbp=x.sbp, placement=pos_embed.placement) + x = self.pos_drop(x + pos_embed) + + # transformer block + x = self.blocks(x) + return x + + def forward_head(self, x): + x = self.norm(x) + outcome = x[:, 0] + outcome = self.head(outcome) + return outcome + + def forward(self, images, labels=None): + """ + + Args: + images (flow.Tensor): training samples. + labels (flow.LongTensor, optional): training targets + + Returns: + dict: + A dict containing :code:`loss_value` or :code:`logits` + depending on training or evaluation mode. + :code:`{"losses": loss_value}` when training, + :code:`{"prediction_scores": logits}` when evaluating. + """ + x = self.forward_features(images) + x = self.forward_head(x) + + if labels is not None and self.training: + losses = self.loss_func(x, labels) + return {"losses": losses} + else: + return {"prediction_scores": x} + + @staticmethod + def set_pipeline_stage_id(model): + dist_utils = dist.get_dist_util() + + # Set pipeline parallelism stage_id + if hasattr(model.pos_embed, "config"): + # Old API in OneFlow 0.8 + for module_block in model.modules(): + if isinstance(module_block.origin, PatchEmbedding): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, TransformerLayer): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + + # Set pos_embed and cls_token stage id + model.pos_embed.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + model.cls_token.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + model.pos_drop.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + model.norm.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.head.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.loss_func.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + else: + for module_block in model.modules(): + if isinstance(module_block.to(nn.Module), PatchEmbedding): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + + # Set pos_embed and cls_token stage id + model.pos_embed.to(flow.nn.graph.GraphTensor).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + model.cls_token.to(flow.nn.graph.GraphTensor).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + model.pos_drop.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + model.norm.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.head.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + model.loss_func.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) diff --git a/libai/onnx_export/gpt2_to_onnx.py b/libai/onnx_export/gpt2_to_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..3ce8c839e8f5b7727b804a39a87404315651035b --- /dev/null +++ b/libai/onnx_export/gpt2_to_onnx.py @@ -0,0 +1,86 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import oneflow as flow +from oneflow import nn +from oneflow_onnx.oneflow2onnx.util import convert_to_onnx_and_check + +from libai.config import LazyConfig +from libai.models.utils import GPT2LoaderLiBai +from projects.MagicPrompt.gpt2 import GPTModel + + +def get_model(config_file): + cfg = LazyConfig.load(config_file) + + cfg.model.cfg.pretrained_model_path = None + cfg.dataloader = None + cfg.tokenization = None + + print("Building model....") + loader = GPT2LoaderLiBai(GPTModel, cfg.cfg, "/path/to/model") + model = loader.load() + print("Build model finished.") + + return model + + +class gpt2Graph(nn.Graph): + def __init__(self, eager_model): + super().__init__() + self.model = eager_model + + def build( + self, + input_ids, + ): + out = self.model( + input_ids, + ) + return out + + +if __name__ == "__main__": + model = get_model("projects/MagicPrompt/configs/gpt2_inference.py") + model.eval() + + gpt2_graph = gpt2Graph(model) + # Build the static graph model + input_ids = flow.ones( + 1, 5, dtype=flow.int64, sbp=flow.sbp.broadcast, placement=flow.placement("cuda", ranks=[0]) + ) + + # check your model.forward is valid + # output = gpt2_graph( + # input_ids + # ) + + print("Compiling the graph which may make some time, please wait for a moment....") + + gpt2_graph._compile( + input_ids, + ) + + convert_to_onnx_and_check( + gpt2_graph, + external_data=False, + opset=11, + flow_weight_dir=None, + onnx_model_path="./", + dynamic_batch_size=False, + device="gpu_global", + input_tensor_range=[0, 10], + ) diff --git a/libai/onnx_export/onnx_inference/gpt2_onnx_infer.py b/libai/onnx_export/onnx_inference/gpt2_onnx_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..72c8addd90257637c20941ddc89a239a94d7624d --- /dev/null +++ b/libai/onnx_export/onnx_inference/gpt2_onnx_infer.py @@ -0,0 +1,64 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from collections import OrderedDict +from typing import List + +import numpy as np +import onnxruntime as ort + + +class OnnxModel: + def __init__( + self, + onnx_filename, + providers: List[str] = None, + ort_optimize: bool = True, + ): + ort_sess_opt = ort.SessionOptions() + ort_sess_opt.graph_optimization_level = ( + ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED + if ort_optimize + else ort.GraphOptimizationLevel.ORT_DISABLE_ALL + ) + if providers is None: + if ort.__version__ > "1.9.0": + providers = [ + "TensorrtExecutionProvider", + "CUDAExecutionProvider", + "CPUExecutionProvider", + ] + else: + providers = ["CPUExecutionProvider"] + self.sess = ort.InferenceSession( + onnx_filename, sess_options=ort_sess_opt, providers=providers + ) + + def forward(self, input_list): + ipt_dict = OrderedDict() + for idx, ipt in enumerate(self.sess.get_inputs()): + ipt_dict[ipt.name] = input_list[idx] + onnx_res = self.sess.run([], ipt_dict) + return onnx_res + + +if __name__ == "__main__": + onnx_model = OnnxModel("model.onnx") + input_list = [ + np.ones((1, 5)).astype(np.int64).astype(np.int64), + ] + + print(onnx_model.forward(input_list)) diff --git a/libai/onnx_export/onnx_inference/t5_onnx_infer.py b/libai/onnx_export/onnx_inference/t5_onnx_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..b07d44cd34115416af2de5029ecd387bcd587764 --- /dev/null +++ b/libai/onnx_export/onnx_inference/t5_onnx_infer.py @@ -0,0 +1,68 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from collections import OrderedDict +from typing import List + +import numpy as np +import onnxruntime as ort + + +class OnnxModel: + def __init__( + self, + onnx_filename, + providers: List[str] = None, + ort_optimize: bool = True, + ): + ort_sess_opt = ort.SessionOptions() + ort_sess_opt.graph_optimization_level = ( + ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED + if ort_optimize + else ort.GraphOptimizationLevel.ORT_DISABLE_ALL + ) + if providers is None: + if ort.__version__ > "1.9.0": + providers = [ + "TensorrtExecutionProvider", + "CUDAExecutionProvider", + "CPUExecutionProvider", + ] + else: + providers = ["CPUExecutionProvider"] + self.sess = ort.InferenceSession( + onnx_filename, sess_options=ort_sess_opt, providers=providers + ) + + def forward(self, input_list): + ipt_dict = OrderedDict() + for idx, ipt in enumerate(self.sess.get_inputs()): + ipt_dict[ipt.name] = input_list[idx] + onnx_res = self.sess.run([], ipt_dict) + return onnx_res + + +if __name__ == "__main__": + onnx_model = OnnxModel("model.onnx") + input_list = [ + np.ones((1, 5)).astype(np.int64), + np.ones((1, 3)).astype(np.int64), + np.ones((1, 5, 5)).astype(bool), + np.ones((1, 3, 3)).astype(bool), + np.ones((1, 3, 5)).astype(bool), + ] + + print(onnx_model.forward(input_list)) diff --git a/libai/onnx_export/t5_to_onnx.py b/libai/onnx_export/t5_to_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..1059908851f5c2845491e087176964823a46f445 --- /dev/null +++ b/libai/onnx_export/t5_to_onnx.py @@ -0,0 +1,130 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import oneflow as flow +from oneflow import nn +from oneflow_onnx.oneflow2onnx.util import convert_to_onnx_and_check + +from libai.config import LazyConfig +from projects.MT5.mt5_model import MT5Model +from projects.MT5.utils.mt5_loader import T5LoaderHuggerFace + + +def get_model(config_file): + cfg = LazyConfig.load(config_file) + + cfg.model.cfg.model_type = "mt5" + cfg.model.cfg.pretrained_model_path = None + cfg.dataloader = None + cfg.tokenization = None + + print("Building model....") + loader = T5LoaderHuggerFace(MT5Model, cfg.model.cfg, "/path/to/model") + model = loader.load() + print("Build model finished.") + + return model + + +class t5Graph(nn.Graph): + def __init__(self, eager_model): + super().__init__() + self.model = eager_model + + def build( + self, + encoder_input_ids, + encoder_attn_mask, + decoder_input_ids, + decoder_attn_mask, + encoder_decoder_attn_mask, + ): + out = self.model( + encoder_input_ids, + encoder_attn_mask, + decoder_input_ids, + decoder_attn_mask, + encoder_decoder_attn_mask, + ) + return out + + +if __name__ == "__main__": + model = get_model("projects/MT5/configs/mt5_pretrain.py") + model.eval() + + t5_graph = t5Graph(model) + # Build the static graph model + encoder_input_ids = flow.ones( + 1, 5, dtype=flow.int64, sbp=flow.sbp.broadcast, placement=flow.placement("cuda", ranks=[0]) + ) + encoder_attn_mask = flow.ones( + 1, 3, dtype=flow.int64, sbp=flow.sbp.broadcast, placement=flow.placement("cuda", ranks=[0]) + ) + decoder_input_ids = flow.ones( + 1, + 5, + 5, + dtype=flow.bool, + sbp=flow.sbp.broadcast, + placement=flow.placement("cuda", ranks=[0]), + ) + decoder_attn_mask = flow.ones( + 1, + 3, + 3, + dtype=flow.bool, + sbp=flow.sbp.broadcast, + placement=flow.placement("cuda", ranks=[0]), + ) + encoder_decoder_attn_mask = flow.ones( + 1, + 3, + 5, + dtype=flow.bool, + sbp=flow.sbp.broadcast, + placement=flow.placement("cuda", ranks=[0]), + ) + + # check your model.forward is valid + # output = t5_graph( + # encoder_input_ids, + # encoder_attn_mask, + # decoder_input_ids, + # decoder_attn_mask, + # encoder_decoder_attn_mask + # ) + # print(output) + + print("Compiling the graph which may make some time, please wait for a moment....") + t5_graph._compile( + encoder_input_ids, + encoder_attn_mask, + decoder_input_ids, + decoder_attn_mask, + encoder_decoder_attn_mask, + ) + + convert_to_onnx_and_check( + t5_graph, + external_data=False, + opset=11, + flow_weight_dir=None, + onnx_model_path="./", + dynamic_batch_size=False, + device="gpu_global", + input_tensor_range=[0, 10], + ) diff --git a/libai/optim/__init__.py b/libai/optim/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..78df6e7688f24c15017c0e9b28b167671345f8b9 --- /dev/null +++ b/libai/optim/__init__.py @@ -0,0 +1,16 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .build import build_optimizer, get_default_optimizer_params diff --git a/libai/optim/__pycache__/__init__.cpython-39.pyc b/libai/optim/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..53ddd2fd7891a550f11f3cdf157c1d895ee4e55c Binary files /dev/null and b/libai/optim/__pycache__/__init__.cpython-39.pyc differ diff --git a/libai/optim/__pycache__/build.cpython-39.pyc b/libai/optim/__pycache__/build.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a23b67eb89323871d90a3b99e338daffd504f69c Binary files /dev/null and b/libai/optim/__pycache__/build.cpython-39.pyc differ diff --git a/libai/optim/build.py b/libai/optim/build.py new file mode 100644 index 0000000000000000000000000000000000000000..f8ef8747d542b85356e0f5efe93ba338490e9c66 --- /dev/null +++ b/libai/optim/build.py @@ -0,0 +1,162 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +from collections import defaultdict +from typing import Any, Dict, List + +import oneflow as flow + +from libai.config import instantiate +from libai.layers import LayerNorm + +# -------------------------------------------------------- +# References: +# https://github.com/facebookresearch/detectron2/blob/main/detectron2/solver/build.py +# -------------------------------------------------------- + + +def build_optimizer(cfg, model): + """ + Build an optimizer from config. + """ + + cfg.params.model = model + optim = instantiate(cfg) + return optim + + +def get_default_optimizer_params( + model, + base_lr=None, + weight_decay=None, + weight_decay_norm=None, + weight_decay_bias=None, + clip_grad_max_norm=None, + clip_grad_norm_type=None, + overrides=None, +): + """ + Get default param list for optimizer, with suport for a few types of overrides. + If no overrides are needed, it is equivalent to `model.parameters()`. + + Arguments: + base_lr: lr for every group by default. Can be omitted to use the one in optimizer. + weight_decay: weight decay for every group by default. Can be omitted to use the one + in optimizer. + weight_decay_norm: override weight decay for params in normalization layers + weight_decay_bias: override weight decay for bias parameters + overrides: if not `None`, provides values for optimizer hyperparameters + (LR, weight decay) for module parameters with a given name; e.g. + ``{"embedding": {"lr": 0.01, "weight_decay": 0.1}}`` will set the LR and + weight decay values for all module parameters named `embedding`. + + For common transformer models, ``weight_decay_norm`` and ``weight_decay_bias`` + are usually set to 0. + + Example: + :: + + flow.optim.AdamW( + get_default_optimizer_params(model, weight_decay_norm=0, weight_decay_bias=0), + lr=0.01, + weight_decay=1e-4 + ) + """ + if overrides is None: + overrides = {} + defaults = {} + if base_lr is not None: + defaults["lr"] = base_lr + if weight_decay is not None: + defaults["weight_decay"] = weight_decay + if clip_grad_max_norm is not None and clip_grad_norm_type is not None: + defaults["clip_grad_max_norm"] = clip_grad_max_norm + defaults["clip_grad_norm_type"] = clip_grad_norm_type + bias_overrides = {} + if weight_decay_bias is not None: + bias_overrides["weight_decay"] = weight_decay_bias + if len(bias_overrides): + if "bias" in overrides: + raise ValueError("Conflicting overrides for 'bias'") + overrides["bias"] = bias_overrides + + norm_module_types = ( + LayerNorm, + flow.nn.BatchNorm1d, + flow.nn.BatchNorm2d, + flow.nn.BatchNorm3d, + flow.nn.GroupNorm, + flow.nn.InstanceNorm1d, + flow.nn.InstanceNorm2d, + flow.nn.InstanceNorm3d, + flow.nn.FusedBatchNorm1d, + flow.nn.FusedBatchNorm2d, + flow.nn.FusedBatchNorm3d, + ) + params = [] + memo = set() + for module in model.modules(): + for model_param_name, value in module.named_parameters(recurse=False): + if not value.requires_grad: + continue + # Avoid duplicating parameters + if value in memo: + continue + memo.add(value) + + hyperparams = copy.copy(defaults) + if isinstance(module, norm_module_types) and weight_decay_norm is not None: + hyperparams["weight_decay"] = weight_decay_norm + hyperparams.update(overrides.get(model_param_name, {})) + params.append({"params": [value], **hyperparams}) + return reduce_param_groups(params) + + +def _expand_param_groups(params: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Transform parameter groups into per-parameter structure. + Later items in `params` can overwrite parameters set in previous items. + """ + ret = defaultdict(dict) + for item in params: + assert "params" in item + cur_params = {x: y for x, y in item.items() if x != "params"} + for param in item["params"]: + ret[param].update({"params": [param], **cur_params}) + return list(ret.values()) + + +def reduce_param_groups(params: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Reorganize the parameter groups and merge duplicated groups. + The number of parameter groups needs to be as small as possible in order + to efficiently use the OneFlow multi-tensor optimizer. Therefore instead + of using a parameter_group per single parameter, we reorganize the + parameter groups and merge duplicated groups. This approach speeds + up multi-tensor optimizer significantly. + """ + params = _expand_param_groups(params) + groups = defaultdict(list) # re-group all parameter groups by their hyperparams + for item in params: + cur_params = tuple((x, y) for x, y in item.items() if x != "params") + groups[cur_params].extend(item["params"]) + ret = [] + for param_keys, param_values in groups.items(): + cur = {kv[0]: kv[1] for kv in param_keys} + cur["params"] = param_values + ret.append(cur) + return ret diff --git a/libai/scheduler/__init__.py b/libai/scheduler/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2fae5205056312282c0e3010847464ae07974adb --- /dev/null +++ b/libai/scheduler/__init__.py @@ -0,0 +1,24 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .build import build_lr_scheduler +from .lr_scheduler import ( + WarmupCosineAnnealingLR, + WarmupCosineLR, + WarmupExponentialLR, + WarmupMultiStepLR, + WarmupPolynomialLR, + WarmupStepLR, +) diff --git a/libai/scheduler/__pycache__/__init__.cpython-39.pyc b/libai/scheduler/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff958ef26b47dee883d0a855085b2fea6417bca2 Binary files /dev/null and b/libai/scheduler/__pycache__/__init__.cpython-39.pyc differ diff --git a/libai/scheduler/__pycache__/build.cpython-39.pyc b/libai/scheduler/__pycache__/build.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..554648ae4cc8e5f7b286264102a980b0dd50f65a Binary files /dev/null and b/libai/scheduler/__pycache__/build.cpython-39.pyc differ diff --git a/libai/scheduler/__pycache__/lr_scheduler.cpython-39.pyc b/libai/scheduler/__pycache__/lr_scheduler.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d1336a1b7a2f764a00f3320241759eb6ab66efb3 Binary files /dev/null and b/libai/scheduler/__pycache__/lr_scheduler.cpython-39.pyc differ diff --git a/libai/scheduler/build.py b/libai/scheduler/build.py new file mode 100644 index 0000000000000000000000000000000000000000..aa8c6a18fe5ac7ca621533412b37e96e8c48972d --- /dev/null +++ b/libai/scheduler/build.py @@ -0,0 +1,23 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libai.config import instantiate + + +def build_lr_scheduler(cfg, optimizer): + """Build learning rate scheduler, defined by ``cfg``.""" + cfg.optimizer = optimizer + scheduler = instantiate(cfg) + return scheduler diff --git a/libai/scheduler/lr_scheduler.py b/libai/scheduler/lr_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..3f32a813f41467eac7e96c3e5016809393b5ede0 --- /dev/null +++ b/libai/scheduler/lr_scheduler.py @@ -0,0 +1,257 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import oneflow as flow + +logger = logging.getLogger(__name__) + + +def WarmupCosineLR( + optimizer: flow.optim.Optimizer, + max_iter: int, + warmup_factor: float, + warmup_iter: int, + alpha: float = 0.0, + warmup_method: str = "linear", +): + """Create a schedule with a learning rate that decreases following + the values of the Cosine function between the initial lr set in the + optimizer to 0, after a warmup period during which it increases linearly + between 0 and the initial lr set in the optimizer. + + Args: + optimizer (flow.optim.Optimizer): Wrapped optimizer. + max_iter (int): Total training iters. + warmup_factor (float): The warmup factor. + warmup_iter (int): The number of warmup steps. + alpha (float, optional): The learning rate scale factor (:math:`\\alpha`). Defaults to 0.0. + warmup_method (str, optional): The method of warmup, you can choose "linear" or "constant". + In linear mode, the multiplication factor starts with warmup_factor in + the first epoch and then inreases linearly to reach 1. Defaults to "linear". + """ + cosine_decay_lr = flow.optim.lr_scheduler.CosineDecayLR( + optimizer, decay_steps=max_iter, alpha=alpha + ) + if warmup_iter == 0: + logger.warning("warmup iters equals to zero, return CosineLR") + return cosine_decay_lr + elif warmup_iter > max_iter: + logger.warning("warmup iters is larger than the total training iters") + warmup_cosine_lr = flow.optim.lr_scheduler.WarmUpLR( + cosine_decay_lr, + warmup_factor=warmup_factor, + warmup_iters=warmup_iter, + warmup_method=warmup_method, + ) + return warmup_cosine_lr + + +def WarmupCosineAnnealingLR( + optimizer: flow.optim.Optimizer, + max_iter: int, + warmup_factor: float, + warmup_iter: int, + eta_min: float = 0.0, + warmup_method: str = "linear", +): + """Create a schedule with a learning rate that decreases following + the values of the Cosine Annealing function between the initial + lr set in the optimizer to 0, after a warmup period during which + it increases linearly between 0 and the initial lr set in the optimizer. + + Args: + optimizer (flow.optim.Optimizer): Wrapped optimizer. + max_iter (int): Total training iters. + warmup_factor (float): The warmup factor. + warmup_iter (int): The number of warmup steps. + eta_min (float, optional): Minimum learning rate. Defaults to 0.0. + warmup_method (str, optional): The method of warmup, you can choose "linear" or "constant". + In linear mode, the multiplication factor starts with warmup_factor in the first epoch + and then inreases linearly to reach 1. Defaults to "linear". + """ + cosine_annealing_lr = flow.optim.lr_scheduler.CosineAnnealingLR( + optimizer, T_max=max_iter, eta_min=eta_min + ) + if warmup_iter == 0: + logger.warning("warmup iters equals to zero, return CosineAnnealingLR") + return cosine_annealing_lr + warmup_cosine_annealing_lr = flow.optim.lr_scheduler.WarmUpLR( + cosine_annealing_lr, + warmup_factor=warmup_factor, + warmup_iters=warmup_iter, + warmup_method=warmup_method, + ) + return warmup_cosine_annealing_lr + + +def WarmupStepLR( + optimizer: flow.optim.Optimizer, + max_iter: int, + warmup_factor: float, + warmup_iter: int, + step_size: int, + gamma: float = 0.1, + warmup_method: str = "linear", +): + """Create a schedule with a learning rate that decreases following the values of the Step + function between the initial lr set in the optimizer to 0, after a warmup period during which + it increases linearly between 0 and the initial lr set in the optimizer. + Args: + optimizer (flow.optim.Optimizer): Wrapped optimizer. + max_iter (int): Total training iters. + warmup_factor (float): The warmup factor. + warmup_iter (int): The number of warmup steps. + step_size (int): Period of learning rate decay. + gamma (float, optional): Multiplicative factor of learning rate decay. Defaults to 0.1. + warmup_method (str, optional): The method of warmup, you can choose "linear" or "constant". + In linear mode, the multiplication factor starts with warmup_factor in the first + epoch and then inreases linearly to reach 1. Defaults to "linear". + """ + step_lr = flow.optim.lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma) + if warmup_iter == 0: + logger.warning("warmup iters equals to zero, return StepLR") + return step_lr + warmup_step_lr = flow.optim.lr_scheduler.WarmUpLR( + step_lr, + warmup_factor=warmup_factor, + warmup_iters=warmup_iter, + warmup_method=warmup_method, + ) + return warmup_step_lr + + +def WarmupMultiStepLR( + optimizer: flow.optim.Optimizer, + max_iter: int, + warmup_factor: float, + warmup_iter: int, + milestones: list, + gamma: float = 0.1, + warmup_method: str = "linear", +): + """Create a schedule with a learning rate that decreases following the values of the MultiStep + function between the initial lr set in the optimizer to 0, after a warmup period during which + it increases linearly between 0 and the initial lr set in the optimizer. + + Args: + optimizer (flow.optim.Optimizer): Wrapped optimizer. + max_iter (int): Total training iters. + warmup_factor (float): The warmup factor. + warmup_iter (int): The number of warmup steps. + milestones (list): List of step indices. Must be increasing. + gamma (float, optional): Multiplicative factor of learning rate decay. Defaults to 0.1. + warmup_method (str, optional): The method of warmup, you can choose "linear" or "constant". + In linear mode, the multiplication factor starts with warmup_factor in the first + epoch and then inreases linearly to reach 1. Defaults to "linear". + """ + multistep_lr = flow.optim.lr_scheduler.MultiStepLR( + optimizer, milestones=milestones, gamma=gamma + ) + if warmup_iter == 0: + logger.warning("warmup iters equals to zero, return MultiStepLR") + return multistep_lr + warmup_multistep_lr = flow.optim.lr_scheduler.WarmUpLR( + multistep_lr, + warmup_factor=warmup_factor, + warmup_iters=warmup_iter, + warmup_method=warmup_method, + ) + return warmup_multistep_lr + + +def WarmupExponentialLR( + optimizer: flow.optim.Optimizer, + max_iter: int, + gamma: float, + warmup_factor: float, + warmup_iter: int, + warmup_method: str = "linear", +): + """Create a schedule with a learning rate that decreases following the values of + the Exponential function between the initial lr set in the optimizer to 0, + after a warmup period during which it increases linearly between 0 and the + initial lr set in the optimizer. + + Args: + optimizer (flow.optim.Optimizer): Wrapped optimizer. + max_iter (int): Total training iters. + gamma (float): Multiplicative factor of learning rate decay. + warmup_factor (float): The warmup factor. + warmup_iter (int): The number of warmup steps. + warmup_method (str, optional): The method of warmup, you can choose "linear" or "constant". + In linear mode, the multiplication factor starts with warmup_factor in the first epoch + and then inreases linearly to reach 1. Defaults to "linear". + """ + exponential_lr = flow.optim.lr_scheduler.ExponentialLR(optimizer, gamma=gamma) + if warmup_iter == 0: + logger.warning("warmup iters equals to zero, return ExponentialLR") + return exponential_lr + warmup_exponential_lr = flow.optim.lr_scheduler.WarmUpLR( + exponential_lr, + warmup_factor=warmup_factor, + warmup_iters=warmup_iter, + warmup_method=warmup_method, + ) + return warmup_exponential_lr + + +def WarmupPolynomialLR( + optimizer: flow.optim.Optimizer, + max_iter: int, + warmup_factor: float, + warmup_iter: int, + end_learning_rate: float = 0.0001, + power: float = 1.0, + cycle: bool = False, + warmup_method: str = "linear", +): + """Create a schedule with a learning rate that decreases as a polynomial decay from + the initial lr set in the optimizer to end lr defined by `lr_end`, + after a warmup period during which it increases linearly from 0 to the + initial lr set in the optimizer. + + + Args: + optimizer (flow.optim.Optimizer): Wrapped optimizer. + max_iter (int): Total training iters. + warmup_factor (float): The warmup factor. + warmup_iter (int): The number of warmup steps. + end_learning_rate (float, optional): The final learning rate. Defaults to 0.0001. + power (float, optional): The power of polynomial. Defaults to 1.0. + cycle (bool, optional): If cycle is True, the scheduler will decay the learning rate + every decay steps. Defaults to False. + warmup_method (str, optional): The method of warmup, you can choose "linear" or "constant". + In linear mode, the multiplication factor starts with warmup_factor in the first + epoch and then inreases linearly to reach 1. Defaults to "linear". + """ + polynomial_lr = flow.optim.lr_scheduler.PolynomialLR( + optimizer, + decay_batch=max_iter, + end_learning_rate=end_learning_rate, + power=power, + cycle=cycle, + ) + if warmup_iter == 0: + logger.warning("warmup iters equals to zero, return PolynomialLR") + return polynomial_lr + warmup_polynomial_lr = flow.optim.lr_scheduler.WarmUpLR( + polynomial_lr, + warmup_factor=warmup_factor, + warmup_iters=warmup_iter, + warmup_method=warmup_method, + ) + return warmup_polynomial_lr diff --git a/libai/tokenizer/__init__.py b/libai/tokenizer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1fbeb1bb195db8ed8abc725b330d43ef8e5bb5dd --- /dev/null +++ b/libai/tokenizer/__init__.py @@ -0,0 +1,21 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .build import build_tokenizer +from .tokenization_bert import BertTokenizer +from .tokenization_roberta import RobertaTokenizer +from .tokenization_gpt2 import GPT2Tokenizer +from .tokenization_t5 import T5Tokenizer +from .tokenization_base import PreTrainedTokenizer diff --git a/libai/tokenizer/__pycache__/__init__.cpython-39.pyc b/libai/tokenizer/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0c51523f0036db522104b36731b59ff94ba59907 Binary files /dev/null and b/libai/tokenizer/__pycache__/__init__.cpython-39.pyc differ diff --git a/libai/tokenizer/__pycache__/build.cpython-39.pyc b/libai/tokenizer/__pycache__/build.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d6311514f3f934837f2b5c5327d61b702386a55b Binary files /dev/null and b/libai/tokenizer/__pycache__/build.cpython-39.pyc differ diff --git a/libai/tokenizer/__pycache__/tokenization_base.cpython-39.pyc b/libai/tokenizer/__pycache__/tokenization_base.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5e8f166b844ee8ab86085f38719c94051d6a58f0 Binary files /dev/null and b/libai/tokenizer/__pycache__/tokenization_base.cpython-39.pyc differ diff --git a/libai/tokenizer/__pycache__/tokenization_bert.cpython-39.pyc b/libai/tokenizer/__pycache__/tokenization_bert.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e0bbd2376aac851cedb3b63e253967243610c438 Binary files /dev/null and b/libai/tokenizer/__pycache__/tokenization_bert.cpython-39.pyc differ diff --git a/libai/tokenizer/__pycache__/tokenization_gpt2.cpython-39.pyc b/libai/tokenizer/__pycache__/tokenization_gpt2.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1282194b5120a3e1cc619116145486c060065ceb Binary files /dev/null and b/libai/tokenizer/__pycache__/tokenization_gpt2.cpython-39.pyc differ diff --git a/libai/tokenizer/__pycache__/tokenization_roberta.cpython-39.pyc b/libai/tokenizer/__pycache__/tokenization_roberta.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..df1314e197bcd97d6f4bda76033f2cc3e965fba1 Binary files /dev/null and b/libai/tokenizer/__pycache__/tokenization_roberta.cpython-39.pyc differ diff --git a/libai/tokenizer/__pycache__/tokenization_t5.cpython-39.pyc b/libai/tokenizer/__pycache__/tokenization_t5.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a093157dbf3111399cd4ada13f24e285bd4844a1 Binary files /dev/null and b/libai/tokenizer/__pycache__/tokenization_t5.cpython-39.pyc differ diff --git a/libai/tokenizer/build.py b/libai/tokenizer/build.py new file mode 100644 index 0000000000000000000000000000000000000000..5a18a20838639c414e8061c24f8c8d2c13628275 --- /dev/null +++ b/libai/tokenizer/build.py @@ -0,0 +1,33 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from libai.config import instantiate + +logger = logging.getLogger(__name__) + + +def build_tokenizer(cfg): + """Initialize tokenizer.""" + tokenizer = instantiate(cfg.tokenizer) + + if cfg.append_eod and tokenizer.eod_token is None: + if tokenizer.eos_token is not None: + tokenizer.eod_token = tokenizer.eos_token + else: + tokenizer.eod_token = tokenizer.pad_token + + return tokenizer diff --git a/libai/tokenizer/tokenization_base.py b/libai/tokenizer/tokenization_base.py new file mode 100644 index 0000000000000000000000000000000000000000..d5bd5a9ea570dd59887a5faffc1bfc5eb7a0853b --- /dev/null +++ b/libai/tokenizer/tokenization_base.py @@ -0,0 +1,1277 @@ +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""copy from HuggingFace transformer repo, to tokenize the sentence. +This class only focus on tokenization, converting token to id and their inverse operation. +It does not construct inputs using special symbols.""" + +import copy +import itertools +import json +import logging +import os +import unicodedata +from io import open +from typing import Dict, List, Optional, Union + +import numpy as np +import oneflow as flow + +from libai.utils import distributed as dist +from libai.utils.file_io import PathManager +from libai.utils.file_utils import cached_path + +logger = logging.getLogger(__name__) + + +def _is_whitespace(char): + """Checks whether `char` is a whitespace character.""" + # \t, \n, and \r are technically control characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `char` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat.startswith("C"): + return True + return False + + +def _is_punctuation(char): + """Checks whether `char` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ( + (cp >= 33 and cp <= 47) + or (cp >= 58 and cp <= 64) + or (cp >= 91 and cp <= 96) + or (cp >= 123 and cp <= 126) + ): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False + + +SPECIAL_TOKENS_MAP_FILE = "special_tokens_map.json" +ADDED_TOKENS_FILE = "added_tokens.json" +TOKENIZER_CONFIG_FILE = "tokenizer_config.json" + + +class PreTrainedTokenizer(object): + """ + Base class for all tokenizers. + + Handle all the shared methods for tokenization and special tokens, methods + dowloading/caching/loading pretrained tokenizers as well as adding tokens to the vocabulary. + This class also contains the added tokens in a unified way on top of all tokenizers, so we don't + have to handle the specific vocabulary augmentation methods of the various underlying + dictionary structures (BPE, sentencepiece...). + + Class attributes (overridden by derived classes): + + ``vocab_files_names``: a python ``dict`` with, as keys, the ``__init__`` keyword name of + each vocabulary file required by the model, and as associated values, the filename for + saving the associated file (string). + + ``pretrained_vocab_files_map``: a python ``dict of dict`` the high-level keys being the + ``__init__`` keyword name of each vocabulary file required by the model, the low-level + being the `short-cut-names` (string) of the pretrained models with, as associated values, + the `url` (string) to the associated pretrained vocabulary file. + + ``max_model_input_sizes``: a python ``dict`` with, as keys, the `short-cut-names` (string) + of the pretrained models, and as associated values, the maximum length of the sequence + inputs of this model, or None if the model has no maximum input size. + + ``pretrained_init_configuration``: a python ``dict`` with, as keys, the `short-cut-names` + (string) of the pretrained models, and as associated values, a dictionnary of specific + arguments to pass to the ``__init__`` method of the tokenizer class for this pretrained + model when loading the tokenizer with the ``from_pretrained()`` method. + + Args: + bos_token (:obj:`str`, `optional`): A special token representing the beginning of a + sentence. + eos_token (:obj:`str`, `optional`): A special token representing the end of a sentence. + unk_token (:obj:`str`, `optional`): A special token representing an out-of-vocabulary token. + sep_token (:obj:`str`, `optional`): A special token separating two different sentences in + the same input (used by BERT for instance). + pad_token (:obj:`str`, `optional`): A special token used to make arrays of tokens the same + size for batching purpose. + Will then be ignored by attention mechanisms or loss computation. + cls_token (:obj:`str`, `optional`): A special token representing the class of the input + (used by BERT for instance). + mask_token (:obj:`str`, `optional`): A special token representing a masked token (used by + masked-language modeling pretraining objectives, like BERT). + eod_token (:obj:`str`, `optional`): A special token representing the end of a document. + additional_special_tokens (tuple or list of :obj:`str`, `optional`): + A tuple or a list of additional special tokens. + """ + + vocab_files_names = {} + pretrained_vocab_files_map = {} + pretrained_init_configuration = {} + max_model_input_sizes = {} + + SPECIAL_TOKENS_ATTRIBUTES = [ + "bos_token", + "eos_token", + "unk_token", + "sep_token", + "pad_token", + "cls_token", + "mask_token", + "eod_token", + "additional_special_tokens", + ] + + def __init__(self, verbose=True, **kwargs): + self._bos_token = None + self._eos_token = None + self._unk_token = None + self._sep_token = None + self._pad_token = None + self._cls_token = None + self._mask_token = None + self._eod_token = None + self._additional_special_tokens = [] + self.verbose = verbose + + # Added tokens - We store this for both slow and fast tokenizers + # until the serialization of Fast tokenizers is updated + self.added_tokens_encoder: Dict[str, int] = {} + self.added_tokens_decoder: Dict[int, str] = {} + self.unique_no_split_tokens: List[str] = [] + + # inputs and kwargs for saving and re-loading + # (see ``from_pretrained`` and ``save_pretrained``) + self.init_inputs = () + self.init_kwargs = {} + + # We directly set the hidden value to allow initialization with special tokens + # which are not yet in the vocabulary. Necessary for serialization/de-serialization + for key, value in kwargs.items(): + if value is None: + continue + if key in self.SPECIAL_TOKENS_ATTRIBUTES: + if key == "additional_special_tokens": + assert all( + isinstance(t, str) for t in value + ), "One of the tokens is not a string" + setattr(self, key, list(value)) + elif isinstance(value, str): + setattr(self, key, value) + else: + raise TypeError(f"special token {key} has to be str but got: {type(value)}") + + @classmethod + def from_pretrained(cls, *inputs, **kwargs): + r""" + Instantiate a :class:`~PreTrainedTokenizer` (or a derived class) from a + predefined tokenizer. + + Args: + pretrained_model_name_or_path(`str` or `os.PathLike`): + Can be either: + + - a string with the `shortcut name` of a predefined tokenizer to load from cache + or download, e.g.: ``bert-base-uncased``. + + - a path to a `directory` containing vocabulary files required by the tokenizer, + for instance saved using the :func:`~PreTrainedTokenizer.save_pretrained` + method, e.g., ``./my_model_directory/``. + + - (not applicable to all derived classes) a path or url to a single saved + vocabulary file if and only if the tokenizer only requires a single vocabulary + file (e.g. Bert, XLNet), e.g., ``./my_model_directory/vocab.txt``. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded predefined tokenizer vocabulary files + should be cached if the standard cache should not be used. + force_download: (`optional`) boolean, default False: + Force to (re-)download the vocabulary files and override the cached versions if + they exist. + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, + e.g., {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + inputs: (`optional`) positional arguments: will be passed to the + Tokenizer ``__init__`` method. + kwargs: (`optional`) keyword arguments: will be passed to the + Tokenizer ``__init__`` method. Can be used to set special tokens + like ``bos_token``, ``eos_token``, ``unk_token``, ``sep_token``, + ``pad_token``, ``cls_token``, ``mask_token``, ``additional_special_tokens``. + See parameters in the doc string of :class:`~PreTrainedTokenizer` + for details. + + Examples: + + .. code-block:: python + + # We can't instantiate directly the base class `PreTrainedTokenizer` so let's + # show our examples on a derived class: BertTokenizer + # Download vocabulary from S3 and cache. + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + # If vocabulary files are in a directory (e.g. tokenizer was + # saved using `save_pretrained('./test/saved_model/')`) + tokenizer = BertTokenizer.from_pretrained('./test/saved_model/') + # If the tokenizer uses a single vocabulary file, you can point directly to this file + tokenizer = BertTokenizer.from_pretrained('./test/saved_model/my_vocab.txt') + # You can link tokens to special vocabulary when instantiating + tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', unk_token='') + # You should be sure '' is in the vocabulary when doing that. + # Otherwise use tokenizer.add_special_tokens({'unk_token': ''}) instead) + assert tokenizer.unk_token == '' + """ + return cls._from_pretrained(*inputs, **kwargs) + + @classmethod + def _from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs): + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + proxies = kwargs.pop("proxies", None) + + s3_models = list(cls.max_model_input_sizes.keys()) + vocab_files = {} + init_configuration = {} + if pretrained_model_name_or_path in s3_models: + # Get the vocabulary from AWS S3 bucket + for file_id, map_list in cls.pretrained_vocab_files_map.items(): + vocab_files[file_id] = map_list[pretrained_model_name_or_path] + if ( + cls.pretrained_init_configuration + and pretrained_model_name_or_path in cls.pretrained_init_configuration + ): + init_configuration = cls.pretrained_init_configuration[ + pretrained_model_name_or_path + ] + else: + # Get the vocabulary from local files + logger.info( + "Model name '{}' not found in model shortcut name list ({}). " + "Assuming '{}' is a path or url to a directory containing tokenizer files.".format( + pretrained_model_name_or_path, + ", ".join(s3_models), + pretrained_model_name_or_path, + ) + ) + + # Look for the tokenizer main vocabulary files + for file_id, file_name in cls.vocab_files_names.items(): + if os.path.isdir(pretrained_model_name_or_path): + # If a directory is provided we look for the standard filenames + full_file_name = os.path.join(pretrained_model_name_or_path, file_name) + else: + # If a path to a file is provided we use it (will only work for non-BPE + # tokenizer using a single vocabulary file) + full_file_name = pretrained_model_name_or_path + if not os.path.exists(full_file_name): + logger.info("Didn't find file {}. We won't load it.".format(full_file_name)) + full_file_name = None + vocab_files[file_id] = full_file_name + + # Look for the additional tokens files + additional_files_names = { + "added_tokens_file": ADDED_TOKENS_FILE, + "special_tokens_map_file": SPECIAL_TOKENS_MAP_FILE, + "tokenizer_config_file": TOKENIZER_CONFIG_FILE, + } + + # If a path to a file was provided, get the parent directory + saved_directory = pretrained_model_name_or_path + if os.path.exists(saved_directory) and not os.path.isdir(saved_directory): + saved_directory = os.path.dirname(saved_directory) + + for file_id, file_name in additional_files_names.items(): + full_file_name = os.path.join(saved_directory, file_name) + if not os.path.exists(full_file_name): + logger.info("Didn't find file {}. We won't load it.".format(full_file_name)) + full_file_name = None + vocab_files[file_id] = full_file_name + + if all(full_file_name is None for full_file_name in vocab_files.values()): + logger.error( + "Model name '{}' was not found in model name list ({}). " + "We assumed '{}' was a path or url but couldn't find tokenizer files" + "at this path or url.".format( + pretrained_model_name_or_path, + ", ".join(s3_models), + pretrained_model_name_or_path, + ) + ) + return None + + # Get files from url, cache, or disk depending on the case + try: + resolved_vocab_files = {} + for file_id, file_path in vocab_files.items(): + if file_path is None: + resolved_vocab_files[file_id] = None + else: + resolved_vocab_files[file_id] = cached_path( + file_path, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + ) + except EnvironmentError as e: + if pretrained_model_name_or_path in s3_models: + logger.error("Couldn't reach server to download vocabulary.") + else: + logger.error( + "Model name '{}' was not found in model name list ({}). " + "We assumed '{}' was a path or url but couldn't find files {} " + "at this path or url.".format( + pretrained_model_name_or_path, + ", ".join(s3_models), + pretrained_model_name_or_path, + str(vocab_files.keys()), + ) + ) + raise e + + for file_id, file_path in vocab_files.items(): + if file_path == resolved_vocab_files[file_id]: + logger.info("loading file {}".format(file_path)) + else: + logger.info( + "loading file {} from cache at {}".format( + file_path, resolved_vocab_files[file_id] + ) + ) + + # Prepare tokenizer initialization kwargs + # Did we saved some inputs and kwargs to reload ? + tokenizer_config_file = resolved_vocab_files.pop("tokenizer_config_file", None) + if tokenizer_config_file is not None: + init_kwargs = json.load(open(tokenizer_config_file, encoding="utf-8")) + saved_init_inputs = init_kwargs.pop("init_inputs", ()) + if not init_inputs: + init_inputs = saved_init_inputs + else: + init_kwargs = init_configuration + + # Update with newly provided kwargs + init_kwargs.update(kwargs) + + # Merge resolved_vocab_files arguments in init_kwargs. + added_tokens_file = resolved_vocab_files.pop("added_tokens_file", None) + special_tokens_map_file = resolved_vocab_files.pop("special_tokens_map_file", None) + for args_name, file_path in resolved_vocab_files.items(): + if args_name not in init_kwargs: + init_kwargs[args_name] = file_path + + if special_tokens_map_file is not None: + special_tokens_map = json.load(open(special_tokens_map_file, encoding="utf-8")) + for key, value in special_tokens_map.items(): + if key not in init_kwargs: + init_kwargs[key] = value + + # Instantiate tokenizer. + tokenizer = cls(*init_inputs, **init_kwargs) + + # Save inputs and kwargs for saving and re-loading with ``save_pretrained`` + tokenizer.init_inputs = init_inputs + tokenizer.init_kwargs = init_kwargs + + # Add supplementary tokens. + special_tokens = tokenizer.all_special_tokens + if added_tokens_file is not None: + with open(added_tokens_file, encoding="utf-8") as added_tokens_handle: + added_tok_encoder = json.load(added_tokens_handle) + + # Sort added tokens by index + added_tok_encoder_sorted = list(sorted(added_tok_encoder.items(), key=lambda x: x[1])) + + for token, index in added_tok_encoder_sorted: + assert index == len(tokenizer), ( + f"Non-consecutive added token '{token}' found. " + f"Should have index {len(tokenizer)} but has index {index} in saved vocabulary." + ) + tokenizer.add_tokens(token, special_tokens=bool(token in special_tokens)) + + # Check all our special tokens are registered as "no split" token + # (we don't cut them) and are in the vocab + added_tokens = tokenizer.sanitize_special_tokens() + if added_tokens: + logger.warning( + "Special tokens have been added in the vocabulary," + "make sure the associated word embedding are fine-tuned or trained." + ) + + return tokenizer + + def save_pretrained(self, save_directory): + """ + Save the tokenizer vocabulary files together with: + + - added tokens, + - special-tokens-to-class-attributes-mapping, + - tokenizer instantiation positional and keywords inputs (e.g. do_lower_case for Bert). + + This won't save modifications other than ``added tokens`` and ``special token mapping``, + you may have applied to the tokenizer after the instantiation (e.g. modifying + tokenizer.do_lower_case after creation). + This method make sure the full tokenizer can then be re-loaded using the + :func:`~PreTrainedTokenizer.from_pretrained` class method. + """ + if not PathManager.isdir(save_directory): + logger.error("Saving directory ({}) should be a directory".format(save_directory)) + return + PathManager.mkdirs(save_directory) + + special_tokens_map_file = os.path.join(save_directory, SPECIAL_TOKENS_MAP_FILE) + added_tokens_file = os.path.join(save_directory, ADDED_TOKENS_FILE) + tokenizer_config_file = os.path.join(save_directory, TOKENIZER_CONFIG_FILE) + + tokenizer_config = copy.deepcopy(self.init_kwargs) + if len(self.init_inputs) > 0: + tokenizer_config["init_inputs"] = copy.deepcopy(self.init_inputs) + for file_id in self.vocab_files_names.keys(): + tokenizer_config.pop(file_id, None) + + with open(tokenizer_config_file, "w", encoding="utf-8") as f: + f.write(json.dumps(tokenizer_config, ensure_ascii=False)) + + with open(special_tokens_map_file, "w", encoding="utf-8") as f: + f.write(json.dumps(self.special_tokens_map, ensure_ascii=False)) + + added_vocab = self.get_added_vocab() + if added_vocab: + with open(added_tokens_file, "w", encoding="utf-8") as f: + out_str = json.dumps(added_vocab, ensure_ascii=False) + f.write(out_str) + + vocab_files = self.save_vocabulary(save_directory) + + return vocab_files + (special_tokens_map_file, added_tokens_file) + + def save_vocabulary(self, save_directory): + """Save the tokenizer vocabulary to a directory. This method does *NOT* save added tokens + and special token mappings. + Please use :func:`~PreTrainedTokenizer.save_pretrained` to save the + full Tokenizer state if you want to reload it using the + :func:`~PreTrainedTokenizer.from_pretrained` class method. + """ + raise NotImplementedError + + @property + def vocab_size(self) -> int: + """Size of the base vocabulary (without the added tokens).""" + raise NotImplementedError + + def padded_vocab_size(self, multiple=1) -> int: + """Padded the vocabulary with dummy tokens and return the new size.""" + vocab_size = len(self) + while vocab_size % multiple != 0: + vocab_size += 1 + return vocab_size + + def __len__(self): + """Size of the full vocabulary with the added tokens.""" + return self.vocab_size + len(self.added_tokens_encoder) + + def get_vocab(self) -> Dict[str, int]: + """ + Returns the vocabulary as a dictionary of token to index. + :obj:`tokenizer.get_vocab()[token]` is equivalent to + :obj:`tokenizer.convert_tokens_to_ids(token)` + when :obj:`token` is in the vocab. + + Returns: + :obj:`Dict[str, int]`: The vocabulary. + """ + raise NotImplementedError + + def get_added_vocab(self) -> Dict[str, int]: + """ + Returns the added tokens in the vocabulary as a dictionary of token to index. + + Returns: + :obj:`Dict[str, int]`: The added tokens. + """ + return self.added_tokens_encoder + + def add_tokens(self, new_tokens: Union[str, List[str]], special_tokens: bool = False) -> int: + """ + Add a list of new tokens to the tokenizer class. If the new tokens are not in the + vocabulary, they are added to it with indices starting from the length of + the current vocabulary. + + .. Note:: + When adding new tokens to the vocabulary, you should make sure to also resize + the token embedding matrix of the model so that its embedding matrix matches + the tokenizer. + In order to do that, please use the + :meth:`~PreTrainedModel.resize_token_embeddings` method. + + Args: + new_tokens (:obj:`str`, or a list of `str`): + Tokens are only added if they are not already in the vocabulary. + special_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`): + Can be used to specify if the token is a special token. This mostly change + the normalization behavior + (special tokens like CLS or [MASK] are usually not lower-cased for instance). + + Returns: + :obj:`int`: Number of tokens added to the vocabulary. + + Examples: + + .. code-block:: python + + # Let's see how to increase the vocabulary of Bert model and tokenizer + tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased') + model = BertModel.from_pretrained('bert-base-uncased') + num_added_toks = tokenizer.add_tokens(['new_tok1', 'my_new-tok2']) + print('We have added', num_added_toks, 'tokens') + # Notice: resize_token_embeddings expect to receive the full size of the new + # vocabulary, i.e., the length of the tokenizer. + model.resize_token_embeddings(len(tokenizer)) + """ + if not new_tokens: + return 0 + + if not isinstance(new_tokens, (list, tuple)): + new_tokens = [new_tokens] + + tokens_to_add = [] + for token in new_tokens: + if not isinstance(token, str): + raise TypeError(f"Token {token} is not a string but a {type(token)}.") + if not special_tokens and hasattr(self, "do_lower_case") and self.do_lower_case: + token = token.lower() + if ( + token != self.unk_token + and self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token) + and token not in tokens_to_add + ): + tokens_to_add.append(token) + if self.verbose: + logger.info(f"Adding {token} to the vocabulary") + + added_tok_encoder = dict((tok, len(self) + i) for i, tok in enumerate(tokens_to_add)) + added_tok_decoder = {v: k for k, v in added_tok_encoder.items()} + self.added_tokens_encoder.update(added_tok_encoder) + self.added_tokens_decoder.update(added_tok_decoder) + + if special_tokens: + self.unique_no_split_tokens = sorted( + set(self.unique_no_split_tokens).union(set(new_tokens)) + ) + else: + self.unique_no_split_tokens = sorted( + set(self.unique_no_split_tokens).union(set(tokens_to_add)) + ) + + return len(tokens_to_add) + + def sanitize_special_tokens(self) -> int: + """ + Make sure that all the special tokens attributes of the tokenizer + (:obj:`tokenizer.mask_token`, :obj:`tokenizer.cls_token`, etc.) + are in the vocabulary. + + Add the missing ones to the vocabulary if needed. + + Return: + :obj:`int`: The number of tokens added in the vocaulary during the operation. + """ + return self.add_tokens(self.all_special_tokens, special_tokens=True) + + def add_special_tokens(self, special_tokens_dict: Dict[str, str]) -> int: + """ + Add a dictionary of special tokens (eos, pad, cls, etc.) to the encoder and link them to + class attributes. If special tokens are NOT in the vocabulary, they are added to it + (indexed starting from the last index of the current vocabulary). + + .. Note:: + When adding new tokens to the vocabulary, you should make sure to also resize the + token embedding matrix of the model so that its embedding matrix matches the tokenizer. + In order to do that, please use the + :meth:`~PreTrainedModel.resize_token_embeddings` method. + + Using :obj:`add_special_tokens` will ensure your special tokens can be used in several ways: + - Special tokens are carefully handled by the tokenizer (they are never split). + - You can easily refer to special tokens using tokenizer class attributes like + :obj:`tokenizer.cls_token`. This makes it easy to develop model-agnostic training and + fine-tuning scripts. + When possible, special tokens are already registered for provided pretrained models + (for instance :class:`~BertTokenizer` :obj:`cls_token` is already registered + to be :obj`'[CLS]'` and XLM's one is also registered to be :obj:`''`). + + Args: + special_tokens_dict (dictionary `str` to `str`): + Keys should be in the list of predefined special attributes: [``bos_token``, + ``eos_token``, ``unk_token``, ``sep_token``, ``pad_token``, + ``cls_token``, ``mask_token``, + ``additional_special_tokens``]. + Tokens are only added if they are not already in the vocabulary (tested by + checking if the tokenizer assign the index of the ``unk_token`` to them). + + Returns: + :obj:`int`: Number of tokens added to the vocabulary. + + Examples: + + .. code-block:: python + + # Let's see how to add a new classification token to GPT-2 + tokenizer = GPT2Tokenizer.from_pretrained('gpt2') + model = GPT2Model.from_pretrained('gpt2') + special_tokens_dict = {'cls_token': ''} + num_added_toks = tokenizer.add_special_tokens(special_tokens_dict) + print('We have added', num_added_toks, 'tokens') + # Notice: resize_token_embeddings expect to receive the full size of the new vocabulary, + # i.e., the length of the tokenizer. + model.resize_token_embeddings(len(tokenizer)) + assert tokenizer.cls_token == '' + """ + if not special_tokens_dict: + return 0 + + added_tokens = 0 + for key, value in special_tokens_dict.items(): + assert key in self.SPECIAL_TOKENS_ATTRIBUTES, f"Key {key} is not a special token" + + if self.verbose: + logger.info(f"Assigning {value} to the {key} key of the tokenizer") + setattr(self, key, value) + + if key == "additional_special_tokens": + assert isinstance(value, (list, tuple)) and all( + isinstance(t, str) for t in value + ), f"Tokens {value} for key {key} should all be a string" + added_tokens += self.add_tokens(value, special_tokens=True) + else: + assert isinstance(value, str), f"Token {value} for key {key} should be a string" + added_tokens += self.add_tokens([value], special_tokens=True) + + return added_tokens + + def tokenize(self, text: str, **kwargs) -> List[str]: + """ + Converts a string in a sequence of tokens, using the tokenizer. + Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies + (BPE/SentencePieces/WordPieces). Take care of added tokens. + + Args: + text (:obj:`str`): + The sequence to be encoded. + **kwargs (additional keyword arguments): + Passed along to the model-specific ``prepare_for_tokenization`` + preprocessing method. + + Returns: + :obj:`List[str]`: The list of tokens. + """ + + def split_on_token(tok, text): + result = [] + split_text = text.split(tok) + for i, sub_text in enumerate(split_text): + sub_text = sub_text.strip() + if i == 0 and not sub_text: + result += [tok] + elif i == len(split_text) - 1: + if sub_text: + result += [sub_text] + else: + pass + else: + if sub_text: + result += [sub_text] + result += [tok] + return result + + def split_on_tokens(tok_list, text): + if not text: + return [] + if not tok_list: + return self._tokenize(text, **kwargs) + + tokenized_text = [] + text_list = [text] + for tok in tok_list: + tokenized_text = [] + for sub_text in text_list: + if sub_text not in self.unique_no_split_tokens: + tokenized_text += split_on_token(tok, sub_text) + else: + tokenized_text += [sub_text] + text_list = tokenized_text + return list( + itertools.chain.from_iterable( + ( + self._tokenize(token) + if token not in self.unique_no_split_tokens + else [token] + for token in tokenized_text + ) + ) + ) + + no_split_token = self.unique_no_split_tokens + tokenized_text = split_on_tokens(no_split_token, text) + return tokenized_text + + def _tokenize(self, text, **kwargs): + """ + Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for + word-based vocabulary or sub-words for sub-word-based vocabularies + (BPE/SentencePieces/WordPieces). + Do NOT take care of added tokens. + """ + raise NotImplementedError + + def convert_tokens_to_ids(self, tokens: Union[str, List[str]]) -> Union[int, List[int]]: + """Converts a token string (or a sequence of tokens) in a single integer id + (or a sequence of ids), using the vocabulary. + """ + if tokens is None: + return None + + if isinstance(tokens, str): + return self._convert_token_to_id_with_added_voc(tokens) + + if len(tokens) > 0 and isinstance(tokens[0], list): + ids = [] + for ts in tokens: + ids_x = [] + for token in ts: + ids_x.append(self._convert_token_to_id_with_added_voc(token)) + ids.append(ids_x) + return ids + + ids = [] + for token in tokens: + ids.append(self._convert_token_to_id_with_added_voc(token)) + return ids + + def convert_to_tensors(self, token_ids, return_tensors=None, is_global=False, **kwargs): + if return_tensors is None: + return_token_ids = token_ids + elif return_tensors == "of": + if not is_global: + return_token_ids = flow.tensor(token_ids, dtype=flow.long) + elif is_global: + sbp = kwargs.get("sbp", dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast])) + placement = kwargs.get( + "placement", flow.placement("cuda", list(range(dist.get_world_size()))) + ) + return_token_ids = flow.tensor( + token_ids, sbp=sbp, placement=placement, dtype=flow.long + ) + elif return_tensors == "np": + return_token_ids = np.array(token_ids, dtype=np.int64) + return return_token_ids + + def _convert_token_to_id_with_added_voc(self, token): + if token is None: + return None + + if token in self.added_tokens_encoder: + return self.added_tokens_encoder[token] + return self._convert_token_to_id(token) + + def _convert_token_to_id(self, token): + raise NotImplementedError + + def encode(self, text, return_tensors=None, is_global=False, **kwargs): + if isinstance(text, str): + tokens = self.tokenize(text) + token_ids = self.convert_tokens_to_ids(tokens) + token_ids = self.build_inputs_with_special_tokens(token_ids) + token_ids = self.convert_to_tensors( + token_ids, return_tensors=return_tensors, is_global=is_global, **kwargs + ) + return token_ids + elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], str): + tokens = [self.tokenize(t) for t in text] + token_ids_list = self.convert_tokens_to_ids(tokens) + token_ids_list = [ + self.build_inputs_with_special_tokens(token_ids) for token_ids in token_ids_list + ] + token_ids_list = self.convert_to_tensors( + token_ids_list, return_tensors=return_tensors, is_global=is_global, **kwargs + ) + return token_ids_list + elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int): + return text + else: + raise ValueError( + "Input is not valid. Should be a string, a list/tuple of strings or " + "a list/tuple of integers." + ) + + def convert_ids_to_tokens( + self, ids: Union[int, List[int]], skip_special_tokens: bool = False + ) -> Union[str, List[str]]: + """ + Converts a single index or a sequence of indices in a token or a sequence of tokens, + using the vocabulary and added tokens. + + Args: + ids (:obj:`int` or :obj:`List[int]`): + The token id (or token ids) to convert to tokens. + skip_special_tokens (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether or not to remove special tokens in the decoding. + Returns: + :obj:`str` or :obj:`List[str]`: The decoded token(s). + """ + if isinstance(ids, int): + if ids in self.added_tokens_decoder: + return self.added_tokens_decoder[ids] + else: + return self._convert_id_to_token(ids) + tokens = [] + for index in ids: + if skip_special_tokens and index in self.all_special_ids: + continue + if index in self.added_tokens_decoder: + tokens.append(self.added_tokens_decoder[index]) + else: + tokens.append(self._convert_id_to_token(index)) + return tokens + + def _convert_id_to_token(self, index: int) -> str: + raise NotImplementedError + + def convert_tokens_to_string(self, tokens: List[str]) -> str: + """ + Converts a sequence of tokens to a single string. The most simple way to do it is + ``" ".join(tokens)`` but we often want to remove sub-word tokenization artifacts + at the same time. + + Args: + tokens (:obj:`List[str]`): The token to join in a string. + + Returns: + :obj:`str`: The joined tokens. + """ + return " ".join(tokens) + + def decode( + self, + token_ids, + skip_special_tokens=False, + clean_up_tokenization_spaces=True, + spaces_between_special_tokens: bool = True, + ): + """ + Converts a sequence of ids (integer) in a string, using the tokenizer and vocabulary + with options to remove special tokens and clean up tokenization spaces. + Similar to doing ``self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))``. + + Args: + token_ids: list of tokenized input ids. Can be obtained using the `encode` or + `encode_plus` methods. + skip_special_tokens: if set to True, will replace special tokens. + clean_up_tokenization_spaces: if set to True, will clean up the tokenization spaces. + """ + # Convert inputs to python lists + if isinstance(token_ids, flow.Tensor): + token_ids = token_ids.tolist() + + filtered_tokens = self.convert_ids_to_tokens( + token_ids, skip_special_tokens=skip_special_tokens + ) + + # To avoid mixing byte-level and unicode for byte-level BPT + # we need to build string separately for added tokens and byte-level tokens + # cf. https://github.com/huggingface/transformers/issues/1133 + sub_texts = [] + current_sub_text = [] + for token in filtered_tokens: + if skip_special_tokens and token in self.all_special_ids: + continue + if token in self.added_tokens_encoder: + if current_sub_text: + sub_texts.append(self.convert_tokens_to_string(current_sub_text)) + current_sub_text = [] + sub_texts.append(token) + else: + current_sub_text.append(token) + if current_sub_text: + sub_texts.append(self.convert_tokens_to_string(current_sub_text)) + + if spaces_between_special_tokens: + text = " ".join(sub_texts) + else: + text = "".join(sub_texts) + + if clean_up_tokenization_spaces: + clean_text = self.clean_up_tokenization(text) + return clean_text + else: + return text + + @property + def bos_token(self) -> str: + """ + :obj:`str`: Beginning of sentence token. Log an error if used while not having been set. + """ + if self._bos_token is None and self.verbose: + logger.error("Using bos_token, but it is not set yet.") + return None + return str(self._bos_token) + + @property + def eos_token(self) -> str: + """ + :obj:`str`: End of sentence token. Log an error if used while not having been set. + """ + if self._eos_token is None and self.verbose: + logger.error("Using eos_token, but it is not set yet.") + return None + return str(self._eos_token) + + @property + def unk_token(self) -> str: + """ + :obj:`str`: Unknown token. Log an error if used while not having been set. + """ + if self._unk_token is None and self.verbose: + logger.error("Using unk_token, but it is not set yet.") + return None + return str(self._unk_token) + + @property + def sep_token(self) -> str: + """ + :obj:`str`: Separation token, to separate context and query in an input sequence. + Log an error if used while not having been set. + """ + if self._sep_token is None and self.verbose: + logger.error("Using sep_token, but it is not set yet.") + return None + return str(self._sep_token) + + @property + def pad_token(self) -> str: + """ + :obj:`str`: Padding token. Log an error if used while not having been set. + """ + if self._pad_token is None and self.verbose: + logger.error("Using pad_token, but it is not set yet.") + return None + return str(self._pad_token) + + @property + def cls_token(self) -> str: + """ + :obj:`str`: Classification token, to extract a summary of an input sequence leveraging + self-attention along the full depth of the model. + Log an error if used while not having been set. + """ + if self._cls_token is None and self.verbose: + logger.error("Using cls_token, but it is not set yet.") + return None + return str(self._cls_token) + + @property + def mask_token(self) -> str: + """ + :obj:`str`: Mask token, to use when training a model with masked-language modeling. + Log an error if used while not having been set. + """ + if self._mask_token is None and self.verbose: + logger.error("Using mask_token, but it is not set yet.") + return None + return str(self._mask_token) + + @property + def eod_token(self) -> str: + """ + :obj:`str`: End of document token. Log an error if used while not having been set. + """ + if self._eod_token is None and self.verbose: + logger.error("Using eod_token, but it is not set yet.") + return None + return str(self._eod_token) + + @property + def start_token(self) -> str: + """ + :obj:`str`: Start token of sentence. Common name for bos_token and cls_token. + """ + if self._bos_token is not None and self._cls_token is not None: + if self._bos_token == self._cls_token: + return str(self._bos_token) + else: + logger.error("Conflict between bos_token and cls_token.") + return None + elif self._bos_token is None and self._cls_token is not None: + return str(self._cls_token) + elif self._bos_token is not None and self._cls_token is None: + return str(self._bos_token) + else: + logger.error("Using start_token, but it is not set yet.") + return None + + @property + def end_token(self) -> str: + """ + :obj:`str`: End token of sentence. Common name for eos_token and sep_token. + Note: eod_token is not considered, because it is often same with eos_token. + """ + if self._eos_token is not None and self._sep_token is not None: + if self._eos_token == self._sep_token: + return str(self._eos_token) + else: + logger.error("Conflict between eos_token and _sep_token.") + return None + elif self._eos_token is None and self._sep_token is not None: + return str(self._sep_token) + elif self._eos_token is not None and self._sep_token is None: + return str(self._eos_token) + else: + logger.error("Using end_token, but it is not set yet.") + return None + + @property + def additional_special_tokens(self) -> List[str]: + """ + :obj:`List[str]`: All the additional special tokens you may want to use. + Log an error if used while not having been set. + """ + if self._additional_special_tokens is None and self.verbose: + logger.error("Using additional_special_tokens, but it is not set yet.") + return None + return [str(tok) for tok in self._additional_special_tokens] + + @bos_token.setter + def bos_token(self, value): + self._bos_token = value + + @eos_token.setter + def eos_token(self, value): + self._eos_token = value + + @unk_token.setter + def unk_token(self, value): + self._unk_token = value + + @sep_token.setter + def sep_token(self, value): + self._sep_token = value + + @pad_token.setter + def pad_token(self, value): + self._pad_token = value + + @cls_token.setter + def cls_token(self, value): + self._cls_token = value + + @mask_token.setter + def mask_token(self, value): + self._mask_token = value + + @eod_token.setter + def eod_token(self, value): + self._eod_token = value + + @additional_special_tokens.setter + def additional_special_tokens(self, value): + self._additional_special_tokens = value + + @property + def bos_token_id(self) -> Optional[int]: + """ + :obj:`Optional[int]`: Id of the beginning of sentence token in the vocabulary. + Returns :obj:`None` if the token has not been set. + """ + if self._bos_token is None: + return None + return self.convert_tokens_to_ids(self.bos_token) + + @property + def eos_token_id(self) -> Optional[int]: + """ + :obj:`Optional[int]`: Id of the end of sentence token in the vocabulary. + Returns :obj:`None` if the token has not been set. + """ + if self._eos_token is None: + return None + return self.convert_tokens_to_ids(self.eos_token) + + @property + def unk_token_id(self) -> Optional[int]: + """ + :obj:`Optional[int]`: Id of the unknown token in the vocabulary. + Returns :obj:`None` if the token has not been set. + """ + if self._unk_token is None: + return None + return self.convert_tokens_to_ids(self.unk_token) + + @property + def sep_token_id(self) -> Optional[int]: + """ + :obj:`Optional[int]`: Id of the separation token in the vocabulary, + to separate context and query in an input sequence. + Returns :obj:`None` if the token has not been set. + """ + if self._sep_token is None: + return None + return self.convert_tokens_to_ids(self.sep_token) + + @property + def pad_token_id(self) -> Optional[int]: + """ + :obj:`Optional[int]`: Id of the padding token in the vocabulary. + Returns :obj:`None` if the token has not been set. + """ + if self._pad_token is None: + return None + return self.convert_tokens_to_ids(self.pad_token) + + @property + def cls_token_id(self) -> Optional[int]: + """ + :obj:`Optional[int]`: Id of the classification token in the vocabulary, + to extract a summary of an input sequence leveraging self-attention + along the full depth of the model. + Returns :obj:`None` if the token has not been set. + """ + if self._cls_token is None: + return None + return self.convert_tokens_to_ids(self.cls_token) + + @property + def mask_token_id(self) -> Optional[int]: + """ + :obj:`Optional[int]`: Id of the mask token in the vocabulary, used when training a + model with masked-language modeling. Returns :obj:`None` if the token has not been set. + """ + if self._mask_token is None: + return None + return self.convert_tokens_to_ids(self.mask_token) + + @property + def eod_token_id(self) -> Optional[int]: + """ + :obj:`Optional[int]`: Id of the end of document token in the vocabulary. + Returns :obj:`None` if the token has not been set. + """ + if self._eod_token is None: + return None + return self.convert_tokens_to_ids(self.eod_token) + + @property + def start_token_id(self) -> Optional[int]: + """ + :obj:`Optional[int]`: Id of the start token in the vocabulary. + Returns :obj:`None` if the token has not been set. + """ + start_token = self.start_token + if start_token is None: + return None + else: + return self.convert_tokens_to_ids(start_token) + + @property + def end_token_id(self) -> Optional[int]: + """ + :obj:`Optional[int]`: Id of the end token in the vocabulary. + Returns :obj:`None` if the token has not been set. + """ + end_token = self.end_token + if end_token is None: + return None + else: + return self.convert_tokens_to_ids(end_token) + + @property + def additional_special_tokens_ids(self) -> List[int]: + """ + :obj:`List[int]`: Ids of all the additional special tokens in the vocabulary. + Log an error if used while not having been set. + """ + return self.convert_tokens_to_ids(self.additional_special_tokens) + + @property + def special_tokens_map(self) -> Dict[str, Union[str, List[str]]]: + """ + A dictionary mapping special token class attributes + (:obj:`cls_token`, :obj:`unk_token`, etc.) to their values + (:obj:`''`, :obj:`''`, etc.). + """ + set_attr = {} + for attr in self.SPECIAL_TOKENS_ATTRIBUTES: + attr_value = getattr(self, "_" + attr) + if attr_value: + set_attr[attr] = attr_value + return set_attr + + @property + def all_special_tokens(self) -> List[str]: + """ + :obj:`List[str]`: All the special tokens + (:obj:`''`, :obj:`''`, etc.) mapped to class attributes. + """ + all_toks = [] + set_attr = self.special_tokens_map + for attr_value in set_attr.values(): + all_toks = all_toks + ( + list(attr_value) if isinstance(attr_value, (list, tuple)) else [attr_value] + ) + all_toks = list(set(all_toks)) + return all_toks + + @property + def all_special_ids(self) -> List[int]: + """ + :obj:`List[int]`: List the ids of the special tokens + (:obj:`''`, :obj:`''`, etc.) mapped to class attributes. + """ + all_toks = self.all_special_tokens + all_ids = list(self.convert_tokens_to_ids(all_toks)) + return all_ids + + @staticmethod + def clean_up_tokenization(out_string): + """Clean up a list of simple English tokenization artifacts like spaces before + punctuations and abbreviated forms. + """ + out_string = ( + out_string.replace(" .", ".") + .replace(" ?", "?") + .replace(" !", "!") + .replace(" ,", ",") + .replace(" ' ", "'") + .replace(" n't", "n't") + .replace(" 'm", "'m") + .replace(" do not", " don't") + .replace(" 's", "'s") + .replace(" 've", "'ve") + .replace(" 're", "'re") + ) + return out_string diff --git a/libai/tokenizer/tokenization_bert.py b/libai/tokenizer/tokenization_bert.py new file mode 100644 index 0000000000000000000000000000000000000000..4d71c7766e1fe466b15e9f49d745862809e80616 --- /dev/null +++ b/libai/tokenizer/tokenization_bert.py @@ -0,0 +1,514 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tokenization classes for bert (wordpieces).""" + +import collections +import logging +import os +import re +import unicodedata +from io import open +from typing import List, Optional + +from .tokenization_base import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace + +logger = logging.getLogger(__name__) + +VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"} + +PRETRAINED_VOCAB_FILES_MAP = { + "vocab_file": { + "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt", + "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt", + "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt", + "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt", + "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt", + } +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + "bert-base-uncased": 512, + "bert-large-uncased": 512, + "bert-base-cased": 512, + "bert-large-cased": 512, + "bert-base-chinese": 512, +} + +PRETRAINED_INIT_CONFIGURATION = { + "bert-base-uncased": {"do_lower_case": True}, + "bert-large-uncased": {"do_lower_case": True}, + "bert-base-cased": {"do_lower_case": False}, + "bert-large-cased": {"do_lower_case": False}, + "bert-base-chinese": {"do_lower_case": False}, +} + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + with open(vocab_file, "r", encoding="utf-8") as reader: + tokens = reader.readlines() + for index, token in enumerate(tokens): + token = token.rstrip("\n") + vocab[token] = index + return vocab + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +def _is_chinese_substr(char): + return re.findall("##[\u4E00-\u9FA5]", char) + + +class BertTokenizer(PreTrainedTokenizer): + """ + Construct a BERT tokenizer. Based on WordPiece. + + Args: + vocab_file (:obj:`str`): + Path to a one-wordpiece-per-line vocabulary file. + do_lower_case (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to lower case the input. + Only has an effect when do_basic_tokenize=True. + do_basic_tokenize (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to do basic tokenization before wordpiece. + never_split (:obj:`Iterable`, `optional`): + List of tokens which will never be split during tokenization. + Only has an effect when do_basic_tokenize=True. + tokenize_chinese_chars (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to tokenize Chinese characters. + This should likely be deactivated for Japanese, + see: https://github.com/huggingface/pytorch-pretrained-BERT/issues/328. + do_chinese_wwm (:obj:`bool`, `optional`, defaults to :obj:`False`): + Whether to do whole word masking for Chinese. + Chinese sentence will be segmented by a third-party tool first. + Each substr will be added '##' prefix and its index will be calucated by + id(##A) = id(A) + vocab_size. + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__( + self, + vocab_file, + do_lower_case=True, + do_basic_tokenize=True, + never_split=None, + unk_token="[UNK]", + sep_token="[SEP]", + pad_token="[PAD]", + cls_token="[CLS]", + mask_token="[MASK]", + tokenize_chinese_chars=True, + do_chinese_wwm=False, + add_bos_token=False, + **kwargs, + ): + super(BertTokenizer, self).__init__( + unk_token=unk_token, + sep_token=sep_token, + pad_token=pad_token, + cls_token=cls_token, + mask_token=mask_token, + **kwargs, + ) + if not os.path.isfile(vocab_file): + raise ValueError( + "Can't find a vocabulary file at path '{}'. To load the " + "vocabulary from a Google pretrained model use " + "`tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format( + vocab_file + ) + ) + self.vocab = load_vocab(vocab_file) + self.ids_to_tokens = collections.OrderedDict( + [(ids, tok) for tok, ids in self.vocab.items()] + ) + self.do_basic_tokenize = do_basic_tokenize + if do_basic_tokenize: + if do_chinese_wwm: + self.basic_tokenizer = BasicTokenizerWithChineseWWM( + do_lower_case=do_lower_case, + never_split=never_split, + tokenize_chinese_chars=tokenize_chinese_chars, + ) + else: + self.basic_tokenizer = BasicTokenizer( + do_lower_case=do_lower_case, + never_split=never_split, + tokenize_chinese_chars=tokenize_chinese_chars, + ) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab, unk_token=self.unk_token) + self.add_bos_token = add_bos_token + + @property + def vocab_size(self): + return len(self.vocab) + + def get_vocab(self): + return dict(self.vocab, **self.added_tokens_encoder) + + def _tokenize(self, text): + split_tokens = [] + if self.do_basic_tokenize: + for token in self.basic_tokenizer.tokenize(text, never_split=self.all_special_tokens): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + else: + split_tokens = self.wordpiece_tokenizer.tokenize(text) + return split_tokens + + def _convert_token_to_id(self, token): + """Converts a token (str) in an id using the vocab. + For Chinese substr, id = vocab_size + id(substr.remove(##)). + """ + index = self.vocab.get(token, self.vocab.get(self.unk_token)) + return index + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab. + For Chinese substr, id = vocab_size + id(substr.remove(##)). + """ + token = self.ids_to_tokens.get(index, self.unk_token) + return token + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (string) to a single string.""" + out_string = " ".join(tokens).replace(" ##", "").strip() + return out_string + + def build_inputs_with_special_tokens( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """Add special tokens to a sequence or a pair of sequence. + BERT format sentence input: + + - single sequence: [CLS] tokens_a [SEP] + - pair of sequences: [CLS] tokens_a [SEP] tokens_b [SEP] + + Args: + token_ids_0 (List[int]): The token ids of sentence 0. + token_ids_1 (List[int], optional): The token ids of sentence 1. Defaults to None. + + Returns: + :obj:`List[str]`: The sequence after adding special toekens. + """ + if self.add_bos_token: + cls = [self.cls_token_id] + sep = [self.sep_token_id] + else: + cls = [] + sep = [] + + if token_ids_1 is None: + return cls + token_ids_0 + sep + + return cls + token_ids_0 + sep + token_ids_1 + sep + + def save_vocabulary(self, save_directory, filename_prefix=None): + """Save the tokenizer vocabulary to a directory or file.""" + index = 0 + if os.path.isdir(save_directory): + vocab_file = os.path.join( + save_directory, + (filename_prefix + "-" if filename_prefix else "") + + VOCAB_FILES_NAMES["vocab_file"], + ) + else: + vocab_file = (filename_prefix + "-" if filename_prefix else "") + save_directory + with open(vocab_file, "w", encoding="utf-8") as writer: + for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]): + if index != token_index: + logger.warning( + "Saving vocabulary to {}: vocabulary indices are not consecutive." + " Please check that the vocabulary is not corrupted!".format(vocab_file) + ) + index = token_index + writer.write(token + "\n") + index += 1 + return (vocab_file,) + + +class BasicTokenizer(object): + """ + Constructs a BasicTokenizer that will run basic + tokenization (punctuation splitting, lower casing, etc.). + """ + + def __init__(self, do_lower_case=True, never_split=None, tokenize_chinese_chars=True): + """Constructs a BasicTokenizer. + Args: + **do_lower_case**: Whether to lower case the input. + **never_split**: (`optional`) list of str + Kept for backward compatibility purposes. + Now implemented directly at the base class level + (see :func:`PreTrainedTokenizer.tokenize`) + List of token not to split. + **tokenize_chinese_chars**: (`optional`) boolean (default True) + Whether to tokenize Chinese characters. + This should likely be deactivated for Japanese: + see: https://github.com/huggingface/pytorch-pretrained-BERT/issues/328 + """ + if never_split is None: + never_split = [] + self.do_lower_case = do_lower_case + self.never_split = set(never_split) + self.tokenize_chinese_chars = tokenize_chinese_chars + + def tokenize(self, text, never_split=None): + """ + Basic Tokenization of a piece of text. + Split on "white spaces" only, for sub-word tokenization, see WordPieceTokenizer. + + Args: + **never_split**: (`optional`) list of str + Kept for backward compatibility purposes. + Now implemented directly at the base class level + (see :func:`PreTrainedTokenizer.tokenize`) + List of token not to split. + """ + # union() returns a new set by concatenating the two sets. + never_split = self.never_split.union(set(never_split)) if never_split else self.never_split + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + if self.tokenize_chinese_chars: + text = self._tokenize_chinese_chars(text) + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case and token not in never_split: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token, never_split)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text, never_split=None): + """Splits punctuation on a piece of text.""" + if never_split is not None and text in never_split: + return [text] + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ( + (cp >= 0x4E00 and cp <= 0x9FFF) + or (cp >= 0x3400 and cp <= 0x4DBF) # + or (cp >= 0x20000 and cp <= 0x2A6DF) # + or (cp >= 0x2A700 and cp <= 0x2B73F) # + or (cp >= 0x2B740 and cp <= 0x2B81F) # + or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0xF900 and cp <= 0xFAFF) + or (cp >= 0x2F800 and cp <= 0x2FA1F) # + ): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xFFFD or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +class BasicTokenizerWithChineseWWM(BasicTokenizer): + """Pre-segmentation for Chinese sentences, which will be used in whole word mask.""" + + def __init__(self, do_lower_case=True, never_split=None, tokenize_chinese_chars=True): + super(BasicTokenizerWithChineseWWM, self).__init__( + do_lower_case=do_lower_case, + never_split=never_split, + tokenize_chinese_chars=tokenize_chinese_chars, + ) + try: + import jieba + + self.pre_tokenizer = lambda x: jieba.lcut(x, HMM=False) + except ImportError: + raise (ImportError("Chinese whole word mask need jieba")) + + def _tokenize_chinese_chars(self, text): + """For Chinese pieces, uses jieba to segment the words and + adds whitespace around CJK character.""" + output = [] + piece = "" + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + piece += char + else: + chinese_words = self.pre_tokenizer(piece) + for word in chinese_words: + output.append(" ") + output.append(word) + output.append(" ") + output.append(char) + piece = "" + + chinese_words = self.pre_tokenizer(piece) + for word in chinese_words: + output.append(" ") + output.append(word) + output.append(" ") + + return "".join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenization.""" + + def __init__(self, vocab, unk_token, max_input_chars_per_word=100): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + + input = "有没有" + output = ["有", "##没", "##有"] + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer`. + Returns: + A list of wordpiece tokens. + """ + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + + if substr.startswith("##"): + if _is_chinese_substr(substr): + if substr[2:] in self.vocab: # for Chinese substr + cur_substr = substr + break + else: + if substr in self.vocab: # for English substr + cur_substr = substr + break + else: + if ( + substr in self.vocab + ): # non-substr, maybe character or whole Chinese word + cur_substr = substr + break + end -= 1 + + if cur_substr is None: + is_bad = True + break + + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens diff --git a/libai/tokenizer/tokenization_gpt2.py b/libai/tokenizer/tokenization_gpt2.py new file mode 100644 index 0000000000000000000000000000000000000000..e8ba83ead38cd7c4f5c7ff4568453b4ca9e46c8b --- /dev/null +++ b/libai/tokenizer/tokenization_gpt2.py @@ -0,0 +1,272 @@ +# coding=utf-8 +# Copyright 2018 The Open AI Team Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tokenization classes for OpenAI GPT (BPE).""" + +import json +import logging +import os +from functools import lru_cache +from io import open +from typing import List, Optional + +import regex as re + +from .tokenization_base import PreTrainedTokenizer + +logger = logging.getLogger(__name__) + +VOCAB_FILES_NAMES = { + "vocab_file": "vocab.json", + "merges_file": "merges.txt", +} + +PRETRAINED_VOCAB_FILES_MAP = { + "vocab_file": {"gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json"}, + "merges_file": {"gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt"}, +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + "gpt2": 1024, +} + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping + to whitespace/control characters the bpe code barfs on. + + The reversible bpe codes work on unicode strings. This means you need a large # of unicode + characters in your vocab if you want to avoid UNKs. When you're at something like a 10B token + dataset you end up needing around 5K for decent coverage. This is a significant percentage + of your normal, say, 32K bpe vocab. To avoid that, we want lookup tables between utf-8 + bytes and unicode strings. + """ + bs = ( + list(range(ord("!"), ord("~") + 1)) + + list(range(ord("¡"), ord("¬") + 1)) + + list(range(ord("®"), ord("ÿ") + 1)) + ) + cs = bs[:] + n = 0 + for b in range(2 ** 8): + if b not in bs: + bs.append(b) + cs.append(2 ** 8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """ + Return set of symbol pairs in a word. + + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +class GPT2Tokenizer(PreTrainedTokenizer): + """ + Construct a GPT-2 tokenizer. Based on byte-level Byte-Pair-Encoding. + + Args: + vocab_file (:obj:`str`): + Path to the vocabulary file. + merges_file (:obj:`str`): + Path to the merges file. + errors (:obj:`str`, `optional`, defaults to :obj:`"replace"`): + Paradigm to follow when decoding bytes to UTF-8. See `bytes.decode + `__ for more information. + unk_token (:obj:`str`, `optional`, defaults to :obj:`<|endoftext|>`): + The unknown token. A token that is not in the vocabulary cannot be + converted to an ID and is set to be this token instead. + bos_token (:obj:`str`, `optional`, defaults to :obj:`<|endoftext|>`): + The beginning of sequence token. + eos_token (:obj:`str`, `optional`, defaults to :obj:`<|endoftext|>`): + The end of sequence token. + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__( + self, + vocab_file, + merges_file, + errors="replace", + unk_token="<|endoftext|>", + bos_token="<|endoftext|>", + eos_token="<|endoftext|>", + add_bos_token=False, + **kwargs, + ): + super(GPT2Tokenizer, self).__init__( + bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, **kwargs + ) + + self.encoder = json.load(open(vocab_file, encoding="utf-8")) + self.decoder = {v: k for k, v in self.encoder.items()} + self.errors = errors # how to handle errors in decoding + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + bpe_data = open(merges_file, encoding="utf-8").read().split("\n")[1:-1] + bpe_merges = [tuple(merge.split()) for merge in bpe_data] + self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges)))) + self.cache = {} + + # Should haved added re.IGNORECASE so BPE merges can happen for + # capitalized versions of contractions + self.pat = re.compile( + r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" + ) + self.add_bos_token = add_bos_token + + @property + def vocab_size(self): + return len(self.encoder) + + def get_vocab(self): + return dict(self.encoder, **self.added_tokens_encoder) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token) + pairs = get_pairs(word) + + if not pairs: + return token + + while True: + bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf"))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except: # noqa + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word) - 1 and word[i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = " ".join(word) + self.cache[token] = word + return word + + def _tokenize(self, text): + """Tokenize a string.""" + bpe_tokens = [] + for token in re.findall(self.pat, text): + # Maps all our bytes to unicode strings, avoiding control tokens + # of the BPE (spaces in our case) + token = "".join(self.byte_encoder[b] for b in token.encode("utf-8")) + bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" ")) + return bpe_tokens + + def _convert_token_to_id(self, token): + """Converts a token (str) in an id using the vocab.""" + return self.encoder.get(token, self.encoder.get(self.unk_token)) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + return self.decoder.get(index) + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (string) to a single string.""" + text = "".join(tokens) + text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors) + return text + + def build_inputs_with_special_tokens( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """Add special tokens to a sequence or a pair of sequence. + GPT2 format sentence input: + + - single sequence: <|endoftext|> tokens_a + - pair of sequences: <|endoftext|> tokens_a <|endoftext|> tokens_b + + Args: + token_ids_0 (List[int]): The token ids of sentence 0. + token_ids_1 (List[int], optional): The token ids of sentence 1. Defaults to None. + + Returns: + :obj:`List[str]`: The sequence after adding special toekens. + """ + if self.add_bos_token: + bos = [self.bos_token_id] + else: + bos = [] + + if token_ids_1 is None: + return bos + token_ids_0 + + return bos + token_ids_0 + bos + token_ids_1 + + def save_vocabulary(self, save_directory, filename_prefix=None): + if not os.path.isdir(save_directory): + logger.error(f"Vocabulary path ({save_directory}) should be a directory") + return + vocab_file = os.path.join( + save_directory, + (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"], + ) + merge_file = os.path.join( + save_directory, + (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"], + ) + + with open(vocab_file, "w", encoding="utf-8") as f: + f.write(json.dumps(self.encoder, ensure_ascii=False)) + + index = 0 + with open(merge_file, "w", encoding="utf-8") as writer: + writer.write("#version: 0.2\n") + for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]): + if index != token_index: + logger.warning( + f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive." + " Please check that the tokenizer is not corrupted!" + ) + index = token_index + writer.write(" ".join(bpe_tokens) + "\n") + index += 1 + + return (vocab_file, merge_file) diff --git a/libai/tokenizer/tokenization_roberta.py b/libai/tokenizer/tokenization_roberta.py new file mode 100644 index 0000000000000000000000000000000000000000..eb1d29fa8d0d3665a108868560e85d8c9a9c2894 --- /dev/null +++ b/libai/tokenizer/tokenization_roberta.py @@ -0,0 +1,328 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tokenization classes for RoBERTa.""" + +import json +import logging +import os +from functools import lru_cache +from typing import List, Optional, Tuple + +import regex as re + +from .tokenization_base import PreTrainedTokenizer + +logger = logging.getLogger(__name__) + +VOCAB_FILES_NAMES = { + "vocab_file": "vocab.json", + "merges_file": "merges.txt", +} + +PRETRAINED_VOCAB_FILES_MAP = { + "vocab_file": { + "roberta-base": "https://huggingface.co/roberta-base/resolve/main/vocab.json", + "roberta-large": "https://huggingface.co/roberta-large/resolve/main/vocab.json", + "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json", + }, + "merges_file": { + "roberta-base": "https://huggingface.co/roberta-base/resolve/main/merges.txt", + "roberta-large": "https://huggingface.co/roberta-large/resolve/main/merges.txt", + "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt", + }, +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + "roberta-base": 512, + "roberta-large": 512, + "roberta-large-mnli": 512, +} + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to + whitespace/control characters the bpe code barfs on. + + The reversible bpe codes work on unicode strings. This means you need a large # of unicode + characters in your vocab if you want to avoid UNKs. When you're at something like a 10B token + dataset you end up needing around 5K for decent coverage. This is a significant percentage of + your normal, say, 32K bpe vocab. To avoid that, we want lookup tables between utf-8 bytes and + unicode strings. + """ + bs = ( + list(range(ord("!"), ord("~") + 1)) + + list(range(ord("¡"), ord("¬") + 1)) + + list(range(ord("®"), ord("ÿ") + 1)) + ) + cs = bs[:] + n = 0 + for b in range(2 ** 8): + if b not in bs: + bs.append(b) + cs.append(2 ** 8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """ + Return set of symbol pairs in a word. + + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +class RobertaTokenizer(PreTrainedTokenizer): + """Constructs a RoBERTa tokenizer, derived from the GPT-2 tokenizer, + using byte-level Byte-Pair-Encoding. + + Args: + vocab_file (:obj:`str`): + Path to the vocabulary file. + merges_file (:obj:`str`): + Path to the merges file. + errors (:obj:`str`, `optional`, defaults to :obj:`"replace"`): + Paradigm to follow when decoding bytes to UTF-8. See `bytes.decode + `__ for more information. + bos_token (:obj:`str`, `optional`, defaults to ``): + The beginning of sequence token. + eos_token (:obj:`str`, `optional`, defaults to ``): + The end of sequence token. + cls_token (:obj:`str`, `optional`, defaults to ``): + The first token of the sequence when built with special tokens. + unk_token (:obj:`str`, `optional`, defaults to ``): + The unknown token. A token that is not in the vocabulary cannot be + converted to an ID and is set to be this token instead. + pad_token (:obj:`str`, `optional`, defaults to ``): A special token + used to make arrays of tokens the same size for batching purpose. + Will then be ignored by attention mechanisms or loss computation. + mask_token (:obj:`str`, `optional`, defaults to ``): A special token + representing a masked token (used by masked-language modeling pretraining + objectives, like BERT). + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + model_input_names = ["input_ids", "attention_mask"] + + def __init__( + self, + vocab_file, + merges_file, + errors="replace", + bos_token="", + eos_token="", + sep_token="", + cls_token="", + unk_token="", + pad_token="", + mask_token="", + add_bos_token=False, + **kwargs, + ): + super(RobertaTokenizer, self).__init__( + errors=errors, + bos_token=bos_token, + eos_token=eos_token, + sep_token=sep_token, + cls_token=cls_token, + unk_token=unk_token, + pad_token=pad_token, + mask_token=mask_token, + **kwargs, + ) + + with open(vocab_file, encoding="utf-8") as file: + self.encoder = json.load(file) + self.decoder = {v: k for k, v in self.encoder.items()} + self.errors = errors + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + with open(merges_file, encoding="utf-8") as file: + bpe_merges = file.read().split("\n")[1:-1] + bpe_merges = [tuple(merge.split()) for merge in bpe_merges] + self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges)))) + self.cache = {} + self.pat = re.compile( + r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" + ) + self.add_bos_token = add_bos_token + + @property + def vocab_size(self): + return len(self.encoder) + + def get_vocab(self): + return dict(self.encoder, **self.added_tokens_encoder) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token) + pairs = get_pairs(word) + + if not pairs: + return token + + while True: + bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf"))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + except ValueError: + new_word.extend(word[i:]) + break + else: + new_word.extend(word[i:j]) + i = j + + if word[i] == first and i < len(word) - 1 and word[i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = " ".join(word) + self.cache[token] = word + return word + + def _tokenize(self, text): + """Tokenize a string.""" + bpe_tokens = [] + for token in re.findall(self.pat, text): + token = "".join(self.byte_encoder[b] for b in token.encode("utf-8")) + bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" ")) + return bpe_tokens + + def _convert_token_to_id(self, token): + """Converts a token (str) in an id using the vocab.""" + return self.encoder.get(token, self.encoder.get(self.unk_token)) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + return self.decoder.get(index) + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (string) in a single string.""" + text = "".join(tokens) + text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors) + return text + + def build_inputs_with_special_tokens( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """Add special tokens to a sequence or a pair of sequence. + RoBERTa format sentence input: + + - single sequence: [CLS] tokens_a [SEP] + - pair of sequences: [CLS] tokens_a [SEP] tokens_b [SEP] + + Args: + token_ids_0 (List[int]): The token ids of sentence 0. + token_ids_1 (List[int], optional): The token ids of sentence 1. Defaults to None. + + Returns: + :obj:`List[str]`: The sequence after adding special toekens. + """ + if self.add_bos_token: + cls = [self.cls_token_id] + sep = [self.sep_token_id] + else: + cls = [] + sep = [] + + if token_ids_1 is None: + return cls + token_ids_0 + sep + + return cls + token_ids_0 + sep + token_ids_1 + sep + + def save_vocabulary( + self, save_directory: str, filename_prefix: Optional[str] = None + ) -> Tuple[str]: + if not os.path.isdir(save_directory): + logger.error(f"Vocabulary path ({save_directory}) should be a directory") + return + vocab_file = os.path.join( + save_directory, + (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"], + ) + merge_file = os.path.join( + save_directory, + (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"], + ) + + with open(vocab_file, "w", encoding="utf-8") as f: + f.write(json.dumps(self.encoder, ensure_ascii=False)) + + index = 0 + with open(merge_file, "w", encoding="utf-8") as writer: + writer.write("#version: 0.2\n") + for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]): + if index != token_index: + logger.warning( + f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive." + " Please check that the tokenizer is not corrupted!" + ) + index = token_index + writer.write(" ".join(bpe_tokens) + "\n") + index += 1 + + return vocab_file, merge_file + + def create_token_type_ids_from_sequences( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Create a mask from the two sequences passed to be used in a sequence-pair + classification task. RoBERTa does not make use of token type ids, therefore + a list of zeros is returned. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, `optional`): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of zeros. + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0] diff --git a/libai/tokenizer/tokenization_t5.py b/libai/tokenizer/tokenization_t5.py new file mode 100644 index 0000000000000000000000000000000000000000..810dff4fc7e7d1e9fef2b5251348e490b6e9d152 --- /dev/null +++ b/libai/tokenizer/tokenization_t5.py @@ -0,0 +1,202 @@ +# coding=utf-8 +# Copyright 2018 T5 Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tokenization class for Google T5 (sentence piece).""" + +import logging +import os +import warnings +from shutil import copyfile +from typing import List, Optional + +import regex as re +import sentencepiece as spm + +from .tokenization_base import PreTrainedTokenizer + +logger = logging.getLogger(__name__) + +VOCAB_FILES_NAMES = {"vocab_file": "spiece.model"} + +PRETRAINED_VOCAB_FILES_MAP = { + "vocab_file": {"t5-base": "https://huggingface.co/t5-base/resolve/main/spiece.model"} +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + "t5-base": 512, +} + + +class T5Tokenizer(PreTrainedTokenizer): + """ + Construct a T5 tokenizer. Based on `SentencePiece `. + + Args: + vocab_file (:obj:`str`): + Path to the vocabulary file. + eos_token (:obj:`str`, `optional`, defaults to :obj:`""`): + The end of sequence token. + unk_token (:obj:`str`, `optional`, defaults to :obj:`""`): + The unknown token. A token that is not in the vocabulary cannot + be converted to an ID and is set to be this token instead. + pad_token (:obj:`str`, `optional`, defaults to :obj:`""`): + The token used for padding, for example when batching sequences of different lengths. + extra_ids (:obj:`int`, `optional`, defaults to 100): + Add a number of extra ids added to the end of the vocabulary for use + as sentinels. These tokens are accessible as "" where + "{%d}" is a number between 0 and extra_ids-1. Extra tokens are indexed + from the end of the vocabulary up to beginning ("" is the + last token in the vocabulary like in T5 preprocessing see `here + `__). + additional_special_tokens (:obj:`List[str]`, `optional`): + Additional special tokens used by the tokenizer. + + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__( + self, + vocab_file, + eos_token="", + unk_token="", + pad_token="", + extra_ids=100, + additional_special_tokens=None, + add_bos_token=False, + **kwargs, + ): + # Add extra_ids to the special token list + if extra_ids > 0 and additional_special_tokens is None: + additional_special_tokens = [f"" for i in range(extra_ids)] + elif extra_ids > 0 and additional_special_tokens is not None: + extra_tokens = len( + set(filter(lambda x: bool("extra_id" in str(x)), additional_special_tokens)) + ) + if extra_tokens != extra_ids: + raise ValueError( + f"Both extra_ids ({extra_ids}) and additional_special_tokens " + f"({additional_special_tokens}) are privided to T5Tokenizer. " + "In this case the additional_special_tokens must include the extra_ids tokens" + ) + + super().__init__( + eos_token=eos_token, + unk_token=unk_token, + pad_token=pad_token, + additional_special_tokens=additional_special_tokens, + **kwargs, + ) + + self.vocab_file = vocab_file + self._extra_ids = extra_ids + + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(vocab_file) + self.add_bos_token = add_bos_token + + @property + def vocab_size(self): + return self.sp_model.get_piece_size() + self._extra_ids + + def get_vocab(self): + vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} + vocab.update(self.added_tokens_encoder) + return vocab + + def _tokenize(self, text): + """Tokenize a string.""" + pieces = self.sp_model.encode(text, out_type=str) + return pieces + + def _convert_token_to_id(self, token): + """Converts a token (str) in an id using the vocab.""" + if token.startswith("", token) + num = int(match.group(1)) + return self.vocab_size - num - 1 + return self.sp_model.piece_to_id(token) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + if index < self.sp_model.get_piece_size(): + token = self.sp_model.IdToPiece(index) + else: + token = f"" + return token + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (string) to a single string.""" + current_sub_tokens = [] + out_string = "" + for token in tokens: + # make sure that special tokens are not decoded using sentencepiece model + if token in self.all_special_tokens: + out_string += self.sp_model.decode_pieces(current_sub_tokens) + token + " " + current_sub_tokens = [] + else: + current_sub_tokens.append(token) + out_string += self.sp_model.decode_pieces(current_sub_tokens) + return out_string.strip() + + def _add_eos_if_not_present(self, token_ids): + if not self.add_bos_token: + return token_ids + if len(token_ids) > 0 and token_ids[-1] == self.eos_token_id: + warnings.warn("This sequence already has {self.eos_token}.") + return token_ids + else: + return token_ids + [self.eos_token_id] + + def build_inputs_with_special_tokens( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """Add special tokens to a sequence or a pair of sequence. + T5 format sentence input: + + - single sequence: tokens_a + - pair of sequences: tokens_a tokens_b + + Args: + token_ids_0 (List[int]): The token ids of sentence 0. + token_ids_1 (List[int], optional): The token ids of sentence 1. Defaults to None. + + Returns: + :obj:`List[str]`: The sequence after adding special toekens. + """ + token_ids_0 = self._add_eos_if_not_present(token_ids_0) + if token_ids_1 is None: + return token_ids_0 + else: + token_ids_1 = self._add_eos_if_not_present(token_ids_1) + return token_ids_0 + token_ids_1 + + def save_vocabulary(self, save_directory, filename_prefix=None): + """Save the tokenizer vocabulary to a directory or file.""" + if not os.path.isdir(save_directory): + logger.error(f"Vocabulary path ({save_directory}) should be a directory") + return + out_vocab_file = os.path.join( + save_directory, + (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"], + ) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): + copyfile(self.vocab_file, out_vocab_file) + logger.info(f"Copy vocab file to {out_vocab_file}") + + return (out_vocab_file,) diff --git a/libai/utils/__init__.py b/libai/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b5f2bb5294bd61570d939e1f776c4b1f4c56b39e --- /dev/null +++ b/libai/utils/__init__.py @@ -0,0 +1,14 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/libai/utils/__pycache__/__init__.cpython-39.pyc b/libai/utils/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9c0f07f7ef122dea39aaacc6f9d8579a7f9d1790 Binary files /dev/null and b/libai/utils/__pycache__/__init__.cpython-39.pyc differ diff --git a/libai/utils/__pycache__/checkpoint.cpython-39.pyc b/libai/utils/__pycache__/checkpoint.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9a563fc4f2a51999ba71f4c6deb70e86ffc164de Binary files /dev/null and b/libai/utils/__pycache__/checkpoint.cpython-39.pyc differ diff --git a/libai/utils/__pycache__/distributed.cpython-39.pyc b/libai/utils/__pycache__/distributed.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4968c5bfc4a23d7a01f0e1174c22e4793621d303 Binary files /dev/null and b/libai/utils/__pycache__/distributed.cpython-39.pyc differ diff --git a/libai/utils/__pycache__/download.cpython-39.pyc b/libai/utils/__pycache__/download.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e737b3ca60dc91353527fedce860543e5e231328 Binary files /dev/null and b/libai/utils/__pycache__/download.cpython-39.pyc differ diff --git a/libai/utils/__pycache__/events.cpython-39.pyc b/libai/utils/__pycache__/events.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..076e170f185b5deca04e55fe446a18f48b5b92e1 Binary files /dev/null and b/libai/utils/__pycache__/events.cpython-39.pyc differ diff --git a/libai/utils/__pycache__/file_io.cpython-39.pyc b/libai/utils/__pycache__/file_io.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ad898d270369d3126b53cd53333032f51cf0b0d9 Binary files /dev/null and b/libai/utils/__pycache__/file_io.cpython-39.pyc differ diff --git a/libai/utils/__pycache__/file_utils.cpython-39.pyc b/libai/utils/__pycache__/file_utils.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ebd7e9bb33e97051cf83212ca846647e30538bcd Binary files /dev/null and b/libai/utils/__pycache__/file_utils.cpython-39.pyc differ diff --git a/libai/utils/__pycache__/history_buffer.cpython-39.pyc b/libai/utils/__pycache__/history_buffer.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8eb281e3c13e5649657e2c637ab15653746fb4a0 Binary files /dev/null and b/libai/utils/__pycache__/history_buffer.cpython-39.pyc differ diff --git a/libai/utils/__pycache__/logger.cpython-39.pyc b/libai/utils/__pycache__/logger.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aa37ff6e8dfec603364d5e4e00fe560cd1a1df0f Binary files /dev/null and b/libai/utils/__pycache__/logger.cpython-39.pyc differ diff --git a/libai/utils/__pycache__/non_blocking_io.cpython-39.pyc b/libai/utils/__pycache__/non_blocking_io.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4f7162b91bcf72fa27c191632fe61edc1ec1ab52 Binary files /dev/null and b/libai/utils/__pycache__/non_blocking_io.cpython-39.pyc differ diff --git a/libai/utils/__pycache__/timer.cpython-39.pyc b/libai/utils/__pycache__/timer.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a23ac55d829b4c6d05c28a98620603a2a224b3b3 Binary files /dev/null and b/libai/utils/__pycache__/timer.cpython-39.pyc differ diff --git a/libai/utils/checkpoint.py b/libai/utils/checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..477ff75d6561c4b4be44de0f3d088b8417e18935 --- /dev/null +++ b/libai/utils/checkpoint.py @@ -0,0 +1,516 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import logging +import os +from collections import defaultdict +from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple + +import numpy as np +import oneflow as flow +from oneflow import nn +from termcolor import colored + +import libai.utils.distributed as dist +from libai.utils.file_io import HTTPURLHandler, PathManagerBase + + +class _IncompatibleKeys( + NamedTuple( + # pyre-fixme[10]: Name `IncompatibleKeys` is used but not defined. + "IncompatibleKeys", + [ + ("missing_keys", List[str]), + ("unexpected_keys", List[str]), + # pyre-fixme[24]: Generic type `tuple` expects at least 1 type parameter. + # pyre-fixme[24]: Generic type `tuple` expects at least 1 type parameter. + # pyre-fixme[24]: Generic type `tuple` expects at least 1 type parameter. + ("incorrect_shapes", List[Tuple]), + ], + ) +): + pass + + +class Checkpointer(object): + """ + A checkpointer that can save/load model as well as extra checkpointable + objects. + """ + + # NOTE: only support data_parallel for saving model + # TODO: save model: support model_parallel and pipeline parallel + + def __init__( + self, + model: nn.Module, + save_dir: str = "", + *, + save_to_disk: bool = True, + **checkpointables: object, + ): + """ + Args: + model (nn.Module): model. + save_dir (str): a directory to save and find checkpoints. + save_to_disk (bool): if True, save checkpoint to disk, otherwise + disable saving for this checkpointer. + checkpointables (object): any checkpointable objects, i.e., objects + that have the `state_dict()` and `load_state_dict()` method. For + example, it can be used like + `Checkpointer(model, "dir", optimizer=optimizer)`. + """ + self.model = model + self.checkpointables = copy.copy(checkpointables) + self.logger = logging.getLogger(__name__) + self.save_dir = save_dir + self.save_to_disk = save_to_disk + # Default PathManager, support HTTP URLs + # A user may want to use a different project-specific PathManagerBase' + self.path_manager: PathManagerBase = PathManagerBase() + self.path_manager.register_handler(HTTPURLHandler()) + + def save(self, name: str, **kwargs: Dict[str, str]): + """ + Dump model and checkpointables to a file. + + Args: + name (str): name of the file. + kwargs (dict): extra arbitrary data to save. + """ + + data = {} + data["model"] = self.model.state_dict() + for key, obj in self.checkpointables.items(): + data[key] = obj.state_dict() + data.update(kwargs) + + basename = name + save_dir = os.path.join(self.save_dir, basename) + assert os.path.basename(save_dir) == basename, basename + if not self.path_manager.exists(save_dir): + self.path_manager.mkdirs(save_dir) + self.logger.info("Saving checkpoint to {}".format(save_dir)) + + for save_name in data: + if save_name == "iteration": + continue + save_file = os.path.join(save_dir, save_name) + # If directory existing, remove it for saving + if self.path_manager.exists(save_file): + self.path_manager.mkdirs(save_file) + + flow.save(data[save_name], save_file, global_dst_rank=0) + + if basename != "model_best": + self.tag_last_checkpoint(basename) + + def load(self, path: str, checkpointables: Optional[List[str]] = None) -> object: + """ + Load from the given checkpoint. When path points to network file, this + function has to be called on all ranks. + + Args: + path (str): path or url to the checkpoint. If empty, will not load + anything. + checkpointables (list): List of checkpointable names to load. If not + specified (None), will load all the possible checkpointables. + + Returns: + dict: + extra data loaded from the checkpoint that has not been + processed. For example, those saved with + :meth:`.save(**extra_data)`. + """ + if not path: + # no checkpoint provided + self.logger.info("No checkpoint found. Training model from scratch") + return {} + self.logger.info("Loading checkpoint from {}".format(path)) + + checkpoint = self._load_file(path) + incompatible = self._load_model(checkpoint) + if incompatible is not None: # handle some existing subclasses that returns None + self._log_incompatible_keys(incompatible) + + for key in self.checkpointables if checkpointables is None else checkpointables: + if key in checkpoint: # pyre-ignore + self.logger.info("Loading {} from {}".format(key, path)) + obj = self.checkpointables[key] + obj.load_state_dict(checkpoint.pop(key)) # pyre-ignore + + # return any further checkpoint data + return checkpoint + + def has_checkpoint(self): + """ + Returns: + bool: whether a checkpoint exists in the target directory. + """ + save_file = os.path.join(self.save_dir, "last_checkpoint") + return self.path_manager.exists(save_file) + + def get_checkpoint_file(self): + """ + Returns: + str: The latest checkpoint file in target directory. + """ + save_file = os.path.join(self.save_dir, "last_checkpoint") + try: + # load checkpoint file in rank0 + if flow.env.get_rank() == 0: + with open(save_file, "r") as f: + last_saved = f.read().strip() + else: + last_saved = None + # broadcast checkpoint file to other ranks + last_saved = dist.broadcast_py_object(last_saved, src=0) + except IOError: + # if file doesn't exist, maybe because it has just been + # deleted by a separate process + return "" + return os.path.join(self.save_dir, last_saved) + + def resume_or_load(self, path: str, *, resume: bool = True): + """ + If `resume` is True, this method attempts to resume from the last + checkpoint (if exists). Otherwise, load checkpoint from the given path. + This is useful when restarting an interrupted training job. + Args: + path (str): path to the checkpoint. + resume (bool): if True, resume from the last checkpoint if it exists. + Returns: + same as :meth:`load`. + """ + if resume and self.has_checkpoint(): + path = self.get_checkpoint_file() + return self.load(path) + else: + return self.load(path, checkpointables=[]) + + def tag_last_checkpoint(self, last_filename_basename: str): + """ + Tag the last checkpoint. + Args: + last_filename_basename (str): the basename of the last filename. + """ + save_file = os.path.join(self.save_dir, "last_checkpoint") + with self.path_manager.open(save_file, "w") as f: + f.write(last_filename_basename) # pyre-ignore + + def _load_file(self, f: str): + """ + Load a checkpoint file. Can be overwritten by subclasses to support + different formats. + Args: + f (str): a locally mounted file path. + Returns: + dict: with keys "model" and optionally others that are saved by + the checkpointer dict["model"] must be a dict which maps strings + to flow.Tensor or numpy arrays. + """ + data = {} + keys = self.path_manager.ls(f) + # broadcast checkpointer keys to other ranks + keys = dist.broadcast_py_object(keys, src=0) + for key in keys: + data[key] = flow.load(os.path.join(f, key), global_src_rank=0) + try: + data["iter"] = int(f.split("_")[-1]) + except: # noqa + self.logger.info(f"iter info in {f} not found, set iter to 0") + data["iter"] = 0 + return data + + def _load_model(self, checkpoint: Any): + """ + Load weights from a checkpoint. + Args: + checkpoint (Any): checkpoint contains the weights. + """ + checkpoint_state_dict = checkpoint.pop("model") + self._convert_ndarray_to_tensor(checkpoint_state_dict) + + # if the state_dict comes from a model that was wrapped in a + # DataParallel or DistributedDataParallel during serialization, + # remove the "module" prefix before performing the matching. + _strip_prefix_if_present(checkpoint_state_dict, "module.") + + model_state_dict = self.model.state_dict() + incorrect_shapes = [] + for k in list(checkpoint_state_dict.keys()): + if k in model_state_dict: + shape_model = tuple(model_state_dict[k].shape) + shape_checkpoint = tuple(checkpoint_state_dict[k].shape) + if shape_model != shape_checkpoint: + incorrect_shapes.append((k, shape_checkpoint, shape_model)) + checkpoint_state_dict.pop(k) + + incompatible = self.model.load_state_dict(checkpoint_state_dict, strict=False) + return _IncompatibleKeys( + missing_keys=incompatible.missing_keys, + unexpected_keys=incompatible.unexpected_keys, + incorrect_shapes=incorrect_shapes, + ) + + def _log_incompatible_keys(self, incompatible: _IncompatibleKeys) -> None: + """ + Log information about the incompatible keys returned by ``_load_model``. + """ + for k, shape_checkpoint, shape_model in incompatible.incorrect_shapes: + self.logger.warning( + "Skip loading parameter '{}' to the model due to incompatible " + "shapes: {} in the checkpoint but {} in the " + "model! You might want to double check if this is expected.".format( + k, shape_checkpoint, shape_model + ) + ) + if incompatible.missing_keys: + missing_keys = _filter_reused_missing_keys(self.model, incompatible.missing_keys) + if missing_keys: + self.logger.info(get_missing_parameters_message(missing_keys)) + if incompatible.unexpected_keys: + self.logger.info(get_unexpected_parameters_message(incompatible.unexpected_keys)) + + def _convert_ndarray_to_tensor(self, state_dict: dict): + """ + In-place convert all numpy arrays in the state_dict to flow tensor. + Args: + state_dict (dict): a state-dict to be loaded to the model. + """ + # model could be an OrderedDict with _metadata attribute + # (as returned by oneflow's state_dict()). We should preserve these + # properties. + for k in list(state_dict.keys()): + v = state_dict[k] + if not isinstance(v, np.ndarray) and not isinstance(v, flow.Tensor): + raise ValueError("Unsupported type found in checkpoint! {}: {}".format(k, type(v))) + # If it's local tensor, convert it to global tensor. + if not v.is_global: + if k in self.model.state_dict(): + model_v = self.model.state_dict()[k] + state_dict[k] = v.to_global(sbp=model_v.sbp, placement=model_v.placement) + + +class PeriodicCheckpointer: + """ + Save checkpoints periodically. When `.step(iteration)` is called, it will + execute `checkpointer.save` on the given checkpointer, if iteration is a + multiple of period or if `max_iter` is reached. + """ + + def __init__( + self, + checkpointer: Checkpointer, + period: int, + max_iter: Optional[int] = None, + max_to_keep: Optional[int] = None, + file_prefix: str = "model", + ): + """ + Args: + checkpointer (Any): the checkpointer object used to save + checkpoints. + period (int): the period to save checkpoint. + max_epoch (int): maximum number of epochs. When it is reached, + a checkpoint named "model_final" will be saved. + """ + self.checkpointer = checkpointer + self.period = int(period) + self.max_iter = max_iter + if max_to_keep is not None: + assert max_to_keep > 0 + self.max_to_keep = max_to_keep + self.recent_checkpoints: List[str] = [] + self.file_prefix = file_prefix + self.path_manager: PathManagerBase = checkpointer.path_manager + + def step(self, iteration: int, **kwargs: Any): + """ + Perform the appropriate action at the given iteration. + + Args: + iteration (int): the current epoch, ranged in [0, max_iter-1]. + kwargs (Any): extra data to save, same as in + :meth:`Checkpointer.save`. + """ + iteration = int(iteration) + additional_state = {"iteration": iteration} + additional_state.update(kwargs) + + if (iteration + 1) % self.period == 0: + self.checkpointer.save( + "{}_{:07d}".format(self.file_prefix, iteration), **additional_state + ) + + if self.max_to_keep is not None: + self.recent_checkpoints.append(self.checkpointer.get_checkpoint_file()) + if len(self.recent_checkpoints) > self.max_to_keep: + file_to_delete = self.recent_checkpoints.pop(0) + if self.path_manager.exists(file_to_delete) and not file_to_delete.endswith( + "{}_{:07d}".format(self.file_prefix, iteration) + ): + self.path_manager.rm(file_to_delete) + + if self.max_iter is not None: + if iteration >= self.max_iter - 1: + self.checkpointer.save(f"{self.file_prefix}_final", **additional_state) + + def save(self, name: str, **kwargs: Any): + """ + Same argument as :meth:`Checkpointer.save`. + Use this method to manually save checkpoints outside the schedule. + + Args: + name (str): file name. + kwargs (Any): extra data to save, same as in + :meth:`Checkpointer.save`. + """ + self.checkpointer.save(name, **kwargs) + + +def _filter_reused_missing_keys(model: nn.Module, keys: List[str]) -> List[str]: + """ + Filter "missing keys" to not include keys that have been loaded with another name. + """ + keyset = set(keys) + param_to_names = defaultdict(set) # param -> names that points to it + for module_prefix, module in _named_modules_with_dup(model): + for name, param in list(module.named_parameters(recurse=False)) + list( + module.named_buffers(recurse=False) # pyre-ignore + ): + full_name = (module_prefix + "." if module_prefix else "") + name + param_to_names[param].add(full_name) + for names in param_to_names.values(): + # if one name appears missing but its alias exists, then this + # name is not considered missing + if any(n in keyset for n in names) and not all(n in keyset for n in names): + [keyset.remove(n) for n in names if n in keyset] + return list(keyset) + + +def get_missing_parameters_message(keys: List[str]) -> str: + """ + Get a logging-friendly message to report parameter names (keys) that are in + the model but not found in a checkpoint. + Args: + keys (list[str]): List of keys that were not found in the checkpoint. + Returns: + str: message. + """ + groups = _group_checkpoint_keys(keys) + msg = "Some model parameters or buffers are not found in the checkpoint:\n" + msg += "\n".join(" " + colored(k + _group_to_str(v), "blue") for k, v in groups.items()) + return msg + + +def get_unexpected_parameters_message(keys: List[str]) -> str: + """ + Get a logging-friendly message to report parameter names (keys) that are in + the checkpoint but not found in the model. + Args: + keys (list[str]): List of keys that were not found in the model. + Returns: + str: message. + """ + groups = _group_checkpoint_keys(keys) + msg = "The checkpoint state_dict contains keys that are not used by the model:\n" + msg += "\n".join(" " + colored(k + _group_to_str(v), "magenta") for k, v in groups.items()) + return msg + + +def _strip_prefix_if_present(state_dict: Dict[str, Any], prefix: str) -> None: + """ + Strip the prefix in metadata, if any. + Args: + state_dict (OrderedDict): a state-dict to be loaded to the model. + prefix (str): prefix. + """ + keys = sorted(state_dict.keys()) + if not all(len(key) == 0 or key.startswith(prefix) for key in keys): + return + + for key in keys: + newkey = key[len(prefix) :] + state_dict[newkey] = state_dict.pop(key) + + # also strip the prefix in metadata, if any.. + try: + metadata = state_dict._metadata # pyre-ignore + except AttributeError: + pass + else: + for key in list(metadata.keys()): + # for the metadata dict, the key can be: + # '': for the DDP module, which we want to remove. + # 'module': for the actual model. + # 'module.xx.xx': for the rest. + + if len(key) == 0: + continue + newkey = key[len(prefix) :] + metadata[newkey] = metadata.pop(key) + + +def _group_checkpoint_keys(keys: List[str]) -> Dict[str, List[str]]: + """ + Group keys based on common prefixes. A prefix is the string up to the final + "." in each key. + Args: + keys (list[str]): list of parameter names, i.e. keys in the model + checkpoint dict. + Returns: + dict[list]: keys with common prefixes are grouped into lists. + """ + groups = defaultdict(list) + for key in keys: + pos = key.rfind(".") + if pos >= 0: + head, tail = key[:pos], [key[pos + 1 :]] + else: + head, tail = key, [] + groups[head].extend(tail) + return groups + + +def _group_to_str(group: List[str]) -> str: + """ + Format a group of parameter name suffixes into a loggable string. + Args: + group (list[str]): list of parameter name suffixes. + Returns: + str: formated string. + """ + if len(group) == 0: + return "" + + if len(group) == 1: + return "." + group[0] + + return ".{" + ", ".join(group) + "}" + + +def _named_modules_with_dup(model: nn.Module, prefix: str = "") -> Iterable[Tuple[str, nn.Module]]: + """ + The same as `model.named_modules()`, except that it includes + duplicated modules that have more than one name. + """ + yield prefix, model + for name, module in model._modules.items(): # pyre-ignore + if module is None: + continue + submodule_prefix = prefix + ("." if prefix else "") + name + yield from _named_modules_with_dup(module, submodule_prefix) diff --git a/libai/utils/distributed.py b/libai/utils/distributed.py new file mode 100644 index 0000000000000000000000000000000000000000..53bc99cb2e44b9b4b2dd1da9e5f347ce432459c4 --- /dev/null +++ b/libai/utils/distributed.py @@ -0,0 +1,481 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import dill +import numpy as np +import oneflow as flow +from omegaconf import OmegaConf + +from libai.config import try_get_key + +logger = logging.getLogger(__name__) + +_DIST_UTIL = None + + +def _merge_devices(devices): + num_gpus_per_node = get_world_size() // get_num_nodes() + node_devices = [node_id * num_gpus_per_node + device_id for node_id, device_id in devices] + return node_devices + + +class _DistributeUtil(object): + def __init__(self, cfg): + + self._init_distributed_env(cfg) + self._init_parallel_size(cfg) + self._init_placement_group(cfg) + self._init_parallel_hierarchy() + + def _init_distributed_env(self, cfg): + """Initialize the distributed environment.""" + + num_nodes = get_num_nodes() + num_gpus_per_node = get_world_size() // num_nodes + + if try_get_key(cfg, "num_gpus_per_node", default=num_gpus_per_node) != num_gpus_per_node: + # This means key(num_gpus_per_node) saved in config is not equal + # to environment variable. + # Give user a warning about inconsistent reproduce environment. + logger.warning( + "'train.dist.num_gpus_per_node' are not equal to environment variable. " + f"{cfg.num_gpus_per_node} != {num_gpus_per_node}" + ) + + if try_get_key(cfg, "num_nodes", default=num_nodes) != num_nodes: + logger.warning( + "'train.dist.num_nodes' are not equal to" + f"environment variable. {cfg.num_nodes} != {num_nodes}" + ) + + # Set the actual value to config + cfg.num_nodes = num_nodes + cfg.num_gpus_per_node = num_gpus_per_node + + self._num_nodes = num_nodes + self._num_gpus_per_node = num_gpus_per_node + self._world_size = num_gpus_per_node * num_nodes + + # Add set device type + self._device_type = try_get_key(cfg, "device_type", default="cuda") + + def _init_parallel_size(self, cfg): + + # tensor parallel size + self._tensor_parallel_size = min(cfg.tensor_parallel_size, self.world_size) + assert self.world_size % self._tensor_parallel_size == 0, ( + f"world size ({self.world_size}) is not divisible by" + f" tensor parallel size ({self._tensor_parallel_size})" + ) + # Set the actual tensor parallel size to cfg + cfg.tensor_parallel_size = self._tensor_parallel_size + + # pipeline parallel size + self._pipeline_parallel_size = min( + cfg.pipeline_parallel_size, self.world_size // cfg.tensor_parallel_size + ) + # Set the actual pipeline parallel size to cfg + cfg.pipeline_parallel_size = self._pipeline_parallel_size + + if cfg.pipeline_parallel_size > 1: + assert ( + try_get_key(cfg, "pipeline_num_layers") is not None + ), "cfg.train.dist.pipeline_num_layers must be set when run pipeline parallel" + + assert cfg.pipeline_num_layers >= self._pipeline_parallel_size, ( + f"number of layers ({cfg.pipeline_num_layers}) is less than" + f" pipeline model parallel size ({self._pipeline_parallel_size})" + ) + if try_get_key(cfg, "custom_pipeline_stage_id") is not None: + assert OmegaConf.is_list( + cfg.custom_pipeline_stage_id + ), "type of cfg.train.dist.custom_pipeline_stage_id must be list" + cfg.custom_pipeline_stage_id = list(cfg.custom_pipeline_stage_id) + assert max(cfg.custom_pipeline_stage_id) < self._world_size, ( + f"the element {max(cfg.custom_pipeline_stage_id)} in" + " cfg.train.dist.custom_pipeline_stage_id is out of range" + f" for total rank {self._world_size}" + ) + assert len(cfg.custom_pipeline_stage_id) == cfg.pipeline_num_layers, ( + "the length of cfg.train.dist.custom_pipeline_stage_id" + f" {len(cfg.custom_pipeline_stage_id)} must be equal to" + " cfg.train.dist.pipeline_num_layers" + f" {cfg.train.dist.pipeline_num_layers}" + ) + else: + # no pipeline parallel, just set 10000 + if try_get_key(cfg, "pipeline_num_layers") is None: + cfg.pipeline_num_layers = 10000 + + self._model_parallel_size = self._pipeline_parallel_size * self._tensor_parallel_size + + assert self.world_size % self._model_parallel_size == 0, ( + f"world size ({self.world_size}) is not divisible by" + f" tensor model parallel size ({self._tensor_parallel_size}) times" + f" pipeline model parallel size ({self._pipeline_parallel_size})" + ) + + # data parallel size + self._data_parallel_size = self.world_size // self._model_parallel_size + # Set the actual data parallel size to cfg + cfg.data_parallel_size = self._data_parallel_size + + def _init_placement_group(self, cfg): + node_ids = [i // self.num_gpus_per_node for i in range(self.world_size)] + device_ids = list(range(self.num_gpus_per_node)) * self.num_nodes + + # [(0, 0), (0, 1), (0, 2), (0, 3), (1, 0), (1, 1), (1, 2), (1, 3)] + devices = [(n, d) for n, d in zip(node_ids, device_ids)] + num_devices_per_stage = self.world_size // self._pipeline_parallel_size + stages_devices = [ + _merge_devices(devices[i : (i + num_devices_per_stage)]) + for i in range(0, self.world_size, num_devices_per_stage) + ] + + # change pipeline_num_layers to make the middle stages contain more layers + if ( + self._pipeline_parallel_size >= 4 + and cfg.pipeline_num_layers >= 8 + and cfg.pipeline_num_layers % self._pipeline_parallel_size == 0 + ): + temp_num_layers_per_stage = cfg.pipeline_num_layers // self._pipeline_parallel_size + actual_pipeline_num_layers = cfg.pipeline_num_layers + min( + self._pipeline_parallel_size - 1, temp_num_layers_per_stage + ) + else: + actual_pipeline_num_layers = cfg.pipeline_num_layers + + num_layers_per_stage = actual_pipeline_num_layers // self._pipeline_parallel_size + stage_offset = actual_pipeline_num_layers % self._pipeline_parallel_size + + # stage_offset can make the later stages contain more layers when pipeline_num_layers + # cannot be divided by pipeline_parallel_size. + # This can make pipeline parallel more memory efficient. + self._layer_stage_ids = [] + for i in range(0, actual_pipeline_num_layers - stage_offset, num_layers_per_stage): + stage_id = i // num_layers_per_stage + if stage_id >= (self._pipeline_parallel_size - stage_offset): + self._layer_stage_ids.append(stage_id) + self._layer_stage_ids.extend([stage_id] * num_layers_per_stage) + self._layer_stage_ids = self._layer_stage_ids[: cfg.pipeline_num_layers] + # when pipeline_parallel_size > 1, we add pipeline_stage_id infomation into cfg + if cfg.pipeline_parallel_size > 1: + cfg.auto_pipeline_stage_id = self._layer_stage_ids + # set pipeline_stage_id by users' setting + if try_get_key(cfg, "custom_pipeline_stage_id") is not None: + self._layer_stage_ids = cfg.custom_pipeline_stage_id + cfg.actual_pipeline_stage_id = self._layer_stage_ids + + self._layer_ranks = [stages_devices[stage_id] for stage_id in self._layer_stage_ids] + + def _init_parallel_hierarchy(self): + if self.is_data_model_parallel(): + self._parallel_hierarchy = ( + self._data_parallel_size, + self._tensor_parallel_size, + ) + else: + self._parallel_hierarchy = None + + @property + def num_nodes(self): + return self._num_nodes + + @property + def num_gpus_per_node(self): + return self._num_gpus_per_node + + @property + def world_size(self): + return self._world_size + + @property + def parallel_hierarchy(self): + return self._parallel_hierarchy + + @property + def tensor_parallel_size(self): + return self._tensor_parallel_size + + @property + def pipeline_parallel_size(self): + return self._pipeline_parallel_size + + @property + def model_parallel_size(self): + return self._tensor_parallel_size + + @property + def data_parallel_size(self): + return self._data_parallel_size + + @property + def device_type(self): + return self._device_type + + def set_device_type(self, device_type): + assert device_type in ["cpu", "cuda"], f"not supported for {device_type}" + self._device_type = device_type + + def get_layer_ranks(self, layer_idx): + layer_ranks = self._layer_ranks[layer_idx] + if self._parallel_hierarchy is None: + return layer_ranks + else: + assert len(self._parallel_hierarchy) == 2 + return np.asarray(layer_ranks).reshape(self._parallel_hierarchy).tolist() + + def get_layer_stage_id(self, layer_idx): + return self._layer_stage_ids[layer_idx] + + def is_tensor_model_parallel(self): + return self._tensor_parallel_size > 1 + + def is_data_parallel(self): + return self._data_parallel_size > 1 + + def is_pipeline_model_parallel(self): + return self._pipeline_parallel_size > 1 + + def is_data_model_parallel(self): + return self.is_tensor_model_parallel() and self.is_data_parallel() + + +def setup_dist_util(cfg): + """Initialize the distributed environment with configuration. + + Example: + + .. code-block:: python + + from omegaconf import DictConfig + + # set the hybrid parallel distributed environment with 2D mesh GPUs + setup_dist_util( + DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + ) + ) + ) + + """ + global _DIST_UTIL + _DIST_UTIL = _DistributeUtil(cfg) + + +def get_dist_util(): + """Get distributed utils if it's been setup. Otherwise, initialize it with + single node/single gpu environment.""" + global _DIST_UTIL + if _DIST_UTIL is None: + logger.warning( + "Distributed env is not set up, configure it by default (single node, single gpu)." + ) + from omegaconf import DictConfig + + setup_dist_util( + DictConfig( + dict( + data_parallel_size=1, + tensor_parallel_size=1, + pipeline_parallel_size=1, + ) + ) + ) + return _DIST_UTIL + + +def get_layer_placement(layer_idx, device_type=None): + """ + Get ``flow.placement`` object with the initialized distributed environment + according to the ``layer_idx``. + + Args: + layer_idx (int): layer index indicating the rank groups. This is very useful for pipeline + parallelism training where different layers are on different ranks. + device_type (str, optional): device type. Defaults to "cuda". + """ + dist_util = get_dist_util() + device_type = dist_util.device_type if device_type is None else device_type + if not flow.cuda.is_available() and device_type == "cuda": + device_type = "cpu" + return flow.placement( + device_type, + dist_util.get_layer_ranks(layer_idx), + ) + + +def get_nd_sbp(sbp_list): + """Get nd sbp signature list, which is consistent with 1D/2D mesh GPUs. + + Args: + sbp_list (list): a sbp list with 2D mesh. + + Returns: + A modified sbp list according to the initialized distributed environment. + """ + assert isinstance(sbp_list, list) + assert len(sbp_list) == 2 + assert all(isinstance(sbp, flow.sbp.sbp) for sbp in sbp_list) + + dist_util = get_dist_util() + if dist_util.is_data_model_parallel(): + return sbp_list + elif dist_util.is_data_parallel(): + return sbp_list[:1] + elif dist_util.is_tensor_model_parallel(): + return sbp_list[1:] + else: + return [flow.sbp.broadcast] + + +def get_hidden_sbp(): + """Hidden states sbp.""" + return get_nd_sbp([flow.sbp.split(0), flow.sbp.broadcast]) + + +def get_data_parallel_rank(): + dist_util = get_dist_util() + return (flow.env.get_rank() // dist_util.model_parallel_size) % dist_util.data_parallel_size + + +def get_data_parallel_size(): + dist_util = get_dist_util() + return dist_util.data_parallel_size + + +def get_tensor_parallel_size(): + dist_util = get_dist_util() + return dist_util.tensor_parallel_size + + +def get_pipeline_parallel_size(): + dist_util = get_dist_util() + return dist_util.pipeline_parallel_size + + +def same_sbp(lhs_sbp, rhs_sbp): + """Determine if two sbp signatures are the same.""" + assert len(lhs_sbp) == len(rhs_sbp) + + for i in range(len(lhs_sbp)): + if lhs_sbp[i] != rhs_sbp[i]: + return False + return True + + +def get_rank() -> int: + return flow.env.get_rank() + + +def get_local_rank() -> int: + return flow.env.get_local_rank() + + +def is_main_process() -> bool: + return get_rank() == 0 + + +def is_last_process() -> bool: + return get_rank() == get_world_size() - 1 + + +def get_world_size(): + return flow.env.get_world_size() + + +def get_num_nodes(): + return flow.env.get_node_size() + + +def set_device_type(device_type): + dist_util = get_dist_util() + dist_util.set_device_type(device_type) + + +def broadcast_py_object(obj, src: int = 0): + rank = flow.env.get_rank() + if src == rank: + obj_bytes = dill.dumps(obj) + return dill.loads(flow._oneflow_internal.cpu_broadcast(obj_bytes, src)) + else: + return dill.loads(flow._oneflow_internal.cpu_broadcast(None, src)) + + +def convert_to_distributed_default_setting(t): + """ + Helper function to convert all eager local tensor in :attr:`nn.Module` in the model to + global tensor with data parallelism as default. + """ + if not t.is_global: + return t.to_global( + sbp=get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=get_layer_placement(0), + ) + else: + return t + + +def ttol(tensor, pure_local=False, ranks=None): + """Global tensor to local tensor.""" + if tensor.is_global: + placement = tensor.placement if not ranks else flow.placement("cuda", ranks) + if pure_local: + tensor = tensor.to_global(placement=placement).to_local() + else: + tensor = tensor.to_global( + sbp=get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), placement=placement + ).to_local() + + return tensor + + +def tton(tensor, local_only=False, ranks=None): + """Global tensor to numpy ndarray.""" + if tensor.is_global: + tensor = ttol(tensor, local_only, ranks) + + return tensor.numpy() + + +def tensor_to_rank0(tensor, device="cuda", to_local=False): + """Global tensor to rank0.""" + assert device in ["cpu", "cuda"], f"not supported for device:{device}" + if tensor.is_global: + # Consider if it's 2d mesh, ranks should be [[0]] instead of [0] + placement = flow.placement(device, ranks=[0] if tensor.placement.ranks.ndim == 1 else [[0]]) + tensor = tensor.to_global( + sbp=get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), placement=placement + ) + if to_local: + tensor = ttol(tensor) + return tensor + + +def synchronize(): + """ + Helper function to synchronize (barrier) among all processes when + using distributed training. + """ + world_size = get_world_size() + if world_size == 1: + return + + flow.comm.barrier() diff --git a/libai/utils/download.py b/libai/utils/download.py new file mode 100644 index 0000000000000000000000000000000000000000..fb07ea284ab50fe34a5deeea808819c359c23b9b --- /dev/null +++ b/libai/utils/download.py @@ -0,0 +1,94 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import os +import shutil +from typing import Callable, List, Optional +from urllib import request + +# -------------------------------------------------------- +# References: +# https://github.com/facebookresearch/iopath/blob/main/iopath/common/download.py +# -------------------------------------------------------- + + +def download(url: str, dir: str, *, filename: Optional[str] = None, progress: bool = True) -> str: + """ + Download a file from a given URL to a directory. If file exists, will not + overwrite the existing file. + Args: + url (str): + dir (str): the directory to download the file + filename (str or None): the basename to save the file. + Will use the name in the URL if not given. + progress (bool): whether to use tqdm to draw a progress bar. + Returns: + str: the path to the downloaded file or the existing one. + """ + os.makedirs(dir, exist_ok=True) + if filename is None: + filename = url.split("/")[-1] + assert len(filename), "Cannot obtain filename from url {}".format(url) + fpath = os.path.join(dir, filename) + logger = logging.getLogger(__name__) + + if os.path.isfile(fpath): + logger.info("File {} exists! Skipping download.".format(filename)) + return fpath + + tmp = fpath + ".tmp" # download to a tmp file first, to be more atomic. + try: + logger.info("Downloading from {} ...".format(url)) + if progress: + import tqdm + + def hook(t: tqdm.tqdm) -> Callable[[int, int, Optional[int]], None]: + last_b: List[int] = [0] + + def inner(b: int, bsize: int, tsize: Optional[int] = None) -> None: + if tsize is not None: + t.total = tsize + t.update((b - last_b[0]) * bsize) # type: ignore + last_b[0] = b + + return inner + + with tqdm.tqdm( # type: ignore + unit="B", unit_scale=True, miniters=1, desc=filename, leave=True + ) as t: + tmp, _ = request.urlretrieve(url, filename=tmp, reporthook=hook(t)) + + else: + tmp, _ = request.urlretrieve(url, filename=tmp) + statinfo = os.stat(tmp) + size = statinfo.st_size + if size == 0: + raise IOError("Downloaded an empty file from {}!".format(url)) + # download to tmp first and move to fpath, to make this function more + # atomic. + shutil.move(tmp, fpath) + except IOError: + logger.error("Failed to download {}".format(url)) + raise + finally: + try: + os.unlink(tmp) + except IOError: + pass + + logger.info("Successfully downloaded " + fpath + ". " + str(size) + " bytes.") + return fpath diff --git a/libai/utils/events.py b/libai/utils/events.py new file mode 100644 index 0000000000000000000000000000000000000000..b8807e2450d48789dab7e91a9ea2e3cd5be1339d --- /dev/null +++ b/libai/utils/events.py @@ -0,0 +1,450 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import datetime +import json +import logging +import os +import time +from collections import defaultdict +from contextlib import contextmanager + +from libai.utils.file_io import PathManager +from libai.utils.history_buffer import HistoryBuffer + +# -------------------------------------------------------- +# References: +# https://github.com/facebookresearch/detectron2/blob/main/detectron2/utils/events.py +# -------------------------------------------------------- + + +__all__ = [ + "get_event_storage", + "JSONWriter", + "CommonMetricPrinter", + "EventStorage", +] + +_CURRENT_STORAGE_STACK = [] + + +def get_event_storage(): + """ + Returns: + The :class:`EventStorage` object that's currently being used. + Throw an error if no :class:`EventStorage` is currently enabled. + """ + assert len( + _CURRENT_STORAGE_STACK + ), "get_event_storage() has to be called inside a 'with EventStorage(...)' context!" + return _CURRENT_STORAGE_STACK[-1] + + +class EventWriter: + """ + Base class for writers that obtain events from :class:`EventStorage` and process them. + """ + + def write(self): + raise NotImplementedError + + def close(self): + pass + + +class JSONWriter(EventWriter): + """ + Write scalars to a json file. + It saves scalars as one json per line (instead of a big json) for easy parsing. + Example of parsing such a json file: + + :: + + $ cat metrics.json | jq -s '.[0:2]' + [ + { + "data_time": 0.008433341979980469, + "iteration": 19, + "total_loss": 1.9228371381759644, + "lr": 0.007173333333333333, + "time": 0.25401854515075684 + }, + { + "data_time": 0.007216215133666992, + "iteration": 39, + "total_loss": 1.282649278640747, + "lr": 0.007706666666666667, + "time": 0.2490077018737793 + } + ] + $ cat metrics.json | jq '.loss_mask' + 0.7126231789588928 + 0.689423680305481 + 0.6776131987571716 + ... + """ + + def __init__(self, json_file, window_size=20): + """ + Args: + json_file (str): path to the json file. New data will be appended if the file exists. + window_size (int): the window size of median smoothing for the scalars whose + `smoothing_hint` are True. + """ + self._file_handle = PathManager.open(json_file, "a") + self._window_size = window_size + self._last_write = -1 + + def write(self): + storage = get_event_storage() + to_save = defaultdict(dict) + + for k, (v, iter) in storage.latest_with_smoothing_hint(self._window_size).items(): + # keep scalars that have not been written + if iter <= self._last_write: + continue + to_save[iter][k] = v + if len(to_save): + all_iters = sorted(to_save.keys()) + self._last_write = max(all_iters) + + for itr, scalars_per_iter in to_save.items(): + scalars_per_iter["iteration"] = itr + self._file_handle.write(json.dumps(scalars_per_iter, sort_keys=True) + "\n") + self._file_handle.flush() + try: + os.fsync(self._file_handle.fileno()) + except AttributeError: + pass + + def close(self): + self._file_handle.close() + + +class TensorboardXWriter(EventWriter): + """ + Write all scalars to a tensorboard file + """ + + def __init__(self, log_dir: str, window_size: int = 20, **kwargs): + """ + Args: + log_dir (str): the directory to save the output events + window_size (int): the scalars will be median-smoothed by this window size + + kwargs: other arguments passed to `tensorboardX.SummaryWriter(...)` + """ + self._window_size = window_size + from tensorboardX import SummaryWriter + + self._writer = SummaryWriter(log_dir=log_dir, **kwargs) + self._last_write = -1 + + def write(self): + storage = get_event_storage() + new_last_write = self._last_write + for k, (v, iter) in storage.latest_with_smoothing_hint(self._window_size).items(): + if iter > self._last_write: + self._writer.add_scalar(k, v, iter) + new_last_write = max(new_last_write, iter) + self._last_write = new_last_write + + # TODO: add write image + + if len(storage._histograms) >= 1: + for params in storage._histograms: + self._writer.add_histogram_raw(**params) + storage.clear_histograms() + + def close(self): + if hasattr(self, "_writer"): # doesn't exist when the code fails at import + self._writer.close() + + +class CommonMetricPrinter(EventWriter): + """ + Print **common** metrics to the terminal, including + iteration time, ETA, memory, all losses, and the learning rate. + It also applies smoothing using a window of 20 elements. + It's meant to print common metrics in common ways. + To print something in more customized ways, please implement a similar printer by yourself. + """ + + def __init__(self, batch_size, max_iter): + """ + Args: + max_iter (int): the maximum number of iterations to train. + Used to compute ETA. + """ + self.logger = logging.getLogger(__name__) + self._batch_size = batch_size + self._max_iter = max_iter + self._last_write = None + + def write(self): + storage = get_event_storage() + iteration = storage.iter + consumed_samples = storage.samples + if iteration == self._max_iter: + # This hook only reports training progress (loss, ETA, etc) but not other data, + # therefore do not write anything after training succeeds, even if this method + # is called. + return + + try: + data_time = storage.history("data_time").avg(20) + except KeyError: + # they may not exist in the first few iterations (due to warmup) + # or when SimpleTrainer is not used + data_time = None + + eta_string = None + try: + iter_time = storage.history("time").global_avg() + eta_seconds = storage.history("time").median(1000) * (self._max_iter - iteration - 1) + storage.put_scalar("eta_seconds", eta_seconds, smoothing_hint=False) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + except KeyError: + iter_time = None + # estimate eta on our own - more noisy + if self._last_write is not None: + estimate_iter_time = (time.perf_counter() - self._last_write[1]) / ( + iteration - self._last_write[0] + ) + eta_seconds = estimate_iter_time * (self._max_iter - iteration - 1) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + self._last_write = (iteration, time.perf_counter()) + + try: + lr = "{:.2e}".format(storage.history("lr").latest()) + except KeyError: + lr = "N/A" + + max_mem_mb = None + + # NOTE: max_mem is parsed by grep in "dev/parse_results.sh" + self.logger.info( + " {eta}{iter} {sample} {losses} {time}{data_time} {tpt} lr: {lr} {memory}".format( + eta=f"eta: {eta_string} " if eta_string else "", + iter=f"iteration: {iteration}/{self._max_iter}", + sample=f"consumed_samples: {consumed_samples}", + losses=" ".join( + [ + "{}: {:.4g}".format(k, v.median(200)) + for k, v in storage.histories().items() + if "loss" in k + ] + ), + time="time: {:.4f} s/iter ".format(iter_time) if iter_time is not None else "", + data_time="data_time: {:.4f} s/iter".format(data_time) + if data_time is not None + else "", + tpt="total_throughput: {:.2f} samples/s".format(self._batch_size / iter_time) + if iter_time is not None + else "", + lr=lr, + memory="max_mem: {:.0f}M".format(max_mem_mb) if max_mem_mb is not None else "", + ) + ) + + +class EventStorage: + """ + The user-facing class that provides metric storage functionalities. + In the future we may add support for storing / logging other types of data if needed. + """ + + def __init__(self, start_iter=0): + """ + + Args: + start_iter (int): the iteration number to start with + """ + self._history = defaultdict(HistoryBuffer) + self._smoothing_hints = {} + self._latest_scalars = {} + self._iter = start_iter + self._batch_size = 0 + self._current_prefix = "" + self._vis_data = [] + self._histograms = [] + + def put_image(self, img_name, img_tensor): + """ + Add an `img_tensor` associated with `img_name` to be shown on + tensorboard. + + Args: + img_name (str): The name of the image to put into tensorboard. + img_tensor (flow.Tensor or numpy.array): An `uint8` or `float` + Tensor of shape `[channel, height, width]` where `channel` is + 3. The image format should be RGB. The elements in img_tensor + can either have values in [0, 1] (float32) or [0, 255] (uint8). + The `img_tensor` will be visualized in tensorboard. + """ + self._vis_data.append((img_name, img_tensor, self._iter)) + + def put_scalar(self, name, value, smoothing_hint=True): + """ + Add a scalar `value` to the `HistoryBuffer` associated with `name`. + + Args: + smoothing_hint (bool): a 'hint' on whether this scalar is noisy and should be + smoothed when logged. The hint will be accessible through + :meth:`EventStorage.smoothing_hints`. A writer may ignore the hint + and apply custom smoothing rule. + It defaults to True because most scalars we save need to be smoothed to + provide any useful signal. + """ + name = self._current_prefix + name + history = self._history[name] + value = float(value) + history.update(value, self._iter) + self._latest_scalars[name] = (value, self._iter) + + existing_hint = self._smoothing_hints.get(name) + if existing_hint is not None: + assert ( + existing_hint == smoothing_hint + ), "Scalar {} was put with a different smoothing_hint!".format(name) + else: + self._smoothing_hints[name] = smoothing_hint + + def put_scalars(self, *, smoothing_hint=True, **kwargs): + """ + Put multiple scalars from keyword arguments. + + Example: + + .. code-block:: python + + storage.put_scalars(loss=my_loss, accuracy=my_accuracy, smoothing_hint=True) + """ + for k, v in kwargs.items(): + self.put_scalar(k, v, smoothing_hint=smoothing_hint) + + def history(self, name): + """ + Returns: + HistoryBuffer: the scalar history for name + """ + ret = self._history.get(name, None) + if ret is None: + raise KeyError("No history metric available for {}!".format(name)) + return ret + + def histories(self): + """ + Returns: + dict[name -> HistoryBuffer]: the HistoryBuffer for all scalars + """ + return self._history + + def latest(self): + """ + Returns: + dict[str -> (float, int)]: mapping from the name of each scalar to the most + recent value and the iteration number its added. + """ + return self._latest_scalars + + def latest_with_smoothing_hint(self, window_size=20): + """ + Similar to :meth:`latest`, but the returned values + are either the un-smoothed original latest value, + or a median of the given window_size, + depending on whether the smoothing_hint is True. + This provides a default behavior that other writers can use. + """ + result = {} + for k, (v, itr) in self._latest_scalars.items(): + result[k] = ( + self._history[k].median(window_size) if self._smoothing_hints[k] else v, + itr, + ) + return result + + def smoothing_hints(self): + """ + Returns: + dict[name -> bool]: the user-provided hint on whether the scalar + is noisy and needs smoothing. + """ + return self._smoothing_hints + + def step(self): + """ + User should either: (1) Call this function to increment storage.iter when needed. + Or (2) Set `storage.iter` to the correct iteration number before each iteration. + + The storage will then be able to associate the new data with an iteration number. + """ + self._iter += 1 + + @property + def iter(self): + """ + Returns the current iteration number. When used together with a trainer, + this is ensured to be the same as trainer.iter. + """ + return self._iter + + @iter.setter + def iter(self, val): + self._iter = int(val) + + @property + def samples(self): + return self._samples + + @samples.setter + def samples(self, val): + self._samples = int(val) + + def __enter__(self): + _CURRENT_STORAGE_STACK.append(self) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + assert _CURRENT_STORAGE_STACK[-1] == self + _CURRENT_STORAGE_STACK.pop() + + @contextmanager + def name_scope(self, name): + """ + Yields: + A context within which all the events added to this storage + will be prefixed by the name scope. + """ + old_prefix = self._current_prefix + self._current_prefix = name.rstrip("/") + "/" + yield + self._current_prefix = old_prefix + + def clear_images(self): + """ + Delete all the stored images for visualization. This should be called + after images are written to tensorboard. + """ + self._vis_data = [] + + def clear_histograms(self): + """ + Delete all the stored histograms for visualization. + This should be called after histograms are written to tensorboard. + """ + self._histograms = [] diff --git a/libai/utils/file_io.py b/libai/utils/file_io.py new file mode 100644 index 0000000000000000000000000000000000000000..d61378bbee704f835d112d3c47f426f64066d606 --- /dev/null +++ b/libai/utils/file_io.py @@ -0,0 +1,1325 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import base64 +import concurrent.futures +import errno +import logging +import os +import shutil +import tempfile +import traceback +from collections import OrderedDict +from typing import IO, Any, Callable, Dict, Iterable, List, MutableMapping, Optional, Set, Union +from urllib.parse import urlparse + +import portalocker + +from libai.utils.download import download +from libai.utils.non_blocking_io import NonBlockingIOManager + +# -------------------------------------------------------- +# References: +# https://github.com/facebookresearch/iopath/blob/main/iopath/common/file_io.py +# https://github.com/facebookresearch/detectron2/blob/main/detectron2/utils/file_io.py +# -------------------------------------------------------- + + +__all__ = ["LazyPath", "PathManager", "get_cache_dir", "file_lock"] + + +def get_cache_dir(cache_dir: Optional[str] = None) -> str: + """ + Returns a default directory to cache static files + (usually downloaded from Internet), if None is provided. + + Args: + cache_dir (None or str): if not None, will be returned as is. + If None, returns the default cache directory as: + + 1) $LIBAI_CACHE, if set + 2) otherwise ~/.oneflow/iopath_cache + """ + if cache_dir is None: + cache_dir = os.path.expanduser(os.getenv("LIBAI_CACHE", "~/.oneflow/iopath_cache")) + try: + g_pathmgr.mkdirs(cache_dir) + assert os.access(cache_dir, os.W_OK) + except (OSError, AssertionError): + tmp_dir = os.path.join(tempfile.gettempdir(), "iopath_cache") + logger = logging.getLogger(__name__) + logger.warning(f"{cache_dir} is not accessible! Using {tmp_dir} instead!") + cache_dir = tmp_dir + return cache_dir + + +def file_lock(path: str): # type: ignore + """ + A file lock. Once entered, it is guaranteed that no one else holds the + same lock. Others trying to enter the lock will block for 30 minutes and + raise an exception. + This is useful to make sure workers don't cache files to the same location. + Args: + path (str): a path to be locked. This function will create a lock named + `path + ".lock"` + Examples: + filename = "/path/to/file" + with file_lock(filename): + if not os.path.isfile(filename): + do_create_file() + """ + dirname = os.path.dirname(path) + try: + os.makedirs(dirname, exist_ok=True) + except OSError: + # makedir is not atomic. Exceptions can happen when multiple workers try + # to create the same dir, despite exist_ok=True. + # When this happens, we assume the dir is created and proceed to creating + # the lock. If failed to create the directory, the next line will raise + # exceptions. + pass + return portalocker.Lock(path + ".lock", timeout=3600) # type: ignore + + +class LazyPath(os.PathLike): + """ + A path that's lazily evaluated when it's used. + + Users should be careful to not use it like a str, because + it behaves differently from a str. + Path manipulation functions in Python such as `os.path.*` all accept + PathLike objects already. + + It can be materialized to a str using `os.fspath`. + """ + + def __init__(self, func: Callable[[], str]) -> None: + """ + Args: + func: a function that takes no arguments and returns the + actual path as a str. It will be called at most once. + """ + self._func = func + self._value: Optional[str] = None + + def _get_value(self) -> str: + if self._value is None: + self._value = self._func() + return self._value # pyre-ignore + + def __fspath__(self) -> str: + return self._get_value() + + # before more like a str after evaluated + def __getattr__(self, name: str): # type: ignore + if self._value is None: + raise AttributeError(f"Uninitialized LazyPath has no attribute: {name}.") + return getattr(self._value, name) + + def __getitem__(self, key): # type: ignore + if self._value is None: + raise TypeError("Uninitialized LazyPath is not subscriptable.") + return self._value[key] # type: ignore + + def __str__(self) -> str: + if self._value is not None: + return self._value # type: ignore + else: + return super().__str__() + + +class PathHandler: + """ + PathHandler is a base class that defines common I/O functionality for a URI + protocol. It routes I/O for a generic URI which may look like "protocol://*" + or a canonical filepath "/foo/bar/baz". + """ + + _strict_kwargs_check = True + + def __init__( + self, + async_executor: Optional[concurrent.futures.Executor] = None, + ) -> None: + """ + When registering a `PathHandler`, the user can optionally pass in a + `Executor` to run the asynchronous file operations. + NOTE: For regular non-async operations of `PathManager`, there is + no need to pass `async_executor`. + + Args: + async_executor (optional `Executor`): Used for async file operations. + Usage: + ``` + path_handler = NativePathHandler(async_executor=exe) + path_manager.register_handler(path_handler) + ``` + """ + self._non_blocking_io_manager = None + self._non_blocking_io_executor = async_executor + + def _check_kwargs(self, kwargs: Dict[str, Any]) -> None: + """ + Checks if the given arguments are empty. Throws a ValueError if strict + kwargs checking is enabled and args are non-empty. If strict kwargs + checking is disabled, only a warning is logged. + + Args: + kwargs (Dict[str, Any]) + """ + if self._strict_kwargs_check: + if len(kwargs) > 0: + raise ValueError("Unused arguments: {}".format(kwargs)) + else: + logger = logging.getLogger(__name__) + for k, v in kwargs.items(): + logger.warning("[PathManager] {}={} argument ignored".format(k, v)) + + def _get_supported_prefixes(self) -> List[str]: + """ + Returns: + List[str]: the list of URI prefixes this PathHandler can support + """ + raise NotImplementedError() + + def _get_local_path(self, path: str, force: bool = False, **kwargs: Any) -> str: + """ + Get a filepath which is compatible with native Python I/O such as `open` + and `os.path`. + If URI points to a remote resource, this function may download and cache + the resource to local disk. In this case, the cache stays on filesystem + (under `file_io.get_cache_dir()`) and will be used by a different run. + Therefore this function is meant to be used with read-only resources. + Args: + path (str): A URI supported by this PathHandler + force(bool): Forces a download from backend if set to True. + Returns: + local_path (str): a file path which exists on the local file system + """ + raise NotImplementedError() + + def _copy_from_local( + self, local_path: str, dst_path: str, overwrite: bool = False, **kwargs: Any + ) -> None: + """ + Copies a local file to the specified URI. + If the URI is another local path, this should be functionally identical + to copy. + Args: + local_path (str): a file path which exists on the local file system + dst_path (str): A URI supported by this PathHandler + overwrite (bool): Bool flag for forcing overwrite of existing URI + Returns: + status (bool): True on success + """ + raise NotImplementedError() + + def _opent( + self, path: str, mode: str = "r", buffering: int = 32, **kwargs: Any + ) -> Iterable[Any]: + raise NotImplementedError() + + def _open( + self, path: str, mode: str = "r", buffering: int = -1, **kwargs: Any + ) -> Union[IO[str], IO[bytes]]: + """ + Open a stream to a URI, similar to the built-in `open`. + + Args: + path (str): A URI supported by this PathHandler + mode (str): Specifies the mode in which the file is opened. It defaults + to 'r'. + buffering (int): An optional integer used to set the buffering policy. + Pass 0 to switch buffering off and an integer >= 1 to indicate the + size in bytes of a fixed-size chunk buffer. When no buffering + argument is given, the default buffering policy depends on the + underlying I/O implementation. + + Returns: + file: a file-like object. + """ + raise NotImplementedError() + + def _opena( + self, + path: str, + mode: str = "r", + callback_after_file_close: Optional[Callable[[None], None]] = None, + buffering: int = -1, + **kwargs: Any, + ) -> Union[IO[str], IO[bytes]]: + """ + Open a stream to a URI with asynchronous permissions. + + NOTE: Writes to the same path are serialized so they are written in + the same order as they were called but writes to distinct paths can + happen concurrently. + + Usage (default / without callback function): + for n in range(50): + results = run_a_large_task(n) + with path_manager.opena(uri, "w") as f: + f.write(results) # Runs in separate thread + # Main process returns immediately and continues to next iteration + path_manager.async_close() + + Usage (advanced / with callback function): + # To write local and then copy to Manifold: + def cb(): + path_manager.copy_from_local( + "checkpoint.pt", "manifold://path/to/bucket" + ) + f = pm.opena("checkpoint.pt", "wb", callback_after_file_close=cb) + flow.save({...}, f) + f.close() + + Args: + ...same args as `_open`... + callback_after_file_close (Callable): An optional argument that can + be passed to perform operations that depend on the asynchronous + writes being completed. The file is first written to the local + disk and then the callback is executed. + buffering (int): An optional argument to set the buffer size for + buffered asynchronous writing. + + Returns: + file: a file-like object with asynchronous methods. + """ + # Restrict mode until `NonBlockingIO` has async read feature. + valid_modes = {"w", "a", "b"} + if not all(m in valid_modes for m in mode): + raise ValueError("`opena` mode must be write or append") + + # TODO: Each `PathHandler` should set its own `self._buffered` + # parameter and pass that in here. Until then, we assume no + # buffering for any storage backend. + if not self._non_blocking_io_manager: + self._non_blocking_io_manager = NonBlockingIOManager( + buffered=False, + executor=self._non_blocking_io_executor, + ) + + try: + return self._non_blocking_io_manager.get_non_blocking_io( + path=self._get_path_with_cwd(path), + io_obj=self._open(path, mode, **kwargs), + callback_after_file_close=callback_after_file_close, + buffering=buffering, + ) + except ValueError: + # When `_strict_kwargs_check = True`, then `open_callable` + # will throw a `ValueError`. This generic `_opena` function + # does not check the kwargs since it may include any `_open` + # args like `encoding`, `ttl`, `has_user_data`, etc. + logger = logging.getLogger(__name__) + logger.exception( + "An exception occurred in `NonBlockingIOManager`. This " + "is most likely due to invalid `opena` args. Make sure " + "they match the `open` args for the `PathHandler`." + ) + self._async_close() + + def _async_join(self, path: Optional[str] = None, **kwargs: Any) -> bool: + """ + Ensures that desired async write threads are properly joined. + + Args: + path (str): Pass in a file path to wait until all asynchronous + activity for that path is complete. If no path is passed in, + then this will wait until all asynchronous jobs are complete. + + Returns: + status (bool): True on success + """ + if not self._non_blocking_io_manager: + logger = logging.getLogger(__name__) + logger.warning( + "This is an async feature. No threads to join because " "`opena` was not used." + ) + self._check_kwargs(kwargs) + return self._non_blocking_io_manager._join(self._get_path_with_cwd(path) if path else None) + + def _async_close(self, **kwargs: Any) -> bool: + """ + Closes the thread pool used for the asynchronous operations. + + Returns: + status (bool): True on success + """ + if not self._non_blocking_io_manager: + logger = logging.getLogger(__name__) + logger.warning( + "This is an async feature. No threadpool to close because " "`opena` was not used." + ) + self._check_kwargs(kwargs) + return self._non_blocking_io_manager._close_thread_pool() + + def _copy(self, src_path: str, dst_path: str, overwrite: bool = False, **kwargs: Any) -> bool: + """ + Copies a source path to a destination path. + + Args: + src_path (str): A URI supported by this PathHandler + dst_path (str): A URI supported by this PathHandler + overwrite (bool): Bool flag for forcing overwrite of existing file + + Returns: + status (bool): True on success + """ + raise NotImplementedError() + + def _mv(self, src_path: str, dst_path: str, **kwargs: Any) -> bool: + """ + Moves (renames) a source path to a destination path. + + Args: + src_path (str): A URI supported by this PathHandler + dst_path (str): A URI supported by this PathHandler + + Returns: + status (bool): True on success + """ + raise NotImplementedError() + + def _exists(self, path: str, **kwargs: Any) -> bool: + """ + Checks if there is a resource at the given URI. + + Args: + path (str): A URI supported by this PathHandler + + Returns: + bool: true if the path exists + """ + raise NotImplementedError() + + def _isfile(self, path: str, **kwargs: Any) -> bool: + """ + Checks if the resource at the given URI is a file. + + Args: + path (str): A URI supported by this PathHandler + + Returns: + bool: true if the path is a file + """ + raise NotImplementedError() + + def _isdir(self, path: str, **kwargs: Any) -> bool: + """ + Checks if the resource at the given URI is a directory. + + Args: + path (str): A URI supported by this PathHandler + + Returns: + bool: true if the path is a directory + """ + raise NotImplementedError() + + def _ls(self, path: str, **kwargs: Any) -> List[str]: + """ + List the contents of the directory at the provided URI. + + Args: + path (str): A URI supported by this PathHandler + + Returns: + List[str]: list of contents in given path + """ + raise NotImplementedError() + + def _mkdirs(self, path: str, **kwargs: Any) -> None: + """ + Recursive directory creation function. Like mkdir(), but makes all + intermediate-level directories needed to contain the leaf directory. + Similar to the native `os.makedirs`. + + Args: + path (str): A URI supported by this PathHandler + """ + raise NotImplementedError() + + def _rm(self, path: str, **kwargs: Any) -> None: + """ + Remove the file (not directory) at the provided URI. + + Args: + path (str): A URI supported by this PathHandler + """ + raise NotImplementedError() + + def _symlink(self, src_path: str, dst_path: str, **kwargs: Any) -> bool: + """ + Symlink the src_path to the dst_path + + Args: + src_path (str): A URI supported by this PathHandler to symlink from + dst_path (str): A URI supported by this PathHandler to symlink to + """ + raise NotImplementedError() + + def _set_cwd(self, path: Union[str, None], **kwargs: Any) -> bool: + """ + Set the current working directory. PathHandler classes prepend the cwd + to all URI paths that are handled. + + Args: + path (str) or None: A URI supported by this PathHandler. Must be a valid + absolute path or None to set the cwd to None. + + Returns: + bool: true if cwd was set without errors + """ + raise NotImplementedError() + + def _get_path_with_cwd(self, path: str) -> str: + """ + Default implementation. PathHandler classes that provide a `_set_cwd` + feature should also override this `_get_path_with_cwd` method. + + Args: + path (str): A URI supported by this PathHandler. + + Returns: + path (str): Full path with the cwd attached. + """ + return path + + +class NativePathHandler(PathHandler): + """ + Handles paths that can be accessed using Python native system calls. This + handler uses `open()` and `os.*` calls on the given path. + """ + + _cwd = None + + def _get_local_path(self, path: str, force: bool = False, **kwargs: Any) -> str: + self._check_kwargs(kwargs) + return os.fspath(path) + + def _copy_from_local( + self, local_path: str, dst_path: str, overwrite: bool = False, **kwargs: Any + ) -> None: + self._check_kwargs(kwargs) + local_path = self._get_path_with_cwd(local_path) + dst_path = self._get_path_with_cwd(dst_path) + assert self._copy(src_path=local_path, dst_path=dst_path, overwrite=overwrite, **kwargs) + + def _open( + self, + path: str, + mode: str = "r", + buffering: int = -1, + encoding: Optional[str] = None, + errors: Optional[str] = None, + newline: Optional[str] = None, + closefd: bool = True, + opener: Optional[Callable] = None, + **kwargs: Any, + ) -> Union[IO[str], IO[bytes]]: + """ + Open a path. + + Args: + path (str): A URI supported by this PathHandler + mode (str): Specifies the mode in which the file is opened. It defaults + to 'r'. + buffering (int): An optional integer used to set the buffering policy. + Pass 0 to switch buffering off and an integer >= 1 to indicate the + size in bytes of a fixed-size chunk buffer. When no buffering + argument is given, the default buffering policy works as follows: + * Binary files are buffered in fixed-size chunks; the size of + the buffer is chosen using a heuristic trying to determine the + underlying device’s “block size” and falling back on + io.DEFAULT_BUFFER_SIZE. On many systems, the buffer will + typically be 4096 or 8192 bytes long. + encoding (Optional[str]): the name of the encoding used to decode or + encode the file. This should only be used in text mode. + errors (Optional[str]): an optional string that specifies how encoding + and decoding errors are to be handled. This cannot be used in binary + mode. + newline (Optional[str]): controls how universal newlines mode works + (it only applies to text mode). It can be None, '', '\n', '\r', + and '\r\n'. + closefd (bool): If closefd is False and a file descriptor rather than + a filename was given, the underlying file descriptor will be kept + open when the file is closed. If a filename is given closefd must + be True (the default) otherwise an error will be raised. + opener (Optional[Callable]): A custom opener can be used by passing + a callable as opener. The underlying file descriptor for the file + object is then obtained by calling opener with (file, flags). + opener must return an open file descriptor (passing os.open as opener + results in functionality similar to passing None). + See https://docs.python.org/3/library/functions.html#open for details. + + Returns: + file: a file-like object. + """ + self._check_kwargs(kwargs) + return open( # type: ignore + self._get_path_with_cwd(path), + mode, + buffering=buffering, + encoding=encoding, + errors=errors, + newline=newline, + closefd=closefd, + opener=opener, + ) + + def _copy(self, src_path: str, dst_path: str, overwrite: bool = False, **kwargs: Any) -> bool: + """ + Copies a source path to a destination path. + + Args: + src_path (str): A URI supported by this PathHandler + dst_path (str): A URI supported by this PathHandler + overwrite (bool): Bool flag for forcing overwrite of existing file + + Returns: + status (bool): True on success + """ + self._check_kwargs(kwargs) + src_path = self._get_path_with_cwd(src_path) + dst_path = self._get_path_with_cwd(dst_path) + if os.path.exists(dst_path) and not overwrite: + logger = logging.getLogger(__name__) + logger.error("Destination file {} already exists.".format(dst_path)) + return False + + try: + shutil.copyfile(src_path, dst_path) + return True + except Exception as e: + logger = logging.getLogger(__name__) + logger.error("Error in file copy - {}".format(str(e))) + return False + + def _mv(self, src_path: str, dst_path: str, **kwargs: Any) -> bool: + """ + Moves (renames) a source path to a destination path. + + Args: + src_path (str): A URI supported by this PathHandler + dst_path (str): A URI supported by this PathHandler + + Returns: + status (bool): True on success + """ + self._check_kwargs(kwargs) + src_path = self._get_path_with_cwd(src_path) + dst_path = self._get_path_with_cwd(dst_path) + if os.path.exists(dst_path): + logger = logging.getLogger(__name__) + logger.error("Destination file {} already exists.".format(dst_path)) + return False + + try: + shutil.move(src_path, dst_path) + return True + except Exception as e: + logger = logging.getLogger(__name__) + logger.error("Error in move operation - {}".format(str(e))) + return False + + def _symlink(self, src_path: str, dst_path: str, **kwargs: Any) -> bool: + """ + Creates a symlink to the src_path at the dst_path + + Args: + src_path (str): A URI supported by this PathHandler + dst_path (str): A URI supported by this PathHandler + + Returns: + status (bool): True on success + """ + self._check_kwargs(kwargs) + src_path = self._get_path_with_cwd(src_path) + dst_path = self._get_path_with_cwd(dst_path) + logger = logging.getLogger(__name__) + if not os.path.exists(src_path): + logger.error("Source path {} does not exist".format(src_path)) + return False + if os.path.exists(dst_path): + logger.error("Destination path {} already exists.".format(dst_path)) + return False + try: + os.symlink(src_path, dst_path) + return True + except Exception as e: + logger.error("Error in symlink - {}".format(str(e))) + return False + + def _exists(self, path: str, **kwargs: Any) -> bool: + self._check_kwargs(kwargs) + return os.path.exists(self._get_path_with_cwd(path)) + + def _isfile(self, path: str, **kwargs: Any) -> bool: + self._check_kwargs(kwargs) + return os.path.isfile(self._get_path_with_cwd(path)) + + def _isdir(self, path: str, **kwargs: Any) -> bool: + self._check_kwargs(kwargs) + return os.path.isdir(self._get_path_with_cwd(path)) + + def _ls(self, path: str, **kwargs: Any) -> List[str]: + self._check_kwargs(kwargs) + return os.listdir(self._get_path_with_cwd(path)) + + def _mkdirs(self, path: str, **kwargs: Any) -> None: + self._check_kwargs(kwargs) + try: + os.makedirs(path, exist_ok=True) + except OSError as e: + # EEXIST it can still happen if multiple processes are creating the dir + if e.errno != errno.EEXIST: + raise + + def _rm(self, path: str, **kwargs: Any) -> None: + self._check_kwargs(kwargs) + os.remove(path) + + def _set_cwd(self, path: Union[str, None], **kwargs: Any) -> bool: + self._check_kwargs(kwargs) + # Remove cwd path if None + if path is None: + self._cwd = None + return True + + # Make sure path is a valid Unix path + if not os.path.exists(path): + raise ValueError(f"{path} is not a valid Unix path") + # Make sure path is an absolute path + if not os.path.isabs(path): + raise ValueError(f"{path} is not an absolute path") + self._cwd = path + return True + + def _get_path_with_cwd(self, path: str) -> str: + return os.path.normpath(path if not self._cwd else os.path.join(self._cwd, path)) + + +class HTTPURLHandler(PathHandler): + """ + Download URLs and cache them to disk. + """ + + def __init__(self) -> None: + self.cache_map: Dict[str, str] = {} + + def _get_supported_prefixes(self) -> List[str]: + return ["http://", "https://", "ftp://"] + + def _get_local_path(self, path: str, force: bool = False, **kwargs: Any) -> str: + """ + This implementation downloads the remote resource and caches it locally. + The resource will only be downloaded if not previously requested. + """ + self._check_kwargs(kwargs) + if force or path not in self.cache_map or not os.path.exists(self.cache_map[path]): + logger = logging.getLogger(__name__) + parsed_url = urlparse(path) + dirname = os.path.join(get_cache_dir(), os.path.dirname(parsed_url.path.lstrip("/"))) + filename = path.split("/")[-1] + cached = os.path.join(dirname, filename) + with file_lock(cached): + if not os.path.isfile(cached): + logger.info("Downloading {} ...".format(path)) + cached = download(path, dirname, filename=filename) + logger.info("URL {} cached in {}".format(path, cached)) + self.cache_map[path] = cached + return self.cache_map[path] + + def _open( + self, path: str, mode: str = "r", buffering: int = -1, **kwargs: Any + ) -> Union[IO[str], IO[bytes]]: + """ + Open a remote HTTP path. The resource is first downloaded and cached + locally. + + Args: + path (str): A URI supported by this PathHandler + mode (str): Specifies the mode in which the file is opened. It defaults + to 'r'. + buffering (int): Not used for this PathHandler. + + Returns: + file: a file-like object. + """ + self._check_kwargs(kwargs) + assert mode in ("r", "rb"), "{} does not support open with {} mode".format( + self.__class__.__name__, mode + ) + assert ( + buffering == -1 + ), f"{self.__class__.__name__} does not support the `buffering` argument" + local_path = self._get_local_path(path, force=False) + return open(local_path, mode) + + +class OneDrivePathHandler(HTTPURLHandler): + """ + Map OneDrive (short) URLs to direct download links + """ + + ONE_DRIVE_PREFIX = "https://1drv.ms/u/s!" + + def create_one_drive_direct_download(self, one_drive_url: str) -> str: + """ + Converts a short OneDrive URI into a download link that can be used with wget + + Args: + one_drive_url (str): A OneDrive URI supported by this PathHandler + + Returns: + result_url (str): A direct download URI for the file + """ + data_b64 = base64.b64encode(bytes(one_drive_url, "utf-8")) + data_b64_string = data_b64.decode("utf-8").replace("/", "_").replace("+", "-").rstrip("=") + result_url = f"https://api.onedrive.com/v1.0/shares/u!{data_b64_string}/root/content" + return result_url + + def _get_supported_prefixes(self) -> List[str]: + return [self.ONE_DRIVE_PREFIX] + + def _get_local_path(self, path: str, force: bool = False, **kwargs: Any) -> str: + """ + This implementation downloads the remote resource and caches it locally. + The resource will only be downloaded if not previously requested. + """ + logger = logging.getLogger(__name__) + direct_url = self.create_one_drive_direct_download(path) + + logger.info(f"URL {path} mapped to direct download link {direct_url}") + + return super()._get_local_path(os.fspath(direct_url), force=force, **kwargs) + + +class PathManagerBase: + """ + A class for users to open generic paths or translate generic paths to file names. + path_manager.method(path) will do the following: + + 1. Find a handler by checking the prefixes in `self._path_handlers`. + 2. Call handler.method(path) on the handler that's found + """ + + def __init__(self) -> None: + self._path_handlers: MutableMapping[str, PathHandler] = OrderedDict() + """ + Dict for path prefix to handler + """ + + self._native_path_handler: PathHandler = NativePathHandler() + """ + A NativePathHandler that works on posix paths. This is used as the fallback. + """ + + self._cwd: Optional[str] = None + """ + Keeps track of the single cwd (if set). + NOTE: Only one PathHandler can have a cwd set at a time. + """ + + self._async_handlers: Set[PathHandler] = set() + """ + Keeps track of the PathHandler subclasses where `opena` was used so + all of the threads can be properly joined when calling + `PathManager.join`. + """ + + def __get_path_handler(self, path: Union[str, os.PathLike]) -> PathHandler: + """ + Finds a PathHandler that supports the given path. Falls back to the native + PathHandler if no other handler is found. + + Args: + path (str or os.PathLike): URI path to resource + + Returns: + handler (PathHandler) + """ + path = os.fspath(path) # pyre-ignore + for p in self._path_handlers.keys(): + if path.startswith(p): + return self._path_handlers[p] + return self._native_path_handler + + def opent( + self, path: str, mode: str = "r", buffering: int = 32, **kwargs: Any + ) -> Iterable[Any]: + """ + Open a tabular data source. Only reading is supported. + The opent() returns a Python iterable collection object, compared to + bytes/text data with open() + + Args: + path (str): A URI supported by this PathHandler + mode (str): Specifies the mode in which the file is opened. It defaults + to 'r' + buffering (int): number of rows fetched and cached + + Returns: + An iterable collection object. + """ + return self.__get_path_handler(path)._opent(path, mode, buffering, **kwargs) + + def open( + self, path: str, mode: str = "r", buffering: int = -1, **kwargs: Any + ) -> Union[IO[str], IO[bytes]]: + """ + Open a stream to a URI, similar to the built-in `open`. + + Args: + path (str): A URI supported by this PathHandler + mode (str): Specifies the mode in which the file is opened. It defaults + to 'r'. + buffering (int): An optional integer used to set the buffering policy. + Pass 0 to switch buffering off and an integer >= 1 to indicate the + size in bytes of a fixed-size chunk buffer. When no buffering + argument is given, the default buffering policy depends on the + underlying I/O implementation. + + Returns: + file: a file-like object. + """ + return self.__get_path_handler(path)._open( # type: ignore + path, mode, buffering=buffering, **kwargs + ) + + # NOTE: This feature is only implemented for `NativePathHandler` and can + # currently only be used in write mode. + def opena( + self, + path: str, + mode: str = "r", + buffering: int = -1, + callback_after_file_close: Optional[Callable[[None], None]] = None, + **kwargs: Any, + ) -> Union[IO[str], IO[bytes]]: + """ + Open a file with asynchronous permissions. `f.write()` calls (and + potentially `f.read()` calls in the future) will be dispatched + asynchronously such that the main program can continue running. + + NOTE: Writes to the same path are serialized so they are written in + the same order as they were called but writes to distinct paths can + happen concurrently. + + Usage (default / without callback function): + for n in range(50): + results = run_a_large_task(n) + # `f` is a file-like object with asynchronous methods + with path_manager.opena(uri, "w") as f: + f.write(results) # Runs in separate thread + # Main process returns immediately and continues to next iteration + path_manager.async_close() + + Usage (advanced / with callback function): + # To asynchronously write to Manifold: + def cb(): + path_manager.copy_from_local( + "checkpoint.pt", "manifold://path/to/bucket" + ) + f = pm.opena("checkpoint.pt", "wb", callback_after_file_close=cb) + oneflow.save({...}, f) + f.close() + + Args: + ... + callback_after_file_close (Callable): An optional argument that can + be passed to perform operations that depend on the asynchronous + writes being completed. The file is first written to the local + disk and then the callback is executed. + + Returns: + file: a file-like object with asynchronous methods. + """ + non_blocking_io = self.__get_path_handler(path)._opena( + path, + mode, + buffering=buffering, + callback_after_file_close=callback_after_file_close, + **kwargs, + ) + # Keep track of the path handlers where `opena` is used so that all of the + # threads can be properly joined on `PathManager.join`. + self._async_handlers.add(self.__get_path_handler(path)) + return non_blocking_io + + def async_join(self, *paths: str, **kwargs: Any) -> bool: + """ + Ensures that desired async write threads are properly joined. + + Usage: + Wait for asynchronous methods operating on specific file paths to + complete. + async_join("path/to/file1.txt") + async_join("path/to/file2.txt", "path/to/file3.txt") + Wait for all asynchronous methods to complete. + async_join() + + Args: + *paths (str): Pass in any number of file paths and `async_join` will wait + until all asynchronous activity for those paths is complete. If no + paths are passed in, then `async_join` will wait until all asynchronous + jobs are complete. + + Returns: + status (bool): True on success + """ + success = True + if not paths: # Join all. + for handler in self._async_handlers: + success = handler._async_join(**kwargs) and success + else: # Join specific paths. + for path in paths: + success = self.__get_path_handler(path)._async_join(path, **kwargs) and success + return success + + def async_close(self, **kwargs: Any) -> bool: + """ + `async_close()` must be called at the very end of any script that uses the + asynchronous `opena` feature. This calls `async_join()` first and then closes + the thread pool used for the asynchronous operations. + + Returns: + status (bool): True on success + """ + success = self.async_join(**kwargs) + for handler in self._async_handlers: + success = handler._async_close(**kwargs) and success + self._async_handlers.clear() + return success + + def copy(self, src_path: str, dst_path: str, overwrite: bool = False, **kwargs: Any) -> bool: + """ + Copies a source path to a destination path. + + Args: + src_path (str): A URI supported by this PathHandler + dst_path (str): A URI supported by this PathHandler + overwrite (bool): Bool flag for forcing overwrite of existing file + + Returns: + status (bool): True on success + """ + + # Copying across handlers is not supported. + assert self.__get_path_handler(src_path) == self.__get_path_handler( # type: ignore + dst_path + ) + return self.__get_path_handler(src_path)._copy(src_path, dst_path, overwrite, **kwargs) + + def mv(self, src_path: str, dst_path: str, **kwargs: Any) -> bool: + """ + Moves (renames) a source path supported by NativePathHandler to + a destination path. + + Args: + src_path (str): A URI supported by NativePathHandler + dst_path (str): A URI supported by NativePathHandler + + Returns: + status (bool): True on success + + Exception: + Asserts if both the src and dest paths are not supported by + NativePathHandler. + """ + + # Moving across handlers is not supported. + assert self.__get_path_handler(src_path) == self.__get_path_handler( # type: ignore + dst_path + ), "Src and dest paths must be supported by the same path handler." + return self.__get_path_handler(src_path)._mv(src_path, dst_path, **kwargs) + + def get_local_path(self, path: str, force: bool = False, **kwargs: Any) -> str: + """ + Get a filepath which is compatible with native Python I/O such as `open` + and `os.path`. + + If URI points to a remote resource, this function may download and cache + the resource to local disk. + + Args: + path (str): A URI supported by this PathHandler + force(bool): Forces a download from backend if set to True. + Returns: + local_path (str): a file path which exists on the local file system + """ + path = os.fspath(path) + return self.__get_path_handler(path)._get_local_path( # type: ignore + path, force=force, **kwargs + ) + + def copy_from_local( + self, local_path: str, dst_path: str, overwrite: bool = False, **kwargs: Any + ) -> None: + """ + Copies a local file to the specified URI. + If the URI is another local path, this should be functionally identical + to copy. + + Args: + local_path (str): a file path which exists on the local file system + dst_path (str): A URI supported by this PathHandler + overwrite (bool): Bool flag for forcing overwrite of existing URI + + Returns: + status (bool): True on success + """ + assert os.path.exists(local_path) + return self.__get_path_handler(dst_path)._copy_from_local( + local_path=local_path, dst_path=dst_path, overwrite=overwrite, **kwargs + ) + + def exists(self, path: str, **kwargs: Any) -> bool: + """ + Checks if there is a resource at the given URI. + + Args: + path (str): A URI supported by this PathHandler + + Returns: + bool: true if the path exists + """ + return self.__get_path_handler(path)._exists(path, **kwargs) # type: ignore + + def isfile(self, path: str, **kwargs: Any) -> bool: + """ + Checks if there the resource at the given URI is a file. + + Args: + path (str): A URI supported by this PathHandler + + Returns: + bool: true if the path is a file + """ + return self.__get_path_handler(path)._isfile(path, **kwargs) # type: ignore + + def isdir(self, path: str, **kwargs: Any) -> bool: + """ + Checks if the resource at the given URI is a directory. + + Args: + path (str): A URI supported by this PathHandler + + Returns: + bool: true if the path is a directory + """ + return self.__get_path_handler(path)._isdir(path, **kwargs) # type: ignore + + def ls(self, path: str, **kwargs: Any) -> List[str]: + """ + List the contents of the directory at the provided URI. + + Args: + path (str): A URI supported by this PathHandler + + Returns: + List[str]: list of contents in given path + """ + return self.__get_path_handler(path)._ls(path, **kwargs) + + def mkdirs(self, path: str, **kwargs: Any) -> None: + """ + Recursive directory creation function. Like mkdir(), but makes all + intermediate-level directories needed to contain the leaf directory. + Similar to the native `os.makedirs`. + + Args: + path (str): A URI supported by this PathHandler + """ + return self.__get_path_handler(path)._mkdirs(path, **kwargs) # type: ignore + + def rm(self, path: str, **kwargs: Any) -> None: + """ + Remove the file (not directory) at the provided URI. + + Args: + path (str): A URI supported by this PathHandler + """ + return self.__get_path_handler(path)._rm(path, **kwargs) # type: ignore + + def symlink(self, src_path: str, dst_path: str, **kwargs: Any) -> bool: + """Symlink the src_path to the dst_path + + Args: + src_path (str): A URI supported by this PathHandler to symlink from + dst_path (str): A URI supported by this PathHandler to symlink to + """ + # Copying across handlers is not supported. + assert self.__get_path_handler(src_path) == self.__get_path_handler( # type: ignore + dst_path + ) + return self.__get_path_handler(src_path)._symlink(src_path, dst_path, **kwargs) + + def set_cwd(self, path: Union[str, None], **kwargs: Any) -> bool: + """ + Set the current working directory. PathHandler classes prepend the cwd + to all URI paths that are handled. + + Args: + path (str) or None: A URI supported by this PathHandler. Must be a valid + absolute Unix path or None to set the cwd to None. + + Returns: + bool: true if cwd was set without errors + """ + if path is None and self._cwd is None: + return True + if self.__get_path_handler(path or self._cwd)._set_cwd(path, **kwargs): # type: ignore + self._cwd = path + return True + return False + + def register_handler(self, handler: PathHandler, allow_override: bool = False) -> None: + """ + Register a path handler associated with `handler._get_supported_prefixes` + URI prefixes. + + Args: + handler (PathHandler) + allow_override (bool): allow overriding existing handler for prefix + """ + logger = logging.getLogger(__name__) + assert isinstance(handler, PathHandler), handler + + # Allow override of `NativePathHandler` which is automatically + # instantiated by `PathManager`. + if isinstance(handler, NativePathHandler): + if allow_override: + self._native_path_handler = handler + else: + raise ValueError( + "`NativePathHandler` is registered by default. Use the " + "`allow_override=True` kwarg to override it." + ) + return + + for prefix in handler._get_supported_prefixes(): + if prefix not in self._path_handlers: + self._path_handlers[prefix] = handler + continue + + old_handler_type = type(self._path_handlers[prefix]) + if allow_override: + # if using the global PathManager, show the warnings + global g_pathmgr + if self == g_pathmgr: + logger.warning( + f"[PathManager] Attempting to register prefix '{prefix}' from " + "the following call stack:\n" + "".join(traceback.format_stack(limit=5)) + # show the most recent callstack + ) + logger.warning( + f"[PathManager] Prefix '{prefix}' is already registered " + f"by {old_handler_type}. We will override the old handler. " + "To avoid such conflicts, create a project-specific PathManager " + "instead." + ) + self._path_handlers[prefix] = handler + else: + raise KeyError( + f"[PathManager] Prefix '{prefix}' already registered by {old_handler_type}!" + ) + + # Sort path handlers in reverse order so longer prefixes take priority, + # eg: http://foo/bar before http://foo + self._path_handlers = OrderedDict( + sorted(self._path_handlers.items(), key=lambda t: t[0], reverse=True) + ) + + def set_strict_kwargs_checking(self, enable: bool) -> None: + """ + Toggles strict kwargs checking. If enabled, a ValueError is thrown if any + unused parameters are passed to a PathHandler function. If disabled, only + a warning is given. + + With a centralized file API, there's a tradeoff of convenience and + correctness delegating arguments to the proper I/O layers. An underlying + `PathHandler` may support custom arguments which should not be statically + exposed on the `PathManager` function. For example, a custom `HTTPURLHandler` + may want to expose a `cache_timeout` argument for `open()` which specifies + how old a locally cached resource can be before it's refetched from the + remote server. This argument would not make sense for a `NativePathHandler`. + If strict kwargs checking is disabled, `cache_timeout` can be passed to + `PathManager.open` which will forward the arguments to the underlying + handler. By default, checking is enabled since it is innately unsafe: + multiple `PathHandler`s could reuse arguments with different semantic + meanings or types. + + Args: + enable (bool) + """ + self._native_path_handler._strict_kwargs_check = enable + for handler in self._path_handlers.values(): + handler._strict_kwargs_check = enable + + +class PathManagerFactory: + """ + PathManagerFactory is the class responsible for creating new PathManager + instances and removing them when no longer needed. + PathManager can be instantiated directly too, but it is recommended that + you use PathManagerFactory to create them. + """ + + GLOBAL_PATH_MANAGER = "global_path_manager" + pm_list = {} + + @staticmethod + def get(key=GLOBAL_PATH_MANAGER) -> PathManagerBase: + """ + Get the path manager instance associated with a key. + A new instance will be created if there is no existing + instance associated with the key passed in. + + Args: + key (str): + """ + if key not in PathManagerFactory.pm_list: + PathManagerFactory.pm_list[key] = PathManagerBase() + return PathManagerFactory.pm_list[key] + + @staticmethod + def remove(key): + """ + Remove the path manager instance associated with a key. + + Args: + key (str): + """ + if key in PathManagerFactory.pm_list: + _pm = PathManagerFactory.pm_list.pop(key) # noqa + del _pm + + +""" +A global instance of PathManager. +This global instance is provided for backward compatibility, but it is +recommended that clients use PathManagerFactory +""" +g_pathmgr = PathManagerFactory.get() + + +PathManager = PathManagerBase() +PathManager.register_handler(HTTPURLHandler()) +PathManager.register_handler(OneDrivePathHandler()) diff --git a/libai/utils/file_utils.py b/libai/utils/file_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2f0a860846afdea5b7f547d67d04ebe9bac30764 --- /dev/null +++ b/libai/utils/file_utils.py @@ -0,0 +1,342 @@ +""" +Utilities for working with the local dataset cache. +This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp +Copyright by the AllenNLP authors. +""" +from __future__ import absolute_import, division, print_function, unicode_literals + +import fnmatch +import hashlib +import json +import logging +import os +import shutil +import sys +import tempfile +from functools import wraps +from io import open +from pathlib import Path + +import boto3 +import requests +import wget +from botocore.config import Config +from botocore.exceptions import ClientError +from tqdm import tqdm + +logger = logging.getLogger(__name__) # pylint: disable=invalid-name + + +cache_home = Path(os.getenv("OF_CACHE_ROOT", Path.home() / ".of_cache")) +default_cache_path = str(cache_home / "libai") + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + + +def url_to_filename(url, etag=None): + """ + Convert `url` into a hashed filename in a repeatable way. + If `etag` is specified, append its hash to the url's, delimited + by a period. + If the url ends with .h5 (Keras HDF5 weights) ands '.h5' to the name + so that TF 2.0 can identify it as a HDF5 file + (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3 + /tensorflow/python/keras/engine/network.py#L1380) + """ + url_bytes = url.encode("utf-8") + url_hash = hashlib.sha256(url_bytes) + filename = url_hash.hexdigest() + + if etag: + etag_bytes = etag.encode("utf-8") + etag_hash = hashlib.sha256(etag_bytes) + filename += "." + etag_hash.hexdigest() + + if url.endswith(".h5"): + filename += ".h5" + + return filename + + +def filename_to_url(filename, cache_dir=None): + """ + Return the url and etag (which may be ``None``) stored for `filename`. + Raise ``EnvironmentError`` if `filename` or its stored metadata do not exist. + """ + if cache_dir is None: + cache_dir = default_cache_path + if sys.version_info[0] == 3 and isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + cache_path = os.path.join(cache_dir, filename) + if not os.path.exists(cache_path): + raise EnvironmentError("file {} not found".format(cache_path)) + + meta_path = cache_path + ".json" + if not os.path.exists(meta_path): + raise EnvironmentError("file {} not found".format(meta_path)) + + with open(meta_path, encoding="utf-8") as meta_file: + metadata = json.load(meta_file) + url = metadata["url"] + etag = metadata["etag"] + + return url, etag + + +def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=None): + """ + Given something that might be a URL (or might be a local path), + determine which. If it's a URL, download the file and cache it, and + return the path to the cached file. If it's already a local path, + make sure the file exists and then return the path. + Args: + cache_dir: specify a cache directory to save the file to (overwrite the default cache dir). + force_download: if True, re-dowload the file even if it's already cached in the cache dir. + """ + if cache_dir is None: + cache_dir = default_cache_path + if sys.version_info[0] == 3 and isinstance(url_or_filename, Path): + url_or_filename = str(url_or_filename) + if sys.version_info[0] == 3 and isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + parsed = urlparse(url_or_filename) + + if parsed.scheme in ("http", "https", "s3"): + # URL, so get it from the cache (downloading if necessary) + return get_from_cache( + url_or_filename, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + ) + elif os.path.exists(url_or_filename): + # File, and it exists. + return url_or_filename + elif parsed.scheme == "": + # File, but it doesn't exist. + raise EnvironmentError("file {} not found".format(url_or_filename)) + else: + # Something unknown + raise ValueError("unable to parse {} as a URL or as a local path".format(url_or_filename)) + + +def split_s3_path(url): + """Split a full s3 path into the bucket name and path.""" + parsed = urlparse(url) + if not parsed.netloc or not parsed.path: + raise ValueError("bad s3 path {}".format(url)) + bucket_name = parsed.netloc + s3_path = parsed.path + # Remove '/' at beginning of path. + if s3_path.startswith("/"): + s3_path = s3_path[1:] + return bucket_name, s3_path + + +def s3_request(func): + """ + Wrapper function for s3 requests in order to create more helpful error + messages. + """ + + @wraps(func) + def wrapper(url, *args, **kwargs): + try: + return func(url, *args, **kwargs) + except ClientError as exc: + if int(exc.response["Error"]["Code"]) == 404: + raise EnvironmentError("file {} not found".format(url)) + else: + raise + + return wrapper + + +@s3_request +def s3_etag(url, proxies=None): + """Check ETag on S3 object.""" + s3_resource = boto3.resource("s3", config=Config(proxies=proxies)) + bucket_name, s3_path = split_s3_path(url) + s3_object = s3_resource.Object(bucket_name, s3_path) + return s3_object.e_tag + + +@s3_request +def s3_get(url, temp_file, proxies=None): + """Pull a file directly from S3.""" + s3_resource = boto3.resource("s3", config=Config(proxies=proxies)) + bucket_name, s3_path = split_s3_path(url) + s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file) + + +def http_get(url, temp_file, proxies=None): + req = requests.get(url, stream=True, proxies=proxies) + content_length = req.headers.get("Content-Length") + total = int(content_length) if content_length is not None else None + progress = tqdm(unit="B", total=total) + for chunk in req.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + progress.update(len(chunk)) + temp_file.write(chunk) + progress.close() + + +def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag_timeout=10): + """ + Given a URL, look for the corresponding dataset in the local cache. + If it's not there, download it. Then return the path to the cached file. + """ + if cache_dir is None: + cache_dir = default_cache_path + if sys.version_info[0] == 3 and isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + + # Get eTag to add to filename, if it exists. + if url.startswith("s3://"): + etag = s3_etag(url, proxies=proxies) + else: + try: + response = requests.head( + url, allow_redirects=True, proxies=proxies, timeout=etag_timeout + ) + if response.status_code != 200: + etag = None + else: + etag = response.headers.get("ETag") + except (EnvironmentError, requests.exceptions.Timeout): + etag = None + + filename = url_to_filename(url, etag) + + # get cache path to put the file + cache_path = os.path.join(cache_dir, filename) + + # If we don't have a connection (etag is None) and can't identify the file + # try to get the last downloaded one + if not os.path.exists(cache_path) and etag is None: + matching_files = fnmatch.filter(os.listdir(cache_dir), filename + ".*") + matching_files = list(filter(lambda s: not s.endswith(".json"), matching_files)) + if matching_files: + cache_path = os.path.join(cache_dir, matching_files[-1]) + + if not os.path.exists(cache_path) or force_download: + # Download to temporary file, then copy to cache dir once finished. + # Otherwise you get corrupt cache entries if the download gets interrupted. + with tempfile.NamedTemporaryFile() as temp_file: + logger.info( + "%s not found in cache or force_download set to True, downloading to %s", + url, + temp_file.name, + ) + + # GET file object + if url.startswith("s3://"): + s3_get(url, temp_file, proxies=proxies) + else: + http_get(url, temp_file, proxies=proxies) + + # we are copying the file before closing it, so flush to avoid truncation + temp_file.flush() + # shutil.copyfileobj() starts at the current position, so go to the start + temp_file.seek(0) + + logger.info("copying %s to cache at %s", temp_file.name, cache_path) + with open(cache_path, "wb") as cache_file: + shutil.copyfileobj(temp_file, cache_file) + + logger.info("creating metadata file for %s", cache_path) + meta = {"url": url, "etag": etag} + meta_path = cache_path + ".json" + with open(meta_path, "w") as meta_file: + output_string = json.dumps(meta) + meta_file.write(output_string) + + logger.info("removing temp file %s", temp_file.name) + + return cache_path + + +def get_md5(fname): + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + result = hash_md5.hexdigest() + return result + + +def download_file(out_path: str, url): + logger.info(f"downloading from {url} to {out_path}") + wget.download(url, out=out_path) + + +def get_data_from_cache(url, cache_dir=None, force_download=False, md5=None): + """ + Given a URL, look for the corresponding dataset in the local cache. + If it's not there, download it. Then return the path to the cached file. + """ + if cache_dir is None: + cache_dir = default_cache_path + if sys.version_info[0] == 3 and isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + + filename = url.split("/")[-1] + + # get cache path to put the file + cache_path = os.path.join(cache_dir, filename) + + # If we have already get the file, just check the md5 if provided + if os.path.exists(cache_path) and md5 is not None: + local_file_md5 = get_md5(cache_path) + if local_file_md5 != md5: + os.unlink(cache_path) + download_file(cache_path, url) + + # If we don't have a connection (etag is None) and can't identify the file + # try to get the last downloaded one + if not os.path.exists(cache_path): + download_file(cache_path, url) + + if not os.path.exists(cache_path) or force_download: + # Download to temporary file, then copy to cache dir once finished. + # Otherwise you get corrupt cache entries if the download gets interrupted. + with tempfile.NamedTemporaryFile() as temp_file: + logger.info( + "%s not found in cache or force_download set to True, downloading to %s", + url, + temp_file.name, + ) + + # GET file object + http_get(url, temp_file) + + # we are copying the file before closing it, so flush to avoid truncation + temp_file.flush() + # shutil.copyfileobj() starts at the current position, so go to the start + temp_file.seek(0) + + logger.info("copying %s to cache at %s", temp_file.name, cache_path) + with open(cache_path, "wb") as cache_file: + shutil.copyfileobj(temp_file, cache_file) + + logger.info("creating metadata file for %s", cache_path) + meta = {"url": url} + meta_path = cache_path + ".json" + with open(meta_path, "w") as meta_file: + output_string = json.dumps(meta) + meta_file.write(output_string) + + logger.info("removing temp file %s", temp_file.name) + + return cache_path diff --git a/libai/utils/history_buffer.py b/libai/utils/history_buffer.py new file mode 100644 index 0000000000000000000000000000000000000000..ec18f7aa11f245f487af11cb07957aa4308d7570 --- /dev/null +++ b/libai/utils/history_buffer.py @@ -0,0 +1,89 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Tuple + +import numpy as np + +# -------------------------------------------------------- +# References: +# https://github.com/facebookresearch/fvcore/blob/main/fvcore/common/history_buffer.py +# -------------------------------------------------------- + + +class HistoryBuffer: + """ + Track a series of scalar values and provide access to smoothed values over a + window or the global average of the series. + """ + + def __init__(self, max_length: int = 1000000): + """ + Args: + max_length: maximal number of values that can be stored in the + buffer. When the capacity of the buffer is exhausted, old + values will be removed. + """ + self._max_length: int = max_length + self._data: List[Tuple[float, float]] = [] # (value, iteration) pairs + self._count: int = 0 + self._global_avg: float = 0 + + def update(self, value: float, iteration: float = None): + """ + Add a new scalar value produced at certain iteration. If the length + of the buffer exceeds self._max_length, the oldest element will be + removed from the buffer. + """ + if iteration is None: + iteration = self._count + if len(self._data) == self._max_length: + self._data.pop(0) + self._data.append((value, iteration)) + + self._count += 1 + self._global_avg += (value - self._global_avg) / self._count + + def latest(self): + """ + Return the latest scalar value added to the buffer. + """ + return self._data[-1][0] + + def median(self, window_size: int): + """ + Return the median of the latest `window_size` values in the buffer. + """ + return np.median([x[0] for x in self._data[-window_size:]]) + + def avg(self, window_size: int): + """ + Return the mean of the latest `window_size` values in the buffer. + """ + return np.mean([x[0] for x in self._data[-window_size:]]) + + def global_avg(self): + """ + Return the mean of all the elements in the buffer. Note that this + includes those getting removed due to limited buffer storage. + """ + return self._global_avg + + def values(self): + """ + Returns: + list[(number, iteration)]: content of the current buffer. + """ + return self._data diff --git a/libai/utils/logger.py b/libai/utils/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..9b508870ddee4250ad1f9a5af32ba5546026bdd0 --- /dev/null +++ b/libai/utils/logger.py @@ -0,0 +1,214 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import atexit +import functools +import logging +import os +import sys +import time +from collections import Counter + +from termcolor import colored + +from libai.utils.file_io import PathManager + +# -------------------------------------------------------- +# References: +# https://github.com/facebookresearch/detectron2/blob/main/detectron2/utils/logger.py +# -------------------------------------------------------- + + +class _ColorfulFormatter(logging.Formatter): + def __init__(self, *args, **kwargs): + self._root_name = kwargs.pop("root_name") + "." + self._abbrev_name = kwargs.pop("abbrev_name", "") + if len(self._abbrev_name): + self._abbrev_name = self._abbrev_name + "." + super(_ColorfulFormatter, self).__init__(*args, **kwargs) + + def formatMessage(self, record): + record.name = record.name.replace(self._root_name, self._abbrev_name) + log = super(_ColorfulFormatter, self).formatMessage(record) + if record.levelno == logging.WARNING: + prefix = colored("WARNING", "red", attrs=["blink"]) + elif record.levelno == logging.ERROR or record.levelno == logging.CRITICAL: + prefix = colored("ERROR", "red", attrs=["blink", "underline"]) + else: + return log + return prefix + " " + log + + +@functools.lru_cache() # so that calling setup_logger multiple times won't add many handlers +def setup_logger(output=None, distributed_rank=0, *, color=True, name="libai", abbrev_name=None): + """ + Args: + output (str): a file name or a directory to save log. If None, will not save log file. + If ends with ".txt" or ".log", assumed to be a file name. + Otherwise, logs will be saved to `output/log.txt`. + name (str): the root module name of this logger + abbrev_name (str): an abbreviation of the module, to avoid long names in logs. + Set to "" to not log the root module in logs. + By default, will abbreviate "detectron2" to "d2" and leave other + modules unchanged. + """ + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.propagate = False + + if abbrev_name is None: + abbrev_name = "lb" if name == "libai" else name + + plain_formatter = logging.Formatter( + "[%(asctime)s] %(name)s %(levelname)s: %(message)s", datefmt="%m/%d %H:%M:%S" + ) + # stdout logging: master only + if distributed_rank == 0: + ch = logging.StreamHandler(stream=sys.stdout) + ch.setLevel(logging.DEBUG) + if color: + formatter = _ColorfulFormatter( + colored("[%(asctime)s %(name)s]: ", "green") + "%(message)s", + datefmt="%m/%d %H:%M:%S", + root_name=name, + abbrev_name=str(abbrev_name), + ) + else: + formatter = plain_formatter + ch.setFormatter(formatter) + logger.addHandler(ch) + + # file logging: all workers + if output is not None: + if output.endswith(".txt") or output.endswith(".log"): + filename = output + else: + filename = os.path.join(output, "log.txt") + if distributed_rank > 0: + filename = filename + ".rank{}".format(distributed_rank) + PathManager.mkdirs(os.path.dirname(filename)) + + fh = logging.StreamHandler(_cached_log_stream(filename)) + fh.setLevel(logging.DEBUG) + fh.setFormatter(plain_formatter) + logger.addHandler(fh) + + return logger + + +# cache the opened file object, so that different calls to `setup_logger` +# with the same file name can safely write to the same file. +@functools.lru_cache(maxsize=None) +def _cached_log_stream(filename): + # use 1K buffer if writing to cloud storage + io = PathManager.open(filename, "a", buffering=1024 if "://" in filename else -1) + atexit.register(io.close) + return io + + +""" +Below are some other convenient logging methods. +They are mainly adopted from +https://github.com/abseil/abseil-py/blob/master/absl/logging/__init__.py +""" + + +def _find_caller(): + """ + Returns: + str: module name of the caller + tuple: a hashable key to be used to identify different callers + """ + frame = sys._getframe(2) + while frame: + code = frame.f_code + if os.path.join("utils", "logger.") not in code.co_filename: + mod_name = frame.f_globals["__name__"] + if mod_name == "__main__": + mod_name = "libai" + return mod_name, (code.co_filename, frame.f_lineno, code.co_name) + frame = frame.f_back + + +_LOG_COUNTER = Counter() +_LOG_TIMER = {} + + +def log_first_n(lvl, msg, n=1, *, name=None, key="caller"): + """ + Log only for the first n times. + + Args: + lvl (int): the logging level + msg (str): + n (int): + name (str): name of the logger to use. Will use the caller's module by default. + key (str or tuple[str]): the string(s) can be one of "caller" or + "message", which defines how to identify duplicated logs. + For example, if called with `n=1, key="caller"`, this function + will only log the first call from the same caller, regardless of + the message content. + If called with `n=1, key="message"`, this function will log the + same content only once, even if they are called from different places. + If called with `n=1, key=("caller", "message")`, this function + will not log only if the same caller has logged the same message before. + """ + if isinstance(key, str): + key = (key,) + assert len(key) > 0 + + caller_module, caller_key = _find_caller() + hash_key = () + if "caller" in key: + hash_key = hash_key + caller_key + if "message" in key: + hash_key = hash_key + (msg,) + + _LOG_COUNTER[hash_key] += 1 + if _LOG_COUNTER[hash_key] <= n: + logging.getLogger(name or caller_module).log(lvl, msg) + + +def log_every_n(lvl, msg, n=1, *, name=None): + """ + Log once per n times. + Args: + lvl (int): the logging level + msg (str): + n (int): + name (str): name of the logger to use. Will use the caller's module by default. + """ + caller_module, key = _find_caller() + _LOG_COUNTER[key] += 1 + if n == 1 or _LOG_COUNTER[key] % n == 1: + logging.getLogger(name or caller_module).log(lvl, msg) + + +def log_every_n_seconds(lvl, msg, n=1, *, name=None): + """ + Log no more than once per n seconds. + Args: + lvl (int): the logging level + msg (str): + n (int): + name (str): name of the logger to use. Will use the caller's module by default. + """ + caller_module, key = _find_caller() + last_logged = _LOG_TIMER.get(key, None) + current_time = time.time() + if last_logged is None or current_time - last_logged >= n: + logging.getLogger(name or caller_module).log(lvl, msg) + _LOG_TIMER[key] = current_time diff --git a/libai/utils/non_blocking_io.py b/libai/utils/non_blocking_io.py new file mode 100644 index 0000000000000000000000000000000000000000..0f0b6b4853de3ba64531b45834902bb94c65c6d6 --- /dev/null +++ b/libai/utils/non_blocking_io.py @@ -0,0 +1,376 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import concurrent.futures +import io +import logging +from dataclasses import dataclass +from queue import Queue +from threading import Thread +from typing import IO, Callable, Optional, Union + +# -------------------------------------------------------- +# References: +# https://github.com/facebookresearch/iopath/blob/main/iopath/common/non_blocking_io.py +# -------------------------------------------------------- + + +""" +This file is used for asynchronous file operations. +When `opena` is called for the first time for a specific +`PathHandler`, a `NonBlockingIOManager` is instantiated. The +manager returns a `NonBlockingIO` (or `NonBlockingBufferedIO`) +instance to the caller, and the manager maintains all of the +thread management and data management. +""" + + +@dataclass +class PathData: + """ + Manage the IO job queue and polling thread for a single + path. This is done to ensure that write calls to the same + path are serialized so they are written in the same order + as they were called. + On each `f.write` call where `f` is of type `NonBlockingIO`, + we send the job to the manager where it is enqueued to the + Queue. The polling Thread picks up on the job, executes it, + waits for it to finish, and then continues to poll. + """ + + queue: Queue + thread: Thread + + +class NonBlockingIOManager: + """ + All `opena` calls pass through this class so that it can + keep track of the threads for proper cleanup at the end + of the script. Each path that is opened with `opena` is + assigned a single queue and polling thread that is kept + open until it is cleaned up by `PathManager.async_join()`. + """ + + def __init__( + self, + buffered: Optional[bool] = False, + executor: Optional[concurrent.futures.Executor] = None, + ) -> None: + """ + Args: + buffered (bool): IO instances will be `NonBlockingBufferedIO` + or `NonBlockingIO` based on this value. This bool is set + manually for each `PathHandler` in `_opena`. + executor: User can optionally attach a custom executor to + perform async operations through `PathHandler.__init__`. + """ + self._path_to_data = {} # Map from path to `PathData` object + self._buffered = buffered + self._IO = NonBlockingBufferedIO if self._buffered else NonBlockingIO + self._pool = executor or concurrent.futures.ThreadPoolExecutor() + + def get_non_blocking_io( + self, + path: str, + io_obj: Union[IO[str], IO[bytes]], + callback_after_file_close: Optional[Callable[[None], None]] = None, + buffering: Optional[int] = -1, + ) -> Union[IO[str], IO[bytes]]: + """ + Called by `PathHandler._opena` with the path and returns a + `NonBlockingIO` instance. + Args: + path (str): A path str to operate on. This path should be + simplified to ensure that each absolute path has only a single + path str that maps onto it. For example, in `NativePathHandler`, + we can use `os.path.normpath`. + io_obj (IO): a reference to the IO object returned by the + `PathHandler._open` function. + callback_after_file_close (Callable): An optional argument that can + be passed to perform operations that depend on the asynchronous + writes being completed. The file is first written to the local + disk and then the callback is executed. + buffering (int): An optional argument to set the buffer size for + buffered asynchronous writing. + """ + if not self._buffered and buffering != -1: + raise ValueError( + "NonBlockingIO is not using a buffered writer but `buffering` " + f"arg is set to non-default value of {buffering} != -1." + ) + + if path not in self._path_to_data: + # Initialize job queue and a polling thread + queue = Queue() + t = Thread(target=self._poll_jobs, args=(queue,)) + t.start() + # Store the `PathData` + self._path_to_data[path] = PathData(queue, t) + + kwargs = {} if not self._buffered else {"buffering": buffering} + return self._IO( + notify_manager=lambda io_callable: ( # Pass async jobs to manager + self._path_to_data[path].queue.put(io_callable) + ), + io_obj=io_obj, + callback_after_file_close=callback_after_file_close, + **kwargs, + ) + + def _poll_jobs(self, queue: Optional[Callable[[], None]]) -> None: + """ + A single thread runs this loop. It waits for an IO callable to be + placed in a specific path's `Queue` where the queue contains + callable functions. It then waits for the IO job to be completed + before looping to ensure write order. + """ + while True: + # `func` is a callable function (specifically a lambda function) + # and can be any of: + # - func = file.write(b) + # - func = file.close() + # - func = None + func = queue.get() # Blocks until item read. + if func is None: # Thread join signal. + break + self._pool.submit(func).result() # Wait for job to finish. + + def _join(self, path: Optional[str] = None) -> bool: + """ + Waits for write jobs for a specific path or waits for all + write jobs for the path handler if no path is provided. + Args: + path (str): Pass in a file path and will wait for the + asynchronous jobs to be completed for that file path. + If no path is passed in, then all threads operating + on all file paths will be joined. + """ + if path and path not in self._path_to_data: + raise ValueError( + f"{path} has no async IO associated with it. " + f"Make sure `opena({path})` is called first." + ) + # If a `_close` call fails, we print the error and continue + # closing the rest of the IO objects. + paths_to_close = [path] if path else list(self._path_to_data.keys()) + success = True + for _path in paths_to_close: + try: + path_data = self._path_to_data.pop(_path) + path_data.queue.put(None) + path_data.thread.join() + except Exception: + logger = logging.getLogger(__name__) + logger.exception(f"`NonBlockingIO` thread for {_path} failed to join.") + success = False + return success + + def _close_thread_pool(self) -> bool: + """ + Closes the ThreadPool. + """ + try: + self._pool.shutdown() + except Exception: + logger = logging.getLogger(__name__) + logger.exception("`NonBlockingIO` thread pool failed to close.") + return False + return True + + +# NOTE: We currently only support asynchronous writes (not reads). +class NonBlockingIO(io.IOBase): + def __init__( + self, + notify_manager: Callable[[Callable[[], None]], None], + io_obj: Union[IO[str], IO[bytes]], + callback_after_file_close: Optional[Callable[[None], None]] = None, + ) -> None: + """ + Returned to the user on an `opena` call. Uses a Queue to manage the + IO jobs that need to be run to ensure order preservation and a + polling Thread that checks the Queue. Implementation for these are + lifted to `NonBlockingIOManager` since `NonBlockingIO` closes upon + leaving the context block. + NOTE: Writes to the same path are serialized so they are written in + the same order as they were called but writes to distinct paths can + happen concurrently. + Args: + notify_manager (Callable): a callback function passed in from the + `NonBlockingIOManager` so that all IO jobs can be stored in + the manager. It takes in a single argument, namely another + callable function. + Example usage: + ``` + notify_manager(lambda: file.write(data)) + notify_manager(lambda: file.close()) + ``` + Here, we tell `NonBlockingIOManager` to add a write callable + to the path's Queue, and then to add a close callable to the + path's Queue. The path's polling Thread then executes the write + callable, waits for it to finish, and then executes the close + callable. Using `lambda` allows us to pass callables to the + manager. + io_obj (IO): a reference to the IO object returned by the + `PathHandler._open` function. + callback_after_file_close (Callable): An optional argument that can + be passed to perform operations that depend on the asynchronous + writes being completed. The file is first written to the local + disk and then the callback is executed. + """ + super().__init__() + self._notify_manager = notify_manager + self._io = io_obj + self._callback_after_file_close = callback_after_file_close + + self._close_called = False + + def readable(self) -> bool: + return False + + def writable(self) -> bool: + return True + + def seekable(self) -> bool: + return True + + def write(self, b: Union[bytes, bytearray]) -> None: + """ + Called on `f.write()`. Gives the manager the write job to call. + """ + self._notify_manager(lambda: self._io.write(b)) + + def seek(self, offset: int, whence: int = 0) -> int: + """ + Called on `f.seek()`. + """ + self._notify_manager(lambda: self._io.seek(offset, whence)) + + def tell(self) -> int: + """ + Called on `f.tell()`. + """ + raise ValueError("ioPath async writes does not support `tell` calls.") + + def truncate(self, size: int = None) -> int: + """ + Called on `f.truncate()`. + """ + self._notify_manager(lambda: self._io.truncate(size)) + + def close(self) -> None: + """ + Called on `f.close()` or automatically by the context manager. + We add the `close` call to the file's queue to make sure that + the file is not closed before all of the write jobs are complete. + """ + # `ThreadPool` first closes the file and then executes the callback. + # We only execute the callback once even if there are multiple + # `f.close` calls. + self._notify_manager(lambda: self._io.close()) + if not self._close_called and self._callback_after_file_close: + self._notify_manager(self._callback_after_file_close) + self._close_called = True + + +# NOTE: To use this class, use `buffered=True` in `NonBlockingIOManager`. +# NOTE: This class expects the IO mode to be buffered. +class NonBlockingBufferedIO(io.IOBase): + MAX_BUFFER_BYTES = 10 * 1024 * 1024 # 10 MiB + + def __init__( + self, + notify_manager: Callable[[Callable[[], None]], None], + io_obj: Union[IO[str], IO[bytes]], + callback_after_file_close: Optional[Callable[[None], None]] = None, + buffering: int = -1, + ) -> None: + """ + Buffered version of `NonBlockingIO`. All write data is stored in an + IO buffer until the buffer is full, or `flush` or `close` is called. + Args: + Same as `NonBlockingIO` args. + buffering (int): An optional argument to set the buffer size for + buffered asynchronous writing. + """ + super().__init__() + self._notify_manager = notify_manager + self._io = io_obj + self._callback_after_file_close = callback_after_file_close + + self._buffers = [io.BytesIO()] + self._buffer_size = buffering if buffering > 0 else self.MAX_BUFFER_BYTES + self._close_called = False + + def readable(self) -> bool: + return False + + def writable(self) -> bool: + return True + + def seekable(self) -> bool: + return False + + def write(self, b: Union[bytes, bytearray]) -> None: + """ + Called on `f.write()`. Gives the manager the write job to call. + """ + buffer = self._buffers[-1] + with memoryview(b) as view: + buffer.write(view) + if buffer.tell() < self._buffer_size: + return + self.flush() + + def close(self) -> None: + """ + Called on `f.close()` or automatically by the context manager. + We add the `close` call to the file's queue to make sure that + the file is not closed before all of the write jobs are complete. + """ + self.flush() + # Close the last buffer created by `flush`. + self._notify_manager(lambda: self._buffers[-1].close()) + # `ThreadPool` first closes the file and then executes the callback. + self._notify_manager(lambda: self._io.close()) + if not self._close_called and self._callback_after_file_close: + self._notify_manager(self._callback_after_file_close) + self._close_called = True + + def flush(self) -> None: + """ + Called on `f.write()` if the buffer is filled (or overfilled). Can + also be explicitly called by user. + NOTE: Buffering is used in a strict manner. Any buffer that exceeds + `self._buffer_size` will be broken into multiple write jobs where + each has a write call with `self._buffer_size` size. + """ + buffer = self._buffers[-1] + if buffer.tell() == 0: + return + pos = 0 + total_size = buffer.seek(0, io.SEEK_END) + view = buffer.getbuffer() + # Chunk the buffer in case it is larger than the buffer size. + while pos < total_size: + item = view[pos : pos + self._buffer_size] + # `item=item` is needed due to Python's late binding closures. + self._notify_manager(lambda item=item: self._io.write(item)) + pos += self._buffer_size + # Close buffer immediately after being written to file and create + # a new buffer. + self._notify_manager(lambda: buffer.close()) + self._buffers.append(io.BytesIO()) diff --git a/libai/utils/timer.py b/libai/utils/timer.py new file mode 100644 index 0000000000000000000000000000000000000000..d6e2057a7319aebab360a05c460a2afbd06491a7 --- /dev/null +++ b/libai/utils/timer.py @@ -0,0 +1,86 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from time import perf_counter +from typing import Optional + +# -------------------------------------------------------- +# References: +# https://github.com/facebookresearch/fvcore/blob/main/fvcore/common/timer.py +# -------------------------------------------------------- + + +class Timer: + """ + A timer which computes the time elapsed since the start/reset of the timer. + """ + + def __init__(self): + self.reset() + + def reset(self): + """ + Reset the timer. + """ + self._start = perf_counter() + self._paused: Optional[float] = None + self._total_paused = 0 + self._count_start = 1 + + def pause(self): + """ + Pause the timer. + """ + if self._paused is not None: + raise ValueError("Trying to pause a Timer that is already paused!") + self._paused = perf_counter() + + def is_paused(self) -> bool: + """ + Returns: + bool: whether the timer is currently paused + """ + return self._paused is not None + + def resume(self): + """ + Resume the timer. + """ + if self._paused is None: + raise ValueError("Trying to resume a Timer that is not paused!") + self._total_paused += perf_counter() - self._paused + self._paused = None + self._count_start += 1 + + def seconds(self) -> float: + """ + Returns: + (float): the total number of seconds since the start/reset of the + timer, excluding the time when the timer is paused. + """ + if self._paused is not None: + end_time: float = self._paused # type: ignore + else: + end_time = perf_counter() + return end_time - self._start - self._total_paused + + def avg_seconds(self) -> float: + """ + Returns: + (float): the average number of seconds between every start/reset and + pause. + """ + return self.seconds() / self._count_start diff --git a/libai/version.py b/libai/version.py new file mode 100644 index 0000000000000000000000000000000000000000..7578d9be94fd819102aab25944868f9467d302d6 --- /dev/null +++ b/libai/version.py @@ -0,0 +1,2 @@ +__version__ = '0.2.0' +git_version = 'Unknown' diff --git a/model.properties b/model.properties new file mode 100644 index 0000000000000000000000000000000000000000..5355535cda1496ca2d9d82fe4c8274cd7d6a82b3 --- /dev/null +++ b/model.properties @@ -0,0 +1,8 @@ +# 模型名称 +modelName=Bert-Large +# 模型描述 +modelDescription=Bert-Large +# 应用场景(多个标签以英文逗号分割) +appScenario=智能聊天助手 +# 框架类型(多个标签以英文逗号分割) +frameType=OneFlow,Libai diff --git a/nlp_data/bert-base-chinese-vocab.txt b/nlp_data/bert-base-chinese-vocab.txt new file mode 100644 index 0000000000000000000000000000000000000000..ca4f9781030019ab9b253c6dcb8c7878b6dc87a5 --- /dev/null +++ b/nlp_data/bert-base-chinese-vocab.txt @@ -0,0 +1,21128 @@ +[PAD] +[unused1] +[unused2] +[unused3] +[unused4] +[unused5] +[unused6] +[unused7] +[unused8] +[unused9] +[unused10] +[unused11] +[unused12] +[unused13] +[unused14] +[unused15] +[unused16] +[unused17] +[unused18] +[unused19] +[unused20] +[unused21] +[unused22] +[unused23] +[unused24] +[unused25] +[unused26] +[unused27] +[unused28] +[unused29] +[unused30] +[unused31] +[unused32] +[unused33] +[unused34] +[unused35] +[unused36] +[unused37] +[unused38] +[unused39] +[unused40] +[unused41] +[unused42] +[unused43] +[unused44] +[unused45] +[unused46] +[unused47] +[unused48] +[unused49] +[unused50] +[unused51] +[unused52] +[unused53] +[unused54] +[unused55] +[unused56] +[unused57] +[unused58] +[unused59] +[unused60] +[unused61] +[unused62] +[unused63] +[unused64] +[unused65] +[unused66] +[unused67] +[unused68] +[unused69] +[unused70] +[unused71] +[unused72] +[unused73] +[unused74] +[unused75] +[unused76] +[unused77] +[unused78] +[unused79] +[unused80] +[unused81] +[unused82] +[unused83] +[unused84] +[unused85] +[unused86] +[unused87] +[unused88] +[unused89] +[unused90] +[unused91] +[unused92] +[unused93] +[unused94] +[unused95] +[unused96] +[unused97] +[unused98] +[unused99] +[UNK] +[CLS] +[SEP] +[MASK] + + +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +[ +\ +] +^ +_ +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ +£ +¤ +¥ +§ +© +« +® +° +± +² +³ +µ +· +¹ +º +» +¼ +× +ß +æ +÷ +ø +đ +ŋ +ɔ +ə +ɡ +ʰ +ˇ +ˈ +ˊ +ˋ +ˍ +ː +˙ +˚ +ˢ +α +β +γ +δ +ε +η +θ +ι +κ +λ +μ +ν +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +а +б +в +г +д +е +ж +з +и +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +ы +ь +я +і +ا +ب +ة +ت +د +ر +س +ع +ل +م +ن +ه +و +ي +۩ +ก +ง +น +ม +ย +ร +อ +า +เ +๑ +་ +ღ +ᄀ +ᄁ +ᄂ +ᄃ +ᄅ +ᄆ +ᄇ +ᄈ +ᄉ +ᄋ +ᄌ +ᄎ +ᄏ +ᄐ +ᄑ +ᄒ +ᅡ +ᅢ +ᅣ +ᅥ +ᅦ +ᅧ +ᅨ +ᅩ +ᅪ +ᅬ +ᅭ +ᅮ +ᅯ +ᅲ +ᅳ +ᅴ +ᅵ +ᆨ +ᆫ +ᆯ +ᆷ +ᆸ +ᆺ +ᆻ +ᆼ +ᗜ +ᵃ +ᵉ +ᵍ +ᵏ +ᵐ +ᵒ +ᵘ +‖ +„ +† +• +‥ +‧ +
 +‰ +′ +″ +‹ +› +※ +‿ +⁄ +ⁱ +⁺ +ⁿ +₁ +₂ +₃ +₄ +€ +℃ +№ +™ +ⅰ +ⅱ +ⅲ +ⅳ +ⅴ +← +↑ +→ +↓ +↔ +↗ +↘ +⇒ +∀ +− +∕ +∙ +√ +∞ +∟ +∠ +∣ +∥ +∩ +∮ +∶ +∼ +∽ +≈ +≒ +≡ +≤ +≥ +≦ +≧ +≪ +≫ +⊙ +⋅ +⋈ +⋯ +⌒ +① +② +③ +④ +⑤ +⑥ +⑦ +⑧ +⑨ +⑩ +⑴ +⑵ +⑶ +⑷ +⑸ +⒈ +⒉ +⒊ +⒋ +ⓒ +ⓔ +ⓘ +─ +━ +│ +┃ +┅ +┆ +┊ +┌ +└ +├ +┣ +═ +║ +╚ +╞ +╠ +╭ +╮ +╯ +╰ +╱ +╳ +▂ +▃ +▅ +▇ +█ +▉ +▋ +▌ +▍ +▎ +■ +□ +▪ +▫ +▬ +▲ +△ +▶ +► +▼ +▽ +◆ +◇ +○ +◎ +● +◕ +◠ +◢ +◤ +☀ +★ +☆ +☕ +☞ +☺ +☼ +♀ +♂ +♠ +♡ +♣ +♥ +♦ +♪ +♫ +♬ +✈ +✔ +✕ +✖ +✦ +✨ +✪ +✰ +✿ +❀ +❤ +➜ +➤ +⦿ +、 +。 +〃 +々 +〇 +〈 +〉 +《 +》 +「 +」 +『 +』 +【 +】 +〓 +〔 +〕 +〖 +〗 +〜 +〝 +〞 +ぁ +あ +ぃ +い +う +ぇ +え +お +か +き +く +け +こ +さ +し +す +せ +そ +た +ち +っ +つ +て +と +な +に +ぬ +ね +の +は +ひ +ふ +へ +ほ +ま +み +む +め +も +ゃ +や +ゅ +ゆ +ょ +よ +ら +り +る +れ +ろ +わ +を +ん +゜ +ゝ +ァ +ア +ィ +イ +ゥ +ウ +ェ +エ +ォ +オ +カ +キ +ク +ケ +コ +サ +シ +ス +セ +ソ +タ +チ +ッ +ツ +テ +ト +ナ +ニ +ヌ +ネ +ノ +ハ +ヒ +フ +ヘ +ホ +マ +ミ +ム +メ +モ +ャ +ヤ +ュ +ユ +ョ +ヨ +ラ +リ +ル +レ +ロ +ワ +ヲ +ン +ヶ +・ +ー +ヽ +ㄅ +ㄆ +ㄇ +ㄉ +ㄋ +ㄌ +ㄍ +ㄎ +ㄏ +ㄒ +ㄚ +ㄛ +ㄞ +ㄟ +ㄢ +ㄤ +ㄥ +ㄧ +ㄨ +ㆍ +㈦ +㊣ +㎡ +㗎 +一 +丁 +七 +万 +丈 +三 +上 +下 +不 +与 +丐 +丑 +专 +且 +丕 +世 +丘 +丙 +业 +丛 +东 +丝 +丞 +丟 +両 +丢 +两 +严 +並 +丧 +丨 +个 +丫 +中 +丰 +串 +临 +丶 +丸 +丹 +为 +主 +丼 +丽 +举 +丿 +乂 +乃 +久 +么 +义 +之 +乌 +乍 +乎 +乏 +乐 +乒 +乓 +乔 +乖 +乗 +乘 +乙 +乜 +九 +乞 +也 +习 +乡 +书 +乩 +买 +乱 +乳 +乾 +亀 +亂 +了 +予 +争 +事 +二 +于 +亏 +云 +互 +五 +井 +亘 +亙 +亚 +些 +亜 +亞 +亟 +亡 +亢 +交 +亥 +亦 +产 +亨 +亩 +享 +京 +亭 +亮 +亲 +亳 +亵 +人 +亿 +什 +仁 +仃 +仄 +仅 +仆 +仇 +今 +介 +仍 +从 +仏 +仑 +仓 +仔 +仕 +他 +仗 +付 +仙 +仝 +仞 +仟 +代 +令 +以 +仨 +仪 +们 +仮 +仰 +仲 +件 +价 +任 +份 +仿 +企 +伉 +伊 +伍 +伎 +伏 +伐 +休 +伕 +众 +优 +伙 +会 +伝 +伞 +伟 +传 +伢 +伤 +伦 +伪 +伫 +伯 +估 +伴 +伶 +伸 +伺 +似 +伽 +佃 +但 +佇 +佈 +位 +低 +住 +佐 +佑 +体 +佔 +何 +佗 +佘 +余 +佚 +佛 +作 +佝 +佞 +佟 +你 +佢 +佣 +佤 +佥 +佩 +佬 +佯 +佰 +佳 +併 +佶 +佻 +佼 +使 +侃 +侄 +來 +侈 +例 +侍 +侏 +侑 +侖 +侗 +供 +依 +侠 +価 +侣 +侥 +侦 +侧 +侨 +侬 +侮 +侯 +侵 +侶 +侷 +便 +係 +促 +俄 +俊 +俎 +俏 +俐 +俑 +俗 +俘 +俚 +保 +俞 +俟 +俠 +信 +俨 +俩 +俪 +俬 +俭 +修 +俯 +俱 +俳 +俸 +俺 +俾 +倆 +倉 +個 +倌 +倍 +倏 +們 +倒 +倔 +倖 +倘 +候 +倚 +倜 +借 +倡 +値 +倦 +倩 +倪 +倫 +倬 +倭 +倶 +债 +值 +倾 +偃 +假 +偈 +偉 +偌 +偎 +偏 +偕 +做 +停 +健 +側 +偵 +偶 +偷 +偻 +偽 +偿 +傀 +傅 +傍 +傑 +傘 +備 +傚 +傢 +傣 +傥 +储 +傩 +催 +傭 +傲 +傳 +債 +傷 +傻 +傾 +僅 +働 +像 +僑 +僕 +僖 +僚 +僥 +僧 +僭 +僮 +僱 +僵 +價 +僻 +儀 +儂 +億 +儆 +儉 +儋 +儒 +儕 +儘 +償 +儡 +優 +儲 +儷 +儼 +儿 +兀 +允 +元 +兄 +充 +兆 +兇 +先 +光 +克 +兌 +免 +児 +兑 +兒 +兔 +兖 +党 +兜 +兢 +入 +內 +全 +兩 +八 +公 +六 +兮 +兰 +共 +兲 +关 +兴 +兵 +其 +具 +典 +兹 +养 +兼 +兽 +冀 +内 +円 +冇 +冈 +冉 +冊 +册 +再 +冏 +冒 +冕 +冗 +写 +军 +农 +冠 +冢 +冤 +冥 +冨 +冪 +冬 +冯 +冰 +冲 +决 +况 +冶 +冷 +冻 +冼 +冽 +冾 +净 +凄 +准 +凇 +凈 +凉 +凋 +凌 +凍 +减 +凑 +凛 +凜 +凝 +几 +凡 +凤 +処 +凪 +凭 +凯 +凰 +凱 +凳 +凶 +凸 +凹 +出 +击 +函 +凿 +刀 +刁 +刃 +分 +切 +刈 +刊 +刍 +刎 +刑 +划 +列 +刘 +则 +刚 +创 +初 +删 +判 +別 +刨 +利 +刪 +别 +刮 +到 +制 +刷 +券 +刹 +刺 +刻 +刽 +剁 +剂 +剃 +則 +剉 +削 +剋 +剌 +前 +剎 +剐 +剑 +剔 +剖 +剛 +剜 +剝 +剣 +剤 +剥 +剧 +剩 +剪 +副 +割 +創 +剷 +剽 +剿 +劃 +劇 +劈 +劉 +劊 +劍 +劏 +劑 +力 +劝 +办 +功 +加 +务 +劣 +动 +助 +努 +劫 +劭 +励 +劲 +劳 +労 +劵 +効 +劾 +势 +勁 +勃 +勇 +勉 +勋 +勐 +勒 +動 +勖 +勘 +務 +勛 +勝 +勞 +募 +勢 +勤 +勧 +勳 +勵 +勸 +勺 +勻 +勾 +勿 +匀 +包 +匆 +匈 +匍 +匐 +匕 +化 +北 +匙 +匝 +匠 +匡 +匣 +匪 +匮 +匯 +匱 +匹 +区 +医 +匾 +匿 +區 +十 +千 +卅 +升 +午 +卉 +半 +卍 +华 +协 +卑 +卒 +卓 +協 +单 +卖 +南 +単 +博 +卜 +卞 +卟 +占 +卡 +卢 +卤 +卦 +卧 +卫 +卮 +卯 +印 +危 +即 +却 +卵 +卷 +卸 +卻 +卿 +厂 +厄 +厅 +历 +厉 +压 +厌 +厕 +厘 +厚 +厝 +原 +厢 +厥 +厦 +厨 +厩 +厭 +厮 +厲 +厳 +去 +县 +叁 +参 +參 +又 +叉 +及 +友 +双 +反 +収 +发 +叔 +取 +受 +变 +叙 +叛 +叟 +叠 +叡 +叢 +口 +古 +句 +另 +叨 +叩 +只 +叫 +召 +叭 +叮 +可 +台 +叱 +史 +右 +叵 +叶 +号 +司 +叹 +叻 +叼 +叽 +吁 +吃 +各 +吆 +合 +吉 +吊 +吋 +同 +名 +后 +吏 +吐 +向 +吒 +吓 +吕 +吖 +吗 +君 +吝 +吞 +吟 +吠 +吡 +否 +吧 +吨 +吩 +含 +听 +吭 +吮 +启 +吱 +吳 +吴 +吵 +吶 +吸 +吹 +吻 +吼 +吽 +吾 +呀 +呂 +呃 +呆 +呈 +告 +呋 +呎 +呐 +呓 +呕 +呗 +员 +呛 +呜 +呢 +呤 +呦 +周 +呱 +呲 +味 +呵 +呷 +呸 +呻 +呼 +命 +咀 +咁 +咂 +咄 +咆 +咋 +和 +咎 +咏 +咐 +咒 +咔 +咕 +咖 +咗 +咘 +咙 +咚 +咛 +咣 +咤 +咦 +咧 +咨 +咩 +咪 +咫 +咬 +咭 +咯 +咱 +咲 +咳 +咸 +咻 +咽 +咿 +哀 +品 +哂 +哄 +哆 +哇 +哈 +哉 +哋 +哌 +响 +哎 +哏 +哐 +哑 +哒 +哔 +哗 +哟 +員 +哥 +哦 +哧 +哨 +哩 +哪 +哭 +哮 +哲 +哺 +哼 +哽 +唁 +唄 +唆 +唇 +唉 +唏 +唐 +唑 +唔 +唠 +唤 +唧 +唬 +售 +唯 +唰 +唱 +唳 +唷 +唸 +唾 +啃 +啄 +商 +啉 +啊 +問 +啓 +啕 +啖 +啜 +啞 +啟 +啡 +啤 +啥 +啦 +啧 +啪 +啫 +啬 +啮 +啰 +啱 +啲 +啵 +啶 +啷 +啸 +啻 +啼 +啾 +喀 +喂 +喃 +善 +喆 +喇 +喉 +喊 +喋 +喎 +喏 +喔 +喘 +喙 +喚 +喜 +喝 +喟 +喧 +喪 +喫 +喬 +單 +喰 +喱 +喲 +喳 +喵 +営 +喷 +喹 +喺 +喻 +喽 +嗅 +嗆 +嗇 +嗎 +嗑 +嗒 +嗓 +嗔 +嗖 +嗚 +嗜 +嗝 +嗟 +嗡 +嗣 +嗤 +嗦 +嗨 +嗪 +嗬 +嗯 +嗰 +嗲 +嗳 +嗶 +嗷 +嗽 +嘀 +嘅 +嘆 +嘈 +嘉 +嘌 +嘍 +嘎 +嘔 +嘖 +嘗 +嘘 +嘚 +嘛 +嘜 +嘞 +嘟 +嘢 +嘣 +嘤 +嘧 +嘩 +嘭 +嘮 +嘯 +嘰 +嘱 +嘲 +嘴 +嘶 +嘸 +嘹 +嘻 +嘿 +噁 +噌 +噎 +噓 +噔 +噗 +噙 +噜 +噠 +噢 +噤 +器 +噩 +噪 +噬 +噱 +噴 +噶 +噸 +噹 +噻 +噼 +嚀 +嚇 +嚎 +嚏 +嚐 +嚓 +嚕 +嚟 +嚣 +嚥 +嚨 +嚮 +嚴 +嚷 +嚼 +囂 +囉 +囊 +囍 +囑 +囔 +囗 +囚 +四 +囝 +回 +囟 +因 +囡 +团 +団 +囤 +囧 +囪 +囫 +园 +困 +囱 +囲 +図 +围 +囹 +固 +国 +图 +囿 +圃 +圄 +圆 +圈 +國 +圍 +圏 +園 +圓 +圖 +團 +圜 +土 +圣 +圧 +在 +圩 +圭 +地 +圳 +场 +圻 +圾 +址 +坂 +均 +坊 +坍 +坎 +坏 +坐 +坑 +块 +坚 +坛 +坝 +坞 +坟 +坠 +坡 +坤 +坦 +坨 +坪 +坯 +坳 +坵 +坷 +垂 +垃 +垄 +型 +垒 +垚 +垛 +垠 +垢 +垣 +垦 +垩 +垫 +垭 +垮 +垵 +埂 +埃 +埋 +城 +埔 +埕 +埗 +域 +埠 +埤 +埵 +執 +埸 +培 +基 +埼 +堀 +堂 +堃 +堅 +堆 +堇 +堑 +堕 +堙 +堡 +堤 +堪 +堯 +堰 +報 +場 +堵 +堺 +堿 +塊 +塌 +塑 +塔 +塗 +塘 +塚 +塞 +塢 +塩 +填 +塬 +塭 +塵 +塾 +墀 +境 +墅 +墉 +墊 +墒 +墓 +増 +墘 +墙 +墜 +增 +墟 +墨 +墩 +墮 +墳 +墻 +墾 +壁 +壅 +壆 +壇 +壊 +壑 +壓 +壕 +壘 +壞 +壟 +壢 +壤 +壩 +士 +壬 +壮 +壯 +声 +売 +壳 +壶 +壹 +壺 +壽 +处 +备 +変 +复 +夏 +夔 +夕 +外 +夙 +多 +夜 +够 +夠 +夢 +夥 +大 +天 +太 +夫 +夭 +央 +夯 +失 +头 +夷 +夸 +夹 +夺 +夾 +奂 +奄 +奇 +奈 +奉 +奋 +奎 +奏 +奐 +契 +奔 +奕 +奖 +套 +奘 +奚 +奠 +奢 +奥 +奧 +奪 +奬 +奮 +女 +奴 +奶 +奸 +她 +好 +如 +妃 +妄 +妆 +妇 +妈 +妊 +妍 +妒 +妓 +妖 +妘 +妙 +妝 +妞 +妣 +妤 +妥 +妨 +妩 +妪 +妮 +妲 +妳 +妹 +妻 +妾 +姆 +姉 +姊 +始 +姍 +姐 +姑 +姒 +姓 +委 +姗 +姚 +姜 +姝 +姣 +姥 +姦 +姨 +姪 +姫 +姬 +姹 +姻 +姿 +威 +娃 +娄 +娅 +娆 +娇 +娉 +娑 +娓 +娘 +娛 +娜 +娟 +娠 +娣 +娥 +娩 +娱 +娲 +娴 +娶 +娼 +婀 +婁 +婆 +婉 +婊 +婕 +婚 +婢 +婦 +婧 +婪 +婭 +婴 +婵 +婶 +婷 +婺 +婿 +媒 +媚 +媛 +媞 +媧 +媲 +媳 +媽 +媾 +嫁 +嫂 +嫉 +嫌 +嫑 +嫔 +嫖 +嫘 +嫚 +嫡 +嫣 +嫦 +嫩 +嫲 +嫵 +嫻 +嬅 +嬉 +嬌 +嬗 +嬛 +嬢 +嬤 +嬪 +嬰 +嬴 +嬷 +嬸 +嬿 +孀 +孃 +子 +孑 +孔 +孕 +孖 +字 +存 +孙 +孚 +孛 +孜 +孝 +孟 +孢 +季 +孤 +学 +孩 +孪 +孫 +孬 +孰 +孱 +孳 +孵 +學 +孺 +孽 +孿 +宁 +它 +宅 +宇 +守 +安 +宋 +完 +宏 +宓 +宕 +宗 +官 +宙 +定 +宛 +宜 +宝 +实 +実 +宠 +审 +客 +宣 +室 +宥 +宦 +宪 +宫 +宮 +宰 +害 +宴 +宵 +家 +宸 +容 +宽 +宾 +宿 +寂 +寄 +寅 +密 +寇 +富 +寐 +寒 +寓 +寛 +寝 +寞 +察 +寡 +寢 +寥 +實 +寧 +寨 +審 +寫 +寬 +寮 +寰 +寵 +寶 +寸 +对 +寺 +寻 +导 +対 +寿 +封 +専 +射 +将 +將 +專 +尉 +尊 +尋 +對 +導 +小 +少 +尔 +尕 +尖 +尘 +尚 +尝 +尤 +尧 +尬 +就 +尴 +尷 +尸 +尹 +尺 +尻 +尼 +尽 +尾 +尿 +局 +屁 +层 +屄 +居 +屆 +屈 +屉 +届 +屋 +屌 +屍 +屎 +屏 +屐 +屑 +展 +屜 +属 +屠 +屡 +屢 +層 +履 +屬 +屯 +山 +屹 +屿 +岀 +岁 +岂 +岌 +岐 +岑 +岔 +岖 +岗 +岘 +岙 +岚 +岛 +岡 +岩 +岫 +岬 +岭 +岱 +岳 +岷 +岸 +峇 +峋 +峒 +峙 +峡 +峤 +峥 +峦 +峨 +峪 +峭 +峯 +峰 +峴 +島 +峻 +峽 +崁 +崂 +崆 +崇 +崎 +崑 +崔 +崖 +崗 +崙 +崛 +崧 +崩 +崭 +崴 +崽 +嵇 +嵊 +嵋 +嵌 +嵐 +嵘 +嵩 +嵬 +嵯 +嶂 +嶄 +嶇 +嶋 +嶙 +嶺 +嶼 +嶽 +巅 +巍 +巒 +巔 +巖 +川 +州 +巡 +巢 +工 +左 +巧 +巨 +巩 +巫 +差 +己 +已 +巳 +巴 +巷 +巻 +巽 +巾 +巿 +币 +市 +布 +帅 +帆 +师 +希 +帐 +帑 +帕 +帖 +帘 +帚 +帛 +帜 +帝 +帥 +带 +帧 +師 +席 +帮 +帯 +帰 +帳 +帶 +帷 +常 +帼 +帽 +幀 +幂 +幄 +幅 +幌 +幔 +幕 +幟 +幡 +幢 +幣 +幫 +干 +平 +年 +并 +幸 +幹 +幺 +幻 +幼 +幽 +幾 +广 +庁 +広 +庄 +庆 +庇 +床 +序 +庐 +库 +应 +底 +庖 +店 +庙 +庚 +府 +庞 +废 +庠 +度 +座 +庫 +庭 +庵 +庶 +康 +庸 +庹 +庾 +廁 +廂 +廃 +廈 +廉 +廊 +廓 +廖 +廚 +廝 +廟 +廠 +廢 +廣 +廬 +廳 +延 +廷 +建 +廿 +开 +弁 +异 +弃 +弄 +弈 +弊 +弋 +式 +弑 +弒 +弓 +弔 +引 +弗 +弘 +弛 +弟 +张 +弥 +弦 +弧 +弩 +弭 +弯 +弱 +張 +強 +弹 +强 +弼 +弾 +彅 +彆 +彈 +彌 +彎 +归 +当 +录 +彗 +彙 +彝 +形 +彤 +彥 +彦 +彧 +彩 +彪 +彫 +彬 +彭 +彰 +影 +彷 +役 +彻 +彼 +彿 +往 +征 +径 +待 +徇 +很 +徉 +徊 +律 +後 +徐 +徑 +徒 +従 +徕 +得 +徘 +徙 +徜 +從 +徠 +御 +徨 +復 +循 +徬 +微 +徳 +徴 +徵 +德 +徹 +徼 +徽 +心 +必 +忆 +忌 +忍 +忏 +忐 +忑 +忒 +忖 +志 +忘 +忙 +応 +忠 +忡 +忤 +忧 +忪 +快 +忱 +念 +忻 +忽 +忿 +怀 +态 +怂 +怅 +怆 +怎 +怏 +怒 +怔 +怕 +怖 +怙 +怜 +思 +怠 +怡 +急 +怦 +性 +怨 +怪 +怯 +怵 +总 +怼 +恁 +恃 +恆 +恋 +恍 +恐 +恒 +恕 +恙 +恚 +恢 +恣 +恤 +恥 +恨 +恩 +恪 +恫 +恬 +恭 +息 +恰 +恳 +恵 +恶 +恸 +恺 +恻 +恼 +恿 +悄 +悅 +悉 +悌 +悍 +悔 +悖 +悚 +悟 +悠 +患 +悦 +您 +悩 +悪 +悬 +悯 +悱 +悲 +悴 +悵 +悶 +悸 +悻 +悼 +悽 +情 +惆 +惇 +惊 +惋 +惑 +惕 +惘 +惚 +惜 +惟 +惠 +惡 +惦 +惧 +惨 +惩 +惫 +惬 +惭 +惮 +惯 +惰 +惱 +想 +惴 +惶 +惹 +惺 +愁 +愆 +愈 +愉 +愍 +意 +愕 +愚 +愛 +愜 +感 +愣 +愤 +愧 +愫 +愷 +愿 +慄 +慈 +態 +慌 +慎 +慑 +慕 +慘 +慚 +慟 +慢 +慣 +慧 +慨 +慫 +慮 +慰 +慳 +慵 +慶 +慷 +慾 +憂 +憊 +憋 +憎 +憐 +憑 +憔 +憚 +憤 +憧 +憨 +憩 +憫 +憬 +憲 +憶 +憾 +懂 +懇 +懈 +應 +懊 +懋 +懑 +懒 +懦 +懲 +懵 +懶 +懷 +懸 +懺 +懼 +懾 +懿 +戀 +戈 +戊 +戌 +戍 +戎 +戏 +成 +我 +戒 +戕 +或 +战 +戚 +戛 +戟 +戡 +戦 +截 +戬 +戮 +戰 +戲 +戳 +戴 +戶 +户 +戸 +戻 +戾 +房 +所 +扁 +扇 +扈 +扉 +手 +才 +扎 +扑 +扒 +打 +扔 +払 +托 +扛 +扣 +扦 +执 +扩 +扪 +扫 +扬 +扭 +扮 +扯 +扰 +扱 +扳 +扶 +批 +扼 +找 +承 +技 +抄 +抉 +把 +抑 +抒 +抓 +投 +抖 +抗 +折 +抚 +抛 +抜 +択 +抟 +抠 +抡 +抢 +护 +报 +抨 +披 +抬 +抱 +抵 +抹 +押 +抽 +抿 +拂 +拄 +担 +拆 +拇 +拈 +拉 +拋 +拌 +拍 +拎 +拐 +拒 +拓 +拔 +拖 +拗 +拘 +拙 +拚 +招 +拜 +拟 +拡 +拢 +拣 +拥 +拦 +拧 +拨 +择 +括 +拭 +拮 +拯 +拱 +拳 +拴 +拷 +拼 +拽 +拾 +拿 +持 +挂 +指 +挈 +按 +挎 +挑 +挖 +挙 +挚 +挛 +挝 +挞 +挟 +挠 +挡 +挣 +挤 +挥 +挨 +挪 +挫 +振 +挲 +挹 +挺 +挽 +挾 +捂 +捅 +捆 +捉 +捋 +捌 +捍 +捎 +捏 +捐 +捕 +捞 +损 +捡 +换 +捣 +捧 +捨 +捩 +据 +捱 +捲 +捶 +捷 +捺 +捻 +掀 +掂 +掃 +掇 +授 +掉 +掌 +掏 +掐 +排 +掖 +掘 +掙 +掛 +掠 +採 +探 +掣 +接 +控 +推 +掩 +措 +掬 +掰 +掲 +掳 +掴 +掷 +掸 +掺 +揀 +揃 +揄 +揆 +揉 +揍 +描 +提 +插 +揖 +揚 +換 +握 +揣 +揩 +揪 +揭 +揮 +援 +揶 +揸 +揹 +揽 +搀 +搁 +搂 +搅 +損 +搏 +搐 +搓 +搔 +搖 +搗 +搜 +搞 +搡 +搪 +搬 +搭 +搵 +搶 +携 +搽 +摀 +摁 +摄 +摆 +摇 +摈 +摊 +摒 +摔 +摘 +摞 +摟 +摧 +摩 +摯 +摳 +摸 +摹 +摺 +摻 +撂 +撃 +撅 +撇 +撈 +撐 +撑 +撒 +撓 +撕 +撚 +撞 +撤 +撥 +撩 +撫 +撬 +播 +撮 +撰 +撲 +撵 +撷 +撸 +撻 +撼 +撿 +擀 +擁 +擂 +擄 +擅 +擇 +擊 +擋 +操 +擎 +擒 +擔 +擘 +據 +擞 +擠 +擡 +擢 +擦 +擬 +擰 +擱 +擲 +擴 +擷 +擺 +擼 +擾 +攀 +攏 +攒 +攔 +攘 +攙 +攜 +攝 +攞 +攢 +攣 +攤 +攥 +攪 +攫 +攬 +支 +收 +攸 +改 +攻 +放 +政 +故 +效 +敌 +敍 +敎 +敏 +救 +敕 +敖 +敗 +敘 +教 +敛 +敝 +敞 +敢 +散 +敦 +敬 +数 +敲 +整 +敵 +敷 +數 +斂 +斃 +文 +斋 +斌 +斎 +斐 +斑 +斓 +斗 +料 +斛 +斜 +斟 +斡 +斤 +斥 +斧 +斩 +斫 +斬 +断 +斯 +新 +斷 +方 +於 +施 +旁 +旃 +旅 +旋 +旌 +旎 +族 +旖 +旗 +无 +既 +日 +旦 +旧 +旨 +早 +旬 +旭 +旮 +旱 +时 +旷 +旺 +旻 +昀 +昂 +昆 +昇 +昉 +昊 +昌 +明 +昏 +易 +昔 +昕 +昙 +星 +映 +春 +昧 +昨 +昭 +是 +昱 +昴 +昵 +昶 +昼 +显 +晁 +時 +晃 +晉 +晋 +晌 +晏 +晒 +晓 +晔 +晕 +晖 +晗 +晚 +晝 +晞 +晟 +晤 +晦 +晨 +晩 +普 +景 +晰 +晴 +晶 +晷 +智 +晾 +暂 +暄 +暇 +暈 +暉 +暌 +暐 +暑 +暖 +暗 +暝 +暢 +暧 +暨 +暫 +暮 +暱 +暴 +暸 +暹 +曄 +曆 +曇 +曉 +曖 +曙 +曜 +曝 +曠 +曦 +曬 +曰 +曲 +曳 +更 +書 +曹 +曼 +曾 +替 +最 +會 +月 +有 +朋 +服 +朐 +朔 +朕 +朗 +望 +朝 +期 +朦 +朧 +木 +未 +末 +本 +札 +朮 +术 +朱 +朴 +朵 +机 +朽 +杀 +杂 +权 +杆 +杈 +杉 +李 +杏 +材 +村 +杓 +杖 +杜 +杞 +束 +杠 +条 +来 +杨 +杭 +杯 +杰 +東 +杳 +杵 +杷 +杼 +松 +板 +极 +构 +枇 +枉 +枋 +析 +枕 +林 +枚 +果 +枝 +枢 +枣 +枪 +枫 +枭 +枯 +枰 +枱 +枳 +架 +枷 +枸 +柄 +柏 +某 +柑 +柒 +染 +柔 +柘 +柚 +柜 +柞 +柠 +柢 +查 +柩 +柬 +柯 +柱 +柳 +柴 +柵 +査 +柿 +栀 +栃 +栄 +栅 +标 +栈 +栉 +栋 +栎 +栏 +树 +栓 +栖 +栗 +校 +栩 +株 +样 +核 +根 +格 +栽 +栾 +桀 +桁 +桂 +桃 +桅 +框 +案 +桉 +桌 +桎 +桐 +桑 +桓 +桔 +桜 +桠 +桡 +桢 +档 +桥 +桦 +桧 +桨 +桩 +桶 +桿 +梁 +梅 +梆 +梏 +梓 +梗 +條 +梟 +梢 +梦 +梧 +梨 +梭 +梯 +械 +梳 +梵 +梶 +检 +棂 +棄 +棉 +棋 +棍 +棒 +棕 +棗 +棘 +棚 +棟 +棠 +棣 +棧 +森 +棱 +棲 +棵 +棹 +棺 +椁 +椅 +椋 +植 +椎 +椒 +検 +椪 +椭 +椰 +椹 +椽 +椿 +楂 +楊 +楓 +楔 +楚 +楝 +楞 +楠 +楣 +楨 +楫 +業 +楮 +極 +楷 +楸 +楹 +楼 +楽 +概 +榄 +榆 +榈 +榉 +榔 +榕 +榖 +榛 +榜 +榨 +榫 +榭 +榮 +榱 +榴 +榷 +榻 +槁 +槃 +構 +槌 +槍 +槎 +槐 +槓 +様 +槛 +槟 +槤 +槭 +槲 +槳 +槻 +槽 +槿 +樁 +樂 +樊 +樑 +樓 +標 +樞 +樟 +模 +樣 +権 +横 +樫 +樯 +樱 +樵 +樸 +樹 +樺 +樽 +樾 +橄 +橇 +橋 +橐 +橘 +橙 +機 +橡 +橢 +橫 +橱 +橹 +橼 +檀 +檄 +檎 +檐 +檔 +檗 +檜 +檢 +檬 +檯 +檳 +檸 +檻 +櫃 +櫚 +櫛 +櫥 +櫸 +櫻 +欄 +權 +欒 +欖 +欠 +次 +欢 +欣 +欧 +欲 +欸 +欺 +欽 +款 +歆 +歇 +歉 +歌 +歎 +歐 +歓 +歙 +歛 +歡 +止 +正 +此 +步 +武 +歧 +歩 +歪 +歯 +歲 +歳 +歴 +歷 +歸 +歹 +死 +歼 +殁 +殃 +殆 +殇 +殉 +殊 +残 +殒 +殓 +殖 +殘 +殞 +殡 +殤 +殭 +殯 +殲 +殴 +段 +殷 +殺 +殼 +殿 +毀 +毁 +毂 +毅 +毆 +毋 +母 +毎 +每 +毒 +毓 +比 +毕 +毗 +毘 +毙 +毛 +毡 +毫 +毯 +毽 +氈 +氏 +氐 +民 +氓 +气 +氖 +気 +氙 +氛 +氟 +氡 +氢 +氣 +氤 +氦 +氧 +氨 +氪 +氫 +氮 +氯 +氰 +氲 +水 +氷 +永 +氹 +氾 +汀 +汁 +求 +汆 +汇 +汉 +汎 +汐 +汕 +汗 +汙 +汛 +汝 +汞 +江 +池 +污 +汤 +汨 +汩 +汪 +汰 +汲 +汴 +汶 +汹 +決 +汽 +汾 +沁 +沂 +沃 +沅 +沈 +沉 +沌 +沏 +沐 +沒 +沓 +沖 +沙 +沛 +沟 +没 +沢 +沣 +沥 +沦 +沧 +沪 +沫 +沭 +沮 +沱 +河 +沸 +油 +治 +沼 +沽 +沾 +沿 +況 +泄 +泉 +泊 +泌 +泓 +法 +泗 +泛 +泞 +泠 +泡 +波 +泣 +泥 +注 +泪 +泫 +泮 +泯 +泰 +泱 +泳 +泵 +泷 +泸 +泻 +泼 +泽 +泾 +洁 +洄 +洋 +洒 +洗 +洙 +洛 +洞 +津 +洩 +洪 +洮 +洱 +洲 +洵 +洶 +洸 +洹 +活 +洼 +洽 +派 +流 +浃 +浄 +浅 +浆 +浇 +浊 +测 +济 +浏 +浑 +浒 +浓 +浔 +浙 +浚 +浜 +浣 +浦 +浩 +浪 +浬 +浮 +浯 +浴 +海 +浸 +涂 +涅 +涇 +消 +涉 +涌 +涎 +涓 +涔 +涕 +涙 +涛 +涝 +涞 +涟 +涠 +涡 +涣 +涤 +润 +涧 +涨 +涩 +涪 +涮 +涯 +液 +涵 +涸 +涼 +涿 +淀 +淄 +淅 +淆 +淇 +淋 +淌 +淑 +淒 +淖 +淘 +淙 +淚 +淞 +淡 +淤 +淦 +淨 +淩 +淪 +淫 +淬 +淮 +深 +淳 +淵 +混 +淹 +淺 +添 +淼 +清 +済 +渉 +渊 +渋 +渍 +渎 +渐 +渔 +渗 +渙 +渚 +減 +渝 +渠 +渡 +渣 +渤 +渥 +渦 +温 +測 +渭 +港 +渲 +渴 +游 +渺 +渾 +湃 +湄 +湊 +湍 +湖 +湘 +湛 +湟 +湧 +湫 +湮 +湯 +湳 +湾 +湿 +満 +溃 +溅 +溉 +溏 +源 +準 +溜 +溝 +溟 +溢 +溥 +溧 +溪 +溫 +溯 +溱 +溴 +溶 +溺 +溼 +滁 +滂 +滄 +滅 +滇 +滋 +滌 +滑 +滓 +滔 +滕 +滙 +滚 +滝 +滞 +滟 +满 +滢 +滤 +滥 +滦 +滨 +滩 +滬 +滯 +滲 +滴 +滷 +滸 +滾 +滿 +漁 +漂 +漆 +漉 +漏 +漓 +演 +漕 +漠 +漢 +漣 +漩 +漪 +漫 +漬 +漯 +漱 +漲 +漳 +漸 +漾 +漿 +潆 +潇 +潋 +潍 +潑 +潔 +潘 +潛 +潜 +潞 +潟 +潢 +潤 +潦 +潧 +潭 +潮 +潰 +潴 +潸 +潺 +潼 +澀 +澄 +澆 +澈 +澍 +澎 +澗 +澜 +澡 +澤 +澧 +澱 +澳 +澹 +激 +濁 +濂 +濃 +濑 +濒 +濕 +濘 +濛 +濟 +濠 +濡 +濤 +濫 +濬 +濮 +濯 +濱 +濺 +濾 +瀅 +瀆 +瀉 +瀋 +瀏 +瀑 +瀕 +瀘 +瀚 +瀛 +瀝 +瀞 +瀟 +瀧 +瀨 +瀬 +瀰 +瀾 +灌 +灏 +灑 +灘 +灝 +灞 +灣 +火 +灬 +灭 +灯 +灰 +灵 +灶 +灸 +灼 +災 +灾 +灿 +炀 +炁 +炅 +炉 +炊 +炎 +炒 +炔 +炕 +炖 +炙 +炜 +炫 +炬 +炭 +炮 +炯 +炳 +炷 +炸 +点 +為 +炼 +炽 +烁 +烂 +烃 +烈 +烊 +烏 +烘 +烙 +烛 +烟 +烤 +烦 +烧 +烨 +烩 +烫 +烬 +热 +烯 +烷 +烹 +烽 +焉 +焊 +焕 +焖 +焗 +焘 +焙 +焚 +焜 +無 +焦 +焯 +焰 +焱 +然 +焼 +煅 +煉 +煊 +煌 +煎 +煒 +煖 +煙 +煜 +煞 +煤 +煥 +煦 +照 +煨 +煩 +煮 +煲 +煸 +煽 +熄 +熊 +熏 +熒 +熔 +熙 +熟 +熠 +熨 +熬 +熱 +熵 +熹 +熾 +燁 +燃 +燄 +燈 +燉 +燊 +燎 +燒 +燔 +燕 +燙 +燜 +營 +燥 +燦 +燧 +燭 +燮 +燴 +燻 +燼 +燿 +爆 +爍 +爐 +爛 +爪 +爬 +爭 +爰 +爱 +爲 +爵 +父 +爷 +爸 +爹 +爺 +爻 +爽 +爾 +牆 +片 +版 +牌 +牍 +牒 +牙 +牛 +牝 +牟 +牠 +牡 +牢 +牦 +牧 +物 +牯 +牲 +牴 +牵 +特 +牺 +牽 +犀 +犁 +犄 +犊 +犍 +犒 +犢 +犧 +犬 +犯 +状 +犷 +犸 +犹 +狀 +狂 +狄 +狈 +狎 +狐 +狒 +狗 +狙 +狞 +狠 +狡 +狩 +独 +狭 +狮 +狰 +狱 +狸 +狹 +狼 +狽 +猎 +猕 +猖 +猗 +猙 +猛 +猜 +猝 +猥 +猩 +猪 +猫 +猬 +献 +猴 +猶 +猷 +猾 +猿 +獄 +獅 +獎 +獐 +獒 +獗 +獠 +獣 +獨 +獭 +獰 +獲 +獵 +獷 +獸 +獺 +獻 +獼 +獾 +玄 +率 +玉 +王 +玑 +玖 +玛 +玟 +玠 +玥 +玩 +玫 +玮 +环 +现 +玲 +玳 +玷 +玺 +玻 +珀 +珂 +珅 +珈 +珉 +珊 +珍 +珏 +珐 +珑 +珙 +珞 +珠 +珣 +珥 +珩 +珪 +班 +珮 +珲 +珺 +現 +球 +琅 +理 +琇 +琉 +琊 +琍 +琏 +琐 +琛 +琢 +琥 +琦 +琨 +琪 +琬 +琮 +琰 +琲 +琳 +琴 +琵 +琶 +琺 +琼 +瑀 +瑁 +瑄 +瑋 +瑕 +瑗 +瑙 +瑚 +瑛 +瑜 +瑞 +瑟 +瑠 +瑣 +瑤 +瑩 +瑪 +瑯 +瑰 +瑶 +瑾 +璀 +璁 +璃 +璇 +璉 +璋 +璎 +璐 +璜 +璞 +璟 +璧 +璨 +環 +璽 +璿 +瓊 +瓏 +瓒 +瓜 +瓢 +瓣 +瓤 +瓦 +瓮 +瓯 +瓴 +瓶 +瓷 +甄 +甌 +甕 +甘 +甙 +甚 +甜 +生 +產 +産 +甥 +甦 +用 +甩 +甫 +甬 +甭 +甯 +田 +由 +甲 +申 +电 +男 +甸 +町 +画 +甾 +畀 +畅 +界 +畏 +畑 +畔 +留 +畜 +畝 +畢 +略 +畦 +番 +畫 +異 +畲 +畳 +畴 +當 +畸 +畹 +畿 +疆 +疇 +疊 +疏 +疑 +疔 +疖 +疗 +疙 +疚 +疝 +疟 +疡 +疣 +疤 +疥 +疫 +疮 +疯 +疱 +疲 +疳 +疵 +疸 +疹 +疼 +疽 +疾 +痂 +病 +症 +痈 +痉 +痊 +痍 +痒 +痔 +痕 +痘 +痙 +痛 +痞 +痠 +痢 +痣 +痤 +痧 +痨 +痪 +痫 +痰 +痱 +痴 +痹 +痺 +痼 +痿 +瘀 +瘁 +瘋 +瘍 +瘓 +瘘 +瘙 +瘟 +瘠 +瘡 +瘢 +瘤 +瘦 +瘧 +瘩 +瘪 +瘫 +瘴 +瘸 +瘾 +療 +癇 +癌 +癒 +癖 +癜 +癞 +癡 +癢 +癣 +癥 +癫 +癬 +癮 +癱 +癲 +癸 +発 +登 +發 +白 +百 +皂 +的 +皆 +皇 +皈 +皋 +皎 +皑 +皓 +皖 +皙 +皚 +皮 +皰 +皱 +皴 +皺 +皿 +盂 +盃 +盅 +盆 +盈 +益 +盎 +盏 +盐 +监 +盒 +盔 +盖 +盗 +盘 +盛 +盜 +盞 +盟 +盡 +監 +盤 +盥 +盧 +盪 +目 +盯 +盱 +盲 +直 +相 +盹 +盼 +盾 +省 +眈 +眉 +看 +県 +眙 +眞 +真 +眠 +眦 +眨 +眩 +眯 +眶 +眷 +眸 +眺 +眼 +眾 +着 +睁 +睇 +睏 +睐 +睑 +睛 +睜 +睞 +睡 +睢 +督 +睥 +睦 +睨 +睪 +睫 +睬 +睹 +睽 +睾 +睿 +瞄 +瞅 +瞇 +瞋 +瞌 +瞎 +瞑 +瞒 +瞓 +瞞 +瞟 +瞠 +瞥 +瞧 +瞩 +瞪 +瞬 +瞭 +瞰 +瞳 +瞻 +瞼 +瞿 +矇 +矍 +矗 +矚 +矛 +矜 +矢 +矣 +知 +矩 +矫 +短 +矮 +矯 +石 +矶 +矽 +矾 +矿 +码 +砂 +砌 +砍 +砒 +研 +砖 +砗 +砚 +砝 +砣 +砥 +砧 +砭 +砰 +砲 +破 +砷 +砸 +砺 +砼 +砾 +础 +硅 +硐 +硒 +硕 +硝 +硫 +硬 +确 +硯 +硼 +碁 +碇 +碉 +碌 +碍 +碎 +碑 +碓 +碗 +碘 +碚 +碛 +碟 +碣 +碧 +碩 +碰 +碱 +碳 +碴 +確 +碼 +碾 +磁 +磅 +磊 +磋 +磐 +磕 +磚 +磡 +磨 +磬 +磯 +磲 +磷 +磺 +礁 +礎 +礙 +礡 +礦 +礪 +礫 +礴 +示 +礼 +社 +祀 +祁 +祂 +祇 +祈 +祉 +祎 +祐 +祕 +祖 +祗 +祚 +祛 +祜 +祝 +神 +祟 +祠 +祢 +祥 +票 +祭 +祯 +祷 +祸 +祺 +祿 +禀 +禁 +禄 +禅 +禍 +禎 +福 +禛 +禦 +禧 +禪 +禮 +禱 +禹 +禺 +离 +禽 +禾 +禿 +秀 +私 +秃 +秆 +秉 +秋 +种 +科 +秒 +秘 +租 +秣 +秤 +秦 +秧 +秩 +秭 +积 +称 +秸 +移 +秽 +稀 +稅 +程 +稍 +税 +稔 +稗 +稚 +稜 +稞 +稟 +稠 +稣 +種 +稱 +稲 +稳 +稷 +稹 +稻 +稼 +稽 +稿 +穀 +穂 +穆 +穌 +積 +穎 +穗 +穢 +穩 +穫 +穴 +究 +穷 +穹 +空 +穿 +突 +窃 +窄 +窈 +窍 +窑 +窒 +窓 +窕 +窖 +窗 +窘 +窜 +窝 +窟 +窠 +窥 +窦 +窨 +窩 +窪 +窮 +窯 +窺 +窿 +竄 +竅 +竇 +竊 +立 +竖 +站 +竜 +竞 +竟 +章 +竣 +童 +竭 +端 +競 +竹 +竺 +竽 +竿 +笃 +笆 +笈 +笋 +笏 +笑 +笔 +笙 +笛 +笞 +笠 +符 +笨 +第 +笹 +笺 +笼 +筆 +等 +筊 +筋 +筍 +筏 +筐 +筑 +筒 +答 +策 +筛 +筝 +筠 +筱 +筲 +筵 +筷 +筹 +签 +简 +箇 +箋 +箍 +箏 +箐 +箔 +箕 +算 +箝 +管 +箩 +箫 +箭 +箱 +箴 +箸 +節 +篁 +範 +篆 +篇 +築 +篑 +篓 +篙 +篝 +篠 +篡 +篤 +篩 +篪 +篮 +篱 +篷 +簇 +簌 +簍 +簡 +簦 +簧 +簪 +簫 +簷 +簸 +簽 +簾 +簿 +籁 +籃 +籌 +籍 +籐 +籟 +籠 +籤 +籬 +籮 +籲 +米 +类 +籼 +籽 +粄 +粉 +粑 +粒 +粕 +粗 +粘 +粟 +粤 +粥 +粧 +粪 +粮 +粱 +粲 +粳 +粵 +粹 +粼 +粽 +精 +粿 +糅 +糊 +糍 +糕 +糖 +糗 +糙 +糜 +糞 +糟 +糠 +糧 +糬 +糯 +糰 +糸 +系 +糾 +紀 +紂 +約 +紅 +紉 +紊 +紋 +納 +紐 +紓 +純 +紗 +紘 +紙 +級 +紛 +紜 +素 +紡 +索 +紧 +紫 +紮 +累 +細 +紳 +紹 +紺 +終 +絃 +組 +絆 +経 +結 +絕 +絞 +絡 +絢 +給 +絨 +絮 +統 +絲 +絳 +絵 +絶 +絹 +綁 +綏 +綑 +經 +継 +続 +綜 +綠 +綢 +綦 +綫 +綬 +維 +綱 +網 +綴 +綵 +綸 +綺 +綻 +綽 +綾 +綿 +緊 +緋 +総 +緑 +緒 +緘 +線 +緝 +緞 +締 +緣 +編 +緩 +緬 +緯 +練 +緹 +緻 +縁 +縄 +縈 +縛 +縝 +縣 +縫 +縮 +縱 +縴 +縷 +總 +績 +繁 +繃 +繆 +繇 +繋 +織 +繕 +繚 +繞 +繡 +繩 +繪 +繫 +繭 +繳 +繹 +繼 +繽 +纂 +續 +纍 +纏 +纓 +纔 +纖 +纜 +纠 +红 +纣 +纤 +约 +级 +纨 +纪 +纫 +纬 +纭 +纯 +纰 +纱 +纲 +纳 +纵 +纶 +纷 +纸 +纹 +纺 +纽 +纾 +线 +绀 +练 +组 +绅 +细 +织 +终 +绊 +绍 +绎 +经 +绑 +绒 +结 +绔 +绕 +绘 +给 +绚 +绛 +络 +绝 +绞 +统 +绡 +绢 +绣 +绥 +绦 +继 +绩 +绪 +绫 +续 +绮 +绯 +绰 +绳 +维 +绵 +绶 +绷 +绸 +绻 +综 +绽 +绾 +绿 +缀 +缄 +缅 +缆 +缇 +缈 +缉 +缎 +缓 +缔 +缕 +编 +缘 +缙 +缚 +缜 +缝 +缠 +缢 +缤 +缥 +缨 +缩 +缪 +缭 +缮 +缰 +缱 +缴 +缸 +缺 +缽 +罂 +罄 +罌 +罐 +网 +罔 +罕 +罗 +罚 +罡 +罢 +罩 +罪 +置 +罰 +署 +罵 +罷 +罹 +羁 +羅 +羈 +羊 +羌 +美 +羔 +羚 +羞 +羟 +羡 +羣 +群 +羥 +羧 +羨 +義 +羯 +羲 +羸 +羹 +羽 +羿 +翁 +翅 +翊 +翌 +翎 +習 +翔 +翘 +翟 +翠 +翡 +翦 +翩 +翰 +翱 +翳 +翹 +翻 +翼 +耀 +老 +考 +耄 +者 +耆 +耋 +而 +耍 +耐 +耒 +耕 +耗 +耘 +耙 +耦 +耨 +耳 +耶 +耷 +耸 +耻 +耽 +耿 +聂 +聆 +聊 +聋 +职 +聒 +联 +聖 +聘 +聚 +聞 +聪 +聯 +聰 +聲 +聳 +聴 +聶 +職 +聽 +聾 +聿 +肃 +肄 +肅 +肆 +肇 +肉 +肋 +肌 +肏 +肓 +肖 +肘 +肚 +肛 +肝 +肠 +股 +肢 +肤 +肥 +肩 +肪 +肮 +肯 +肱 +育 +肴 +肺 +肽 +肾 +肿 +胀 +胁 +胃 +胄 +胆 +背 +胍 +胎 +胖 +胚 +胛 +胜 +胝 +胞 +胡 +胤 +胥 +胧 +胫 +胭 +胯 +胰 +胱 +胳 +胴 +胶 +胸 +胺 +能 +脂 +脅 +脆 +脇 +脈 +脉 +脊 +脍 +脏 +脐 +脑 +脓 +脖 +脘 +脚 +脛 +脣 +脩 +脫 +脯 +脱 +脲 +脳 +脸 +脹 +脾 +腆 +腈 +腊 +腋 +腌 +腎 +腐 +腑 +腓 +腔 +腕 +腥 +腦 +腩 +腫 +腭 +腮 +腰 +腱 +腳 +腴 +腸 +腹 +腺 +腻 +腼 +腾 +腿 +膀 +膈 +膊 +膏 +膑 +膘 +膚 +膛 +膜 +膝 +膠 +膦 +膨 +膩 +膳 +膺 +膻 +膽 +膾 +膿 +臀 +臂 +臃 +臆 +臉 +臊 +臍 +臓 +臘 +臟 +臣 +臥 +臧 +臨 +自 +臬 +臭 +至 +致 +臺 +臻 +臼 +臾 +舀 +舂 +舅 +舆 +與 +興 +舉 +舊 +舌 +舍 +舎 +舐 +舒 +舔 +舖 +舗 +舛 +舜 +舞 +舟 +航 +舫 +般 +舰 +舱 +舵 +舶 +舷 +舸 +船 +舺 +舾 +艇 +艋 +艘 +艙 +艦 +艮 +良 +艰 +艱 +色 +艳 +艷 +艹 +艺 +艾 +节 +芃 +芈 +芊 +芋 +芍 +芎 +芒 +芙 +芜 +芝 +芡 +芥 +芦 +芩 +芪 +芫 +芬 +芭 +芮 +芯 +花 +芳 +芷 +芸 +芹 +芻 +芽 +芾 +苁 +苄 +苇 +苋 +苍 +苏 +苑 +苒 +苓 +苔 +苕 +苗 +苛 +苜 +苞 +苟 +苡 +苣 +若 +苦 +苫 +苯 +英 +苷 +苹 +苻 +茁 +茂 +范 +茄 +茅 +茉 +茎 +茏 +茗 +茜 +茧 +茨 +茫 +茬 +茭 +茯 +茱 +茲 +茴 +茵 +茶 +茸 +茹 +茼 +荀 +荃 +荆 +草 +荊 +荏 +荐 +荒 +荔 +荖 +荘 +荚 +荞 +荟 +荠 +荡 +荣 +荤 +荥 +荧 +荨 +荪 +荫 +药 +荳 +荷 +荸 +荻 +荼 +荽 +莅 +莆 +莉 +莊 +莎 +莒 +莓 +莖 +莘 +莞 +莠 +莢 +莧 +莪 +莫 +莱 +莲 +莴 +获 +莹 +莺 +莽 +莿 +菀 +菁 +菅 +菇 +菈 +菊 +菌 +菏 +菓 +菖 +菘 +菜 +菟 +菠 +菡 +菩 +華 +菱 +菲 +菸 +菽 +萁 +萃 +萄 +萊 +萋 +萌 +萍 +萎 +萘 +萝 +萤 +营 +萦 +萧 +萨 +萩 +萬 +萱 +萵 +萸 +萼 +落 +葆 +葉 +著 +葚 +葛 +葡 +董 +葦 +葩 +葫 +葬 +葭 +葯 +葱 +葳 +葵 +葷 +葺 +蒂 +蒋 +蒐 +蒔 +蒙 +蒜 +蒞 +蒟 +蒡 +蒨 +蒲 +蒸 +蒹 +蒻 +蒼 +蒿 +蓁 +蓄 +蓆 +蓉 +蓋 +蓑 +蓓 +蓖 +蓝 +蓟 +蓦 +蓬 +蓮 +蓼 +蓿 +蔑 +蔓 +蔔 +蔗 +蔘 +蔚 +蔡 +蔣 +蔥 +蔫 +蔬 +蔭 +蔵 +蔷 +蔺 +蔻 +蔼 +蔽 +蕁 +蕃 +蕈 +蕉 +蕊 +蕎 +蕙 +蕤 +蕨 +蕩 +蕪 +蕭 +蕲 +蕴 +蕻 +蕾 +薄 +薅 +薇 +薈 +薊 +薏 +薑 +薔 +薙 +薛 +薦 +薨 +薩 +薪 +薬 +薯 +薰 +薹 +藉 +藍 +藏 +藐 +藓 +藕 +藜 +藝 +藤 +藥 +藩 +藹 +藻 +藿 +蘆 +蘇 +蘊 +蘋 +蘑 +蘚 +蘭 +蘸 +蘼 +蘿 +虎 +虏 +虐 +虑 +虔 +處 +虚 +虛 +虜 +虞 +號 +虢 +虧 +虫 +虬 +虱 +虹 +虻 +虽 +虾 +蚀 +蚁 +蚂 +蚊 +蚌 +蚓 +蚕 +蚜 +蚝 +蚣 +蚤 +蚩 +蚪 +蚯 +蚱 +蚵 +蛀 +蛆 +蛇 +蛊 +蛋 +蛎 +蛐 +蛔 +蛙 +蛛 +蛟 +蛤 +蛭 +蛮 +蛰 +蛳 +蛹 +蛻 +蛾 +蜀 +蜂 +蜃 +蜆 +蜇 +蜈 +蜊 +蜍 +蜒 +蜓 +蜕 +蜗 +蜘 +蜚 +蜜 +蜡 +蜢 +蜥 +蜱 +蜴 +蜷 +蜻 +蜿 +蝇 +蝈 +蝉 +蝌 +蝎 +蝕 +蝗 +蝙 +蝟 +蝠 +蝦 +蝨 +蝴 +蝶 +蝸 +蝼 +螂 +螃 +融 +螞 +螢 +螨 +螯 +螳 +螺 +蟀 +蟄 +蟆 +蟋 +蟎 +蟑 +蟒 +蟠 +蟬 +蟲 +蟹 +蟻 +蟾 +蠅 +蠍 +蠔 +蠕 +蠛 +蠟 +蠡 +蠢 +蠣 +蠱 +蠶 +蠹 +蠻 +血 +衄 +衅 +衆 +行 +衍 +術 +衔 +街 +衙 +衛 +衝 +衞 +衡 +衢 +衣 +补 +表 +衩 +衫 +衬 +衮 +衰 +衲 +衷 +衹 +衾 +衿 +袁 +袂 +袄 +袅 +袈 +袋 +袍 +袒 +袖 +袜 +袞 +袤 +袪 +被 +袭 +袱 +裁 +裂 +装 +裆 +裊 +裏 +裔 +裕 +裘 +裙 +補 +裝 +裟 +裡 +裤 +裨 +裱 +裳 +裴 +裸 +裹 +製 +裾 +褂 +複 +褐 +褒 +褓 +褔 +褚 +褥 +褪 +褫 +褲 +褶 +褻 +襁 +襄 +襟 +襠 +襪 +襬 +襯 +襲 +西 +要 +覃 +覆 +覇 +見 +規 +覓 +視 +覚 +覦 +覧 +親 +覬 +観 +覷 +覺 +覽 +觀 +见 +观 +规 +觅 +视 +览 +觉 +觊 +觎 +觐 +觑 +角 +觞 +解 +觥 +触 +觸 +言 +訂 +計 +訊 +討 +訓 +訕 +訖 +託 +記 +訛 +訝 +訟 +訣 +訥 +訪 +設 +許 +訳 +訴 +訶 +診 +註 +証 +詆 +詐 +詔 +評 +詛 +詞 +詠 +詡 +詢 +詣 +試 +詩 +詫 +詬 +詭 +詮 +詰 +話 +該 +詳 +詹 +詼 +誅 +誇 +誉 +誌 +認 +誓 +誕 +誘 +語 +誠 +誡 +誣 +誤 +誥 +誦 +誨 +說 +説 +読 +誰 +課 +誹 +誼 +調 +諄 +談 +請 +諏 +諒 +論 +諗 +諜 +諡 +諦 +諧 +諫 +諭 +諮 +諱 +諳 +諷 +諸 +諺 +諾 +謀 +謁 +謂 +謄 +謊 +謎 +謐 +謔 +謗 +謙 +講 +謝 +謠 +謨 +謬 +謹 +謾 +譁 +證 +譎 +譏 +識 +譙 +譚 +譜 +警 +譬 +譯 +議 +譲 +譴 +護 +譽 +讀 +變 +讓 +讚 +讞 +计 +订 +认 +讥 +讧 +讨 +让 +讪 +讫 +训 +议 +讯 +记 +讲 +讳 +讴 +讶 +讷 +许 +讹 +论 +讼 +讽 +设 +访 +诀 +证 +诃 +评 +诅 +识 +诈 +诉 +诊 +诋 +词 +诏 +译 +试 +诗 +诘 +诙 +诚 +诛 +话 +诞 +诟 +诠 +诡 +询 +诣 +诤 +该 +详 +诧 +诩 +诫 +诬 +语 +误 +诰 +诱 +诲 +说 +诵 +诶 +请 +诸 +诺 +读 +诽 +课 +诿 +谀 +谁 +调 +谄 +谅 +谆 +谈 +谊 +谋 +谌 +谍 +谎 +谏 +谐 +谑 +谒 +谓 +谔 +谕 +谗 +谘 +谙 +谚 +谛 +谜 +谟 +谢 +谣 +谤 +谥 +谦 +谧 +谨 +谩 +谪 +谬 +谭 +谯 +谱 +谲 +谴 +谶 +谷 +豁 +豆 +豇 +豈 +豉 +豊 +豌 +豎 +豐 +豔 +豚 +象 +豢 +豪 +豫 +豬 +豹 +豺 +貂 +貅 +貌 +貓 +貔 +貘 +貝 +貞 +負 +財 +貢 +貧 +貨 +販 +貪 +貫 +責 +貯 +貰 +貳 +貴 +貶 +買 +貸 +費 +貼 +貽 +貿 +賀 +賁 +賂 +賃 +賄 +資 +賈 +賊 +賑 +賓 +賜 +賞 +賠 +賡 +賢 +賣 +賤 +賦 +質 +賬 +賭 +賴 +賺 +購 +賽 +贅 +贈 +贊 +贍 +贏 +贓 +贖 +贛 +贝 +贞 +负 +贡 +财 +责 +贤 +败 +账 +货 +质 +贩 +贪 +贫 +贬 +购 +贮 +贯 +贰 +贱 +贲 +贴 +贵 +贷 +贸 +费 +贺 +贻 +贼 +贾 +贿 +赁 +赂 +赃 +资 +赅 +赈 +赊 +赋 +赌 +赎 +赏 +赐 +赓 +赔 +赖 +赘 +赚 +赛 +赝 +赞 +赠 +赡 +赢 +赣 +赤 +赦 +赧 +赫 +赭 +走 +赳 +赴 +赵 +赶 +起 +趁 +超 +越 +趋 +趕 +趙 +趟 +趣 +趨 +足 +趴 +趵 +趸 +趺 +趾 +跃 +跄 +跆 +跋 +跌 +跎 +跑 +跖 +跚 +跛 +距 +跟 +跡 +跤 +跨 +跩 +跪 +路 +跳 +践 +跷 +跹 +跺 +跻 +踉 +踊 +踌 +踏 +踐 +踝 +踞 +踟 +踢 +踩 +踪 +踮 +踱 +踴 +踵 +踹 +蹂 +蹄 +蹇 +蹈 +蹉 +蹊 +蹋 +蹑 +蹒 +蹙 +蹟 +蹣 +蹤 +蹦 +蹩 +蹬 +蹭 +蹲 +蹴 +蹶 +蹺 +蹼 +蹿 +躁 +躇 +躉 +躊 +躋 +躍 +躏 +躪 +身 +躬 +躯 +躲 +躺 +軀 +車 +軋 +軌 +軍 +軒 +軟 +転 +軸 +軼 +軽 +軾 +較 +載 +輒 +輓 +輔 +輕 +輛 +輝 +輟 +輩 +輪 +輯 +輸 +輻 +輾 +輿 +轄 +轅 +轆 +轉 +轍 +轎 +轟 +车 +轧 +轨 +轩 +转 +轭 +轮 +软 +轰 +轲 +轴 +轶 +轻 +轼 +载 +轿 +较 +辄 +辅 +辆 +辇 +辈 +辉 +辊 +辍 +辐 +辑 +输 +辕 +辖 +辗 +辘 +辙 +辛 +辜 +辞 +辟 +辣 +辦 +辨 +辩 +辫 +辭 +辮 +辯 +辰 +辱 +農 +边 +辺 +辻 +込 +辽 +达 +迁 +迂 +迄 +迅 +过 +迈 +迎 +运 +近 +返 +还 +这 +进 +远 +违 +连 +迟 +迢 +迤 +迥 +迦 +迩 +迪 +迫 +迭 +述 +迴 +迷 +迸 +迹 +迺 +追 +退 +送 +适 +逃 +逅 +逆 +选 +逊 +逍 +透 +逐 +递 +途 +逕 +逗 +這 +通 +逛 +逝 +逞 +速 +造 +逢 +連 +逮 +週 +進 +逵 +逶 +逸 +逻 +逼 +逾 +遁 +遂 +遅 +遇 +遊 +運 +遍 +過 +遏 +遐 +遑 +遒 +道 +達 +違 +遗 +遙 +遛 +遜 +遞 +遠 +遢 +遣 +遥 +遨 +適 +遭 +遮 +遲 +遴 +遵 +遶 +遷 +選 +遺 +遼 +遽 +避 +邀 +邁 +邂 +邃 +還 +邇 +邈 +邊 +邋 +邏 +邑 +邓 +邕 +邛 +邝 +邢 +那 +邦 +邨 +邪 +邬 +邮 +邯 +邰 +邱 +邳 +邵 +邸 +邹 +邺 +邻 +郁 +郅 +郊 +郎 +郑 +郜 +郝 +郡 +郢 +郤 +郦 +郧 +部 +郫 +郭 +郴 +郵 +郷 +郸 +都 +鄂 +鄉 +鄒 +鄔 +鄙 +鄞 +鄢 +鄧 +鄭 +鄰 +鄱 +鄲 +鄺 +酉 +酊 +酋 +酌 +配 +酐 +酒 +酗 +酚 +酝 +酢 +酣 +酥 +酩 +酪 +酬 +酮 +酯 +酰 +酱 +酵 +酶 +酷 +酸 +酿 +醃 +醇 +醉 +醋 +醍 +醐 +醒 +醚 +醛 +醜 +醞 +醣 +醪 +醫 +醬 +醮 +醯 +醴 +醺 +釀 +釁 +采 +釉 +释 +釋 +里 +重 +野 +量 +釐 +金 +釗 +釘 +釜 +針 +釣 +釦 +釧 +釵 +鈀 +鈉 +鈍 +鈎 +鈔 +鈕 +鈞 +鈣 +鈦 +鈪 +鈴 +鈺 +鈾 +鉀 +鉄 +鉅 +鉉 +鉑 +鉗 +鉚 +鉛 +鉤 +鉴 +鉻 +銀 +銃 +銅 +銑 +銓 +銖 +銘 +銜 +銬 +銭 +銮 +銳 +銷 +銹 +鋁 +鋅 +鋒 +鋤 +鋪 +鋰 +鋸 +鋼 +錄 +錐 +錘 +錚 +錠 +錢 +錦 +錨 +錫 +錮 +錯 +録 +錳 +錶 +鍊 +鍋 +鍍 +鍛 +鍥 +鍰 +鍵 +鍺 +鍾 +鎂 +鎊 +鎌 +鎏 +鎔 +鎖 +鎗 +鎚 +鎧 +鎬 +鎮 +鎳 +鏈 +鏖 +鏗 +鏘 +鏞 +鏟 +鏡 +鏢 +鏤 +鏽 +鐘 +鐮 +鐲 +鐳 +鐵 +鐸 +鐺 +鑄 +鑊 +鑑 +鑒 +鑣 +鑫 +鑰 +鑲 +鑼 +鑽 +鑾 +鑿 +针 +钉 +钊 +钎 +钏 +钒 +钓 +钗 +钙 +钛 +钜 +钝 +钞 +钟 +钠 +钡 +钢 +钣 +钤 +钥 +钦 +钧 +钨 +钩 +钮 +钯 +钰 +钱 +钳 +钴 +钵 +钺 +钻 +钼 +钾 +钿 +铀 +铁 +铂 +铃 +铄 +铅 +铆 +铉 +铎 +铐 +铛 +铜 +铝 +铠 +铡 +铢 +铣 +铤 +铨 +铩 +铬 +铭 +铮 +铰 +铲 +铵 +银 +铸 +铺 +链 +铿 +销 +锁 +锂 +锄 +锅 +锆 +锈 +锉 +锋 +锌 +锏 +锐 +锑 +错 +锚 +锟 +锡 +锢 +锣 +锤 +锥 +锦 +锭 +键 +锯 +锰 +锲 +锵 +锹 +锺 +锻 +镀 +镁 +镂 +镇 +镉 +镌 +镍 +镐 +镑 +镕 +镖 +镗 +镛 +镜 +镣 +镭 +镯 +镰 +镳 +镶 +長 +长 +門 +閃 +閉 +開 +閎 +閏 +閑 +閒 +間 +閔 +閘 +閡 +関 +閣 +閥 +閨 +閩 +閱 +閲 +閹 +閻 +閾 +闆 +闇 +闊 +闌 +闍 +闔 +闕 +闖 +闘 +關 +闡 +闢 +门 +闪 +闫 +闭 +问 +闯 +闰 +闲 +间 +闵 +闷 +闸 +闹 +闺 +闻 +闽 +闾 +阀 +阁 +阂 +阅 +阆 +阇 +阈 +阉 +阎 +阐 +阑 +阔 +阕 +阖 +阙 +阚 +阜 +队 +阡 +阪 +阮 +阱 +防 +阳 +阴 +阵 +阶 +阻 +阿 +陀 +陂 +附 +际 +陆 +陇 +陈 +陋 +陌 +降 +限 +陕 +陛 +陝 +陞 +陟 +陡 +院 +陣 +除 +陨 +险 +陪 +陰 +陲 +陳 +陵 +陶 +陷 +陸 +険 +陽 +隅 +隆 +隈 +隊 +隋 +隍 +階 +随 +隐 +隔 +隕 +隘 +隙 +際 +障 +隠 +隣 +隧 +隨 +險 +隱 +隴 +隶 +隸 +隻 +隼 +隽 +难 +雀 +雁 +雄 +雅 +集 +雇 +雉 +雋 +雌 +雍 +雎 +雏 +雑 +雒 +雕 +雖 +雙 +雛 +雜 +雞 +離 +難 +雨 +雪 +雯 +雰 +雲 +雳 +零 +雷 +雹 +電 +雾 +需 +霁 +霄 +霆 +震 +霈 +霉 +霊 +霍 +霎 +霏 +霑 +霓 +霖 +霜 +霞 +霧 +霭 +霰 +露 +霸 +霹 +霽 +霾 +靂 +靄 +靈 +青 +靓 +靖 +静 +靚 +靛 +靜 +非 +靠 +靡 +面 +靥 +靦 +革 +靳 +靴 +靶 +靼 +鞅 +鞋 +鞍 +鞏 +鞑 +鞘 +鞠 +鞣 +鞦 +鞭 +韆 +韋 +韌 +韓 +韜 +韦 +韧 +韩 +韬 +韭 +音 +韵 +韶 +韻 +響 +頁 +頂 +頃 +項 +順 +須 +頌 +預 +頑 +頒 +頓 +頗 +領 +頜 +頡 +頤 +頫 +頭 +頰 +頷 +頸 +頹 +頻 +頼 +顆 +題 +額 +顎 +顏 +顔 +願 +顛 +類 +顧 +顫 +顯 +顱 +顴 +页 +顶 +顷 +项 +顺 +须 +顼 +顽 +顾 +顿 +颁 +颂 +预 +颅 +领 +颇 +颈 +颉 +颊 +颌 +颍 +颐 +频 +颓 +颔 +颖 +颗 +题 +颚 +颛 +颜 +额 +颞 +颠 +颡 +颢 +颤 +颦 +颧 +風 +颯 +颱 +颳 +颶 +颼 +飄 +飆 +风 +飒 +飓 +飕 +飘 +飙 +飚 +飛 +飞 +食 +飢 +飨 +飩 +飪 +飯 +飲 +飼 +飽 +飾 +餃 +餅 +餉 +養 +餌 +餐 +餒 +餓 +餘 +餚 +餛 +餞 +餡 +館 +餮 +餵 +餾 +饅 +饈 +饋 +饌 +饍 +饑 +饒 +饕 +饗 +饞 +饥 +饨 +饪 +饬 +饭 +饮 +饯 +饰 +饱 +饲 +饴 +饵 +饶 +饷 +饺 +饼 +饽 +饿 +馀 +馁 +馄 +馅 +馆 +馈 +馋 +馍 +馏 +馒 +馔 +首 +馗 +香 +馥 +馨 +馬 +馭 +馮 +馳 +馴 +駁 +駄 +駅 +駆 +駐 +駒 +駕 +駛 +駝 +駭 +駱 +駿 +騁 +騎 +騏 +験 +騙 +騨 +騰 +騷 +驀 +驅 +驊 +驍 +驒 +驕 +驗 +驚 +驛 +驟 +驢 +驥 +马 +驭 +驮 +驯 +驰 +驱 +驳 +驴 +驶 +驷 +驸 +驹 +驻 +驼 +驾 +驿 +骁 +骂 +骄 +骅 +骆 +骇 +骈 +骊 +骋 +验 +骏 +骐 +骑 +骗 +骚 +骛 +骜 +骞 +骠 +骡 +骤 +骥 +骧 +骨 +骯 +骰 +骶 +骷 +骸 +骼 +髂 +髅 +髋 +髏 +髒 +髓 +體 +髖 +高 +髦 +髪 +髮 +髯 +髻 +鬃 +鬆 +鬍 +鬓 +鬚 +鬟 +鬢 +鬣 +鬥 +鬧 +鬱 +鬼 +魁 +魂 +魄 +魅 +魇 +魍 +魏 +魔 +魘 +魚 +魯 +魷 +鮑 +鮨 +鮪 +鮭 +鮮 +鯉 +鯊 +鯖 +鯛 +鯨 +鯰 +鯽 +鰍 +鰓 +鰭 +鰲 +鰻 +鰾 +鱈 +鱉 +鱔 +鱗 +鱷 +鱸 +鱼 +鱿 +鲁 +鲈 +鲍 +鲑 +鲛 +鲜 +鲟 +鲢 +鲤 +鲨 +鲫 +鲱 +鲲 +鲶 +鲷 +鲸 +鳃 +鳄 +鳅 +鳌 +鳍 +鳕 +鳖 +鳗 +鳝 +鳞 +鳥 +鳩 +鳳 +鳴 +鳶 +鴉 +鴕 +鴛 +鴦 +鴨 +鴻 +鴿 +鵑 +鵜 +鵝 +鵡 +鵬 +鵰 +鵲 +鶘 +鶩 +鶯 +鶴 +鷗 +鷲 +鷹 +鷺 +鸚 +鸞 +鸟 +鸠 +鸡 +鸢 +鸣 +鸥 +鸦 +鸨 +鸪 +鸭 +鸯 +鸳 +鸵 +鸽 +鸾 +鸿 +鹂 +鹃 +鹄 +鹅 +鹈 +鹉 +鹊 +鹌 +鹏 +鹑 +鹕 +鹘 +鹜 +鹞 +鹤 +鹦 +鹧 +鹫 +鹭 +鹰 +鹳 +鹵 +鹹 +鹼 +鹽 +鹿 +麂 +麋 +麒 +麓 +麗 +麝 +麟 +麥 +麦 +麩 +麴 +麵 +麸 +麺 +麻 +麼 +麽 +麾 +黃 +黄 +黍 +黎 +黏 +黑 +黒 +黔 +默 +黛 +黜 +黝 +點 +黠 +黨 +黯 +黴 +鼋 +鼎 +鼐 +鼓 +鼠 +鼬 +鼹 +鼻 +鼾 +齁 +齊 +齋 +齐 +齒 +齡 +齢 +齣 +齦 +齿 +龄 +龅 +龈 +龊 +龋 +龌 +龍 +龐 +龔 +龕 +龙 +龚 +龛 +龜 +龟 +︰ +︱ +︶ +︿ +﹁ +﹂ +﹍ +﹏ +﹐ +﹑ +﹒ +﹔ +﹕ +﹖ +﹗ +﹙ +﹚ +﹝ +﹞ +﹡ +﹣ +！ +＂ +＃ +＄ +％ +＆ +＇ +（ +） +＊ +＋ +， +－ +． +／ +０ +１ +２ +３ +４ +５ +６ +７ +８ +９ +： +； +＜ +＝ +＞ +？ +＠ +［ +＼ +］ +＾ +＿ +｀ +ａ +ｂ +ｃ +ｄ +ｅ +ｆ +ｇ +ｈ +ｉ +ｊ +ｋ +ｌ +ｍ +ｎ +ｏ +ｐ +ｑ +ｒ +ｓ +ｔ +ｕ +ｖ +ｗ +ｘ +ｙ +ｚ +｛ +｜ +｝ +～ +｡ +｢ +｣ +､ +･ +ｯ +ｰ +ｲ +ｸ +ｼ +ｽ +ﾄ +ﾉ +ﾌ +ﾗ +ﾙ +ﾝ +ﾞ +ﾟ +￣ +￥ +👍 +🔥 +😂 +😎 +... +yam +10 +2017 +12 +11 +2016 +20 +30 +15 +06 +lofter +##s +2015 +by +16 +14 +18 +13 +24 +17 +2014 +21 +##0 +22 +19 +25 +23 +com +100 +00 +05 +2013 +##a +03 +09 +08 +28 +##2 +50 +01 +04 +##1 +27 +02 +2012 +##3 +26 +##e +07 +##8 +##5 +##6 +##4 +##9 +##7 +29 +2011 +40 +##t +2010 +##o +##d +##i +2009 +##n +app +www +the +##m +31 +##c +##l +##y +##r +##g +2008 +60 +http +200 +qq +##p +80 +##f +google +pixnet +90 +cookies +tripadvisor +500 +##er +##k +35 +##h +facebook +2007 +2000 +70 +##b +of +##x +##u +45 +300 +iphone +32 +1000 +2006 +48 +ip +36 +in +38 +3d +##w +##ing +55 +ctrip +##on +##v +33 +##の +to +34 +400 +id +2005 +it +37 +windows +llc +top +99 +42 +39 +000 +led +at +##an +41 +51 +52 +46 +49 +43 +53 +44 +##z +android +58 +and +59 +2004 +56 +vr +##か +5000 +2003 +47 +blogthis +twitter +54 +##le +150 +ok +2018 +57 +75 +cn +no +ios +##in +##mm +##00 +800 +on +te +3000 +65 +2001 +360 +95 +ig +lv +120 +##ng +##を +##us +##に +pc +てす +── +600 +##te +85 +2002 +88 +##ed +html +ncc +wifi +email +64 +blog +is +##10 +##て +mail +online +##al +dvd +##ic +studio +##は +##℃ +##ia +##と +line +vip +72 +##q +98 +##ce +##en +for +##is +##ra +##es +##j +usb +net +cp +1999 +asia +4g +##cm +diy +new +3c +##お +ta +66 +language +vs +apple +tw +86 +web +##ne +ipad +62 +you +##re +101 +68 +##tion +ps +de +bt +pony +atm +##2017 +1998 +67 +##ch +ceo +##or +go +##na +av +pro +cafe +96 +pinterest +97 +63 +pixstyleme3c +##ta +more +said +##2016 +1997 +mp3 +700 +##ll +nba +jun +##20 +92 +tv +1995 +pm +61 +76 +nbsp +250 +##ie +linux +##ma +cd +110 +hd +##17 +78 +##ion +77 +6000 +am +##th +##st +94 +##se +##et +69 +180 +gdp +my +105 +81 +abc +89 +flash +79 +one +93 +1990 +1996 +##ck +gps +##も +##ly +web885 +106 +2020 +91 +##ge +4000 +1500 +xd +boss +isbn +1994 +org +##ry +me +love +##11 +0fork +73 +##12 +3g +##ter +##ar +71 +82 +##la +hotel +130 +1970 +pk +83 +87 +140 +ie +##os +##30 +##el +74 +##50 +seo +cpu +##ml +p2p +84 +may +##る +sun +tue +internet +cc +posted +youtube +##at +##ン +##man +ii +##ル +##15 +abs +nt +pdf +yahoo +ago +1980 +##it +news +mac +104 +##てす +##me +##り +java +1992 +spa +##de +##nt +hk +all +plus +la +1993 +##mb +##16 +##ve +west +##da +160 +air +##い +##ps +から +##to +1989 +logo +htc +php +https +fi +momo +##son +sat +##ke +##80 +ebd +suv +wi +day +apk +##88 +##um +mv +galaxy +wiki +or +brake +##ス +1200 +する +this +1991 +mon +##こ +❤2017 +po +##ない +javascript +life +home +june +##ss +system +900 +##ー +##０ +pp +1988 +world +fb +4k +br +##as +ic +ai +leonardo +safari +##60 +live +free +xx +wed +win7 +kiehl +##co +lg +o2o +##go +us +235 +1949 +mm +しい +vfm +kanye +##90 +##2015 +##id +jr +##ey +123 +rss +##sa +##ro +##am +##no +thu +fri +350 +##sh +##ki +103 +comments +name +##のて +##pe +##ine +max +1987 +8000 +uber +##mi +##ton +wordpress +office +1986 +1985 +##ment +107 +bd +win10 +##ld +##li +gmail +bb +dior +##rs +##ri +##rd +##ます +up +cad +##® +dr +して +read +##21 +をお +##io +##99 +url +1984 +pvc +paypal +show +policy +##40 +##ty +##18 +with +##★ +##01 +txt +102 +##ba +dna +from +post +mini +ar +taiwan +john +##ga +privacy +agoda +##13 +##ny +word +##24 +##22 +##by +##ur +##hz +1982 +##ang +265 +cookie +netscape +108 +##ka +##～ +##ad +house +share +note +ibm +code +hello +nike +sim +survey +##016 +1979 +1950 +wikia +##32 +##017 +5g +cbc +##tor +##kg +1983 +##rt +##14 +campaign +store +2500 +os +##ct +##ts +##° +170 +api +##ns +365 +excel +##な +##ao +##ら +##し +～～ +##nd +university +163 +には +518 +##70 +##ya +##il +##25 +pierre +ipo +0020 +897 +##23 +hotels +##ian +のお +125 +years +6606 +##ers +##26 +high +##day +time +##ay +bug +##line +##く +##す +##be +xp +talk2yam +yamservice +10000 +coco +##dy +sony +##ies +1978 +microsoft +david +people +##ha +1960 +instagram +intel +その +##ot +iso +1981 +##va +115 +##mo +##land +xxx +man +co +ltxsw +##ation +baby +220 +##pa +##ol +1945 +7000 +tag +450 +##ue +msn +##31 +oppo +##ト +##ca +control +##om +st +chrome +##ure +##ん +be +##き +lol +##19 +した +##bo +240 +lady +##100 +##way +##から +4600 +##ko +##do +##un +4s +corporation +168 +##ni +herme +##28 +ｃｐ +978 +##up +##06 +ui +##ds +ppt +admin +three +します +bbc +re +128 +##48 +ca +##015 +##35 +hp +##ee +tpp +##た +##ive +×× +root +##cc +##ました +##ble +##ity +adobe +park +114 +et +oled +city +##ex +##ler +##ap +china +##book +20000 +view +##ice +global +##km +your +hong +##mg +out +##ms +ng +ebay +##29 +menu +ubuntu +##cy +rom +##view +open +ktv +do +server +##lo +if +english +##ね +##５ +##oo +1600 +##02 +step1 +kong +club +135 +july +inc +1976 +mr +hi +##net +touch +##ls +##ii +michael +lcd +##05 +##33 +phone +james +step2 +1300 +ios9 +##box +dc +##２ +##ley +samsung +111 +280 +pokemon +css +##ent +##les +いいえ +##１ +s8 +atom +play +bmw +##said +sa +etf +ctrl +♥yoyo♥ +##55 +2025 +##2014 +##66 +adidas +amazon +1958 +##ber +##ner +visa +##77 +##der +1800 +connectivity +##hi +firefox +109 +118 +hr +so +style +mark +pop +ol +skip +1975 +as +##27 +##ir +##61 +190 +mba +##う +##ai +le +##ver +1900 +cafe2017 +lte +super +113 +129 +##ron +amd +like +##☆ +are +##ster +we +##sk +paul +data +international +##ft +longchamp +ssd +good +##ート +##ti +reply +##my +↓↓↓ +apr +star +##ker +source +136 +js +112 +get +force +photo +##one +126 +##2013 +##ow +link +bbs +1972 +goods +##lin +python +119 +##ip +game +##ics +##ません +blue +##● +520 +##45 +page +itunes +##03 +1955 +260 +1968 +gt +gif +618 +##ff +##47 +group +くたさい +about +bar +ganji +##nce +music +lee +not +1977 +1971 +1973 +##per +an +faq +comment +##って +days +##ock +116 +##bs +1974 +1969 +v1 +player +1956 +xbox +sql +fm +f1 +139 +##ah +210 +##lv +##mp +##000 +melody +1957 +##３ +550 +17life +199 +1966 +xml +market +##au +##71 +999 +##04 +what +gl +##95 +##age +tips +##68 +book +##ting +mysql +can +1959 +230 +##ung +wonderland +watch +10℃ +##ction +9000 +mar +mobile +1946 +1962 +article +##db +part +▲top +party +って +1967 +1964 +1948 +##07 +##ore +##op +この +dj +##78 +##38 +010 +main +225 +1965 +##ong +art +320 +ad +134 +020 +##73 +117 +pm2 +japan +228 +##08 +ts +1963 +##ica +der +sm +##36 +2019 +##wa +ct +##７ +##や +##64 +1937 +homemesh +search +##85 +##れは +##tv +##di +macbook +##９ +##くたさい +service +##♥ +type +った +750 +##ier +##si +##75 +##います +##ok +best +##ット +goris +lock +##った +cf +3m +big +##ut +ftp +carol +##vi +１０ +1961 +happy +sd +##ac +122 +anti +pe +cnn +iii +1920 +138 +##ラ +1940 +esp +jan +tags +##98 +##51 +august +vol +##86 +154 +##™ +##fs +##れ +##sion +design +ac +##ム +press +jordan +ppp +that +key +check +##６ +##tt +##㎡ +1080p +##lt +power +##42 +1952 +##bc +vivi +##ック +he +133 +121 +jpg +##rry +201 +175 +3500 +1947 +nb +##ted +##rn +しています +1954 +usd +##t00 +master +##ンク +001 +model +##58 +al +##09 +1953 +##34 +ram +goo +ても +##ui +127 +1930 +red +##ary +rpg +item +##pm +##41 +270 +##za +project +##2012 +hot +td +blogabstract +##ger +##62 +650 +##44 +gr2 +##します +##ｍ +black +electronic +nfc +year +asus +また +html5 +cindy +##hd +m3 +132 +esc +##od +booking +##53 +fed +tvb +##81 +##ina +mit +165 +##いる +chan +192 +distribution +next +になる +peter +bios +steam +cm +1941 +にも +pk10 +##ix +##65 +##91 +dec +nasa +##ana +icecat +00z +b1 +will +##46 +li +se +##ji +##み +##ard +oct +##ain +jp +##ze +##bi +cio +##56 +smart +h5 +##39 +##port +curve +vpn +##nm +##dia +utc +##あり +12345678910 +##52 +rmvb +chanel +a4 +miss +##and +##im +media +who +##63 +she +girl +5s +124 +vera +##して +class +vivo +king +##フ +##ei +national +ab +1951 +5cm +888 +145 +ipod +ap +1100 +5mm +211 +ms +2756 +##69 +mp4 +msci +##po +##89 +131 +mg +index +380 +##bit +##out +##zz +##97 +##67 +158 +apec +##８ +photoshop +opec +￥799 +ては +##96 +##tes +##ast +2g +○○ +##ール +￥2899 +##ling +##よ +##ory +1938 +##ical +kitty +content +##43 +step3 +##cn +win8 +155 +vc +1400 +iphone7 +robert +##した +tcl +137 +beauty +##87 +en +dollars +##ys +##oc +step +pay +yy +a1 +##2011 +##lly +##ks +##♪ +1939 +188 +download +1944 +sep +exe +ph +います +school +gb +center +pr +street +##board +uv +##37 +##lan +winrar +##que +##ua +##com +1942 +1936 +480 +gpu +##４ +ettoday +fu +tom +##54 +##ren +##via +149 +##72 +b2b +144 +##79 +##tch +rose +arm +mb +##49 +##ial +##nn +nvidia +step4 +mvp +00㎡ +york +156 +##イ +how +cpi +591 +2765 +gov +kg +joe +##xx +mandy +pa +##ser +copyright +fashion +1935 +don +##け +ecu +##ist +##art +erp +wap +have +##lm +talk +##ek +##ning +##if +ch +##ite +video +1943 +cs +san +iot +look +##84 +##2010 +##ku +october +##ux +trump +##hs +##ide +box +141 +first +##ins +april +##ight +##83 +185 +angel +protected +aa +151 +162 +x1 +m2 +##fe +##× +##ho +size +143 +min +ofo +fun +gomaji +ex +hdmi +food +dns +march +chris +kevin +##のか +##lla +##pp +##ec +ag +ems +6s +720p +##rm +##ham +off +##92 +asp +team +fandom +ed +299 +▌♥ +##ell +info +されています +##82 +sina +4066 +161 +##able +##ctor +330 +399 +315 +dll +rights +ltd +idc +jul +3kg +1927 +142 +ma +surface +##76 +##ク +～～～ +304 +mall +eps +146 +green +##59 +map +space +donald +v2 +sodu +##light +1931 +148 +1700 +まて +310 +reserved +htm +##han +##57 +2d +178 +mod +##ise +##tions +152 +ti +##shi +doc +1933 +icp +055 +wang +##ram +shopping +aug +##pi +##well +now +wam +b2 +からお +##hu +236 +1928 +##gb +266 +f2 +##93 +153 +mix +##ef +##uan +bwl +##plus +##res +core +##ess +tea +5℃ +hktvmall +nhk +##ate +list +##ese +301 +feb +4m +inn +ての +nov +159 +12345 +daniel +##ci +pass +##bet +##nk +coffee +202 +ssl +airbnb +##ute +fbi +woshipm +skype +ea +cg +sp +##fc +##www +yes +edge +alt +007 +##94 +fpga +##ght +##gs +iso9001 +さい +##ile +##wood +##uo +image +lin +icon +american +##em +1932 +set +says +##king +##tive +blogger +##74 +なと +256 +147 +##ox +##zy +##red +##ium +##lf +nokia +claire +##リ +##ding +november +lohas +##500 +##tic +##マ +##cs +##ある +##che +##ire +##gy +##ult +db +january +win +##カ +166 +road +ptt +##ま +##つ +198 +##fa +##mer +anna +pchome +はい +udn +ef +420 +##time +##tte +2030 +##ア +g20 +white +かかります +1929 +308 +garden +eleven +di +##おります +chen +309b +777 +172 +young +cosplay +ちてない +4500 +bat +##123 +##tra +##ては +kindle +npc +steve +etc +##ern +##｜ +call +xperia +ces +travel +sk +s7 +##ous +1934 +##int +みいたたけます +183 +edu +file +cho +qr +##car +##our +186 +##ant +##ｄ +eric +1914 +rends +##jo +##する +mastercard +##2000 +kb +##min +290 +##ino +vista +##ris +##ud +jack +2400 +##set +169 +pos +1912 +##her +##ou +taipei +しく +205 +beta +##ませんか +232 +##fi +express +255 +body +##ill +aphojoy +user +december +meiki +##ick +tweet +richard +##av +##ᆫ +iphone6 +##dd +ちてすか +views +##mark +321 +pd +##００ +times +##▲ +level +##ash +10g +point +5l +##ome +208 +koreanmall +##ak +george +q2 +206 +wma +tcp +##200 +スタッフ +full +mlb +##lle +##watch +tm +run +179 +911 +smith +business +##und +1919 +color +##tal +222 +171 +##less +moon +4399 +##rl +update +pcb +shop +499 +157 +little +なし +end +##mhz +van +dsp +easy +660 +##house +##key +history +##ｏ +oh +##001 +##hy +##web +oem +let +was +##2009 +##gg +review +##wan +182 +##°c +203 +uc +title +##val +united +233 +2021 +##ons +doi +trivago +overdope +sbs +##ance +##ち +grand +special +573032185 +imf +216 +wx17house +##so +##ーム +audi +##he +london +william +##rp +##ake +science +beach +cfa +amp +ps4 +880 +##800 +##link +##hp +crm +ferragamo +bell +make +##eng +195 +under +zh +photos +2300 +##style +##ント +via +176 +da +##gi +company +i7 +##ray +thomas +370 +ufo +i5 +##max +plc +ben +back +research +8g +173 +mike +##pc +##ッフ +september +189 +##ace +vps +february +167 +pantos +wp +lisa +1921 +★★ +jquery +night +long +offer +##berg +##news +1911 +##いて +ray +fks +wto +せます +over +164 +340 +##all +##rus +1924 +##888 +##works +blogtitle +loftpermalink +##→ +187 +martin +test +ling +km +##め +15000 +fda +v3 +##ja +##ロ +ｗedding +かある +outlet +family +##ea +をこ +##top +story +##ness +salvatore +##lu +204 +swift +215 +room +している +oracle +##ul +1925 +sam +b2c +week +pi +rock +##のは +##ａ +##けと +##ean +##300 +##gle +cctv +after +chinese +##back +powered +x2 +##tan +1918 +##nes +##イン +canon +only +181 +##zi +##las +say +##oe +184 +##sd +221 +##bot +##world +##zo +sky +made +top100 +just +1926 +pmi +802 +234 +gap +##vr +177 +les +174 +▲topoct +ball +vogue +vi +ing +ofweek +cos +##list +##ort +▲topmay +##なら +##lon +として +last +##tc +##of +##bus +##gen +real +eva +##コ +a3 +nas +##lie +##ria +##coin +##bt +▲topapr +his +212 +cat +nata +vive +health +⋯⋯ +drive +sir +▲topmar +du +cup +##カー +##ook +##よう +##sy +alex +msg +tour +しました +3ce +##word +193 +ebooks +r8 +block +318 +##より +2200 +nice +pvp +207 +months +1905 +rewards +##ther +1917 +0800 +##xi +##チ +##sc +micro +850 +gg +blogfp +op +1922 +daily +m1 +264 +true +##bb +ml +##tar +##のお +##ky +anthony +196 +253 +##yo +state +218 +##ara +##aa +##rc +##tz +##ston +より +gear +##eo +##ade +ge +see +1923 +##win +##ura +ss +heart +##den +##ita +down +##sm +el +png +2100 +610 +rakuten +whatsapp +bay +dream +add +##use +680 +311 +pad +gucci +mpv +##ode +##fo +island +▲topjun +##▼ +223 +jason +214 +chicago +##❤ +しの +##hone +io +##れる +##ことか +sogo +be2 +##ology +990 +cloud +vcd +##con +2～3 +##ford +##joy +##kb +##こさいます +##rade +but +##ach +docker +##ful +rfid +ul +##ase +hit +ford +##star +580 +##○ +１１ +a2 +sdk +reading +edited +##are +cmos +##mc +238 +siri +light +##ella +##ため +bloomberg +##read +pizza +##ison +jimmy +##vm +college +node +journal +ba +18k +##play +245 +##cer +２０ +magic +##yu +191 +jump +288 +tt +##ings +asr +##lia +3200 +step5 +network +##cd +mc +いします +1234 +pixstyleme +273 +##600 +2800 +money +★★★★★ +1280 +１２ +430 +bl +みの +act +##tus +tokyo +##rial +##life +emba +##ae +saas +tcs +##rk +##wang +summer +##sp +ko +##ving +390 +premium +##その +netflix +##ヒ +uk +mt +##lton +right +frank +two +209 +える +##ple +##cal +021 +##んな +##sen +##ville +hold +nexus +dd +##ius +てお +##mah +##なく +tila +zero +820 +ce +##tin +resort +##ws +charles +old +p10 +5d +report +##360 +##ru +##には +bus +vans +lt +##est +pv +##レ +links +rebecca +##ツ +##dm +azure +##365 +きな +limited +bit +4gb +##mon +1910 +moto +##eam +213 +1913 +var +eos +なとの +226 +blogspot +された +699 +e3 +dos +dm +fc +##ments +##ik +##kw +boy +##bin +##ata +960 +er +##せ +219 +##vin +##tu +##ula +194 +##∥ +station +##ろ +##ature +835 +files +zara +hdr +top10 +nature +950 +magazine +s6 +marriott +##シ +avira +case +##っと +tab +##ran +tony +##home +oculus +im +##ral +jean +saint +cry +307 +rosie +##force +##ini +ice +##bert +のある +##nder +##mber +pet +2600 +##◆ +plurk +▲topdec +##sis +00kg +▲topnov +720 +##ence +tim +##ω +##nc +##ても +##name +log +ips +great +ikea +malaysia +unix +##イト +3600 +##ncy +##nie +12000 +akb48 +##ye +##oid +404 +##chi +##いた +oa +xuehai +##1000 +##orm +##rf +275 +さん +##ware +##リー +980 +ho +##pro +text +##era +560 +bob +227 +##ub +##2008 +8891 +scp +avi +##zen +2022 +mi +wu +museum +qvod +apache +lake +jcb +▲topaug +★★★ +ni +##hr +hill +302 +ne +weibo +490 +ruby +##ーシ +##ヶ +##row +4d +▲topjul +iv +##ish +github +306 +mate +312 +##スト +##lot +##ane +andrew +のハイト +##tina +t1 +rf +ed2k +##vel +##900 +way +final +りの +ns +5a +705 +197 +##メ +sweet +bytes +##ene +▲topjan +231 +##cker +##2007 +##px +100g +topapp +229 +helpapp +rs +low +14k +g4g +care +630 +ldquo +あり +##fork +leave +rm +edition +##gan +##zon +##qq +▲topsep +##google +##ism +gold +224 +explorer +##zer +toyota +category +select +visual +##labels +restaurant +##md +posts +s1 +##ico +もっと +angelababy +123456 +217 +sports +s3 +mbc +1915 +してくたさい +shell +x86 +candy +##new +kbs +face +xl +470 +##here +4a +swissinfo +v8 +▲topfeb +dram +##ual +##vice +3a +##wer +sport +q1 +ios10 +public +int +card +##ｃ +ep +au +rt +##れた +1080 +bill +##mll +kim +３０ +460 +wan +##uk +##ミ +x3 +298 +0t +scott +##ming +239 +e5 +##3d +h7n9 +worldcat +brown +##あります +##vo +##led +##580 +##ax +249 +410 +##ert +paris +##～6 +polo +925 +##lr +599 +##ナ +capital +##hing +bank +cv +1g +##chat +##ｓ +##たい +adc +##ule +2m +##ｅ +digital +hotmail +268 +##pad +870 +bbq +quot +##ring +before +wali +##まて +mcu +2k +2b +という +costco +316 +north +333 +switch +##city +##ｐ +philips +##mann +management +panasonic +##cl +##vd +##ping +##rge +alice +##lk +##ましょう +css3 +##ney +vision +alpha +##ular +##400 +##tter +lz +にお +##ありません +mode +gre +1916 +pci +##tm +237 +1～2 +##yan +##そ +について +##let +##キ +work +war +coach +ah +mary +##ᅵ +huang +##pt +a8 +pt +follow +##berry +1895 +##ew +a5 +ghost +##ション +##wn +##og +south +##code +girls +##rid +action +villa +git +r11 +table +games +##cket +error +##anonymoussaid +##ag +here +##ame +##gc +qa +##■ +##lis +gmp +##gin +vmalife +##cher +yu +wedding +##tis +demo +dragon +530 +soho +social +bye +##rant +river +orz +acer +325 +##↑ +##ース +##ats +261 +del +##ven +440 +ups +##ように +##ター +305 +value +macd +yougou +##dn +661 +##ano +ll +##urt +##rent +continue +script +##wen +##ect +paper +263 +319 +shift +##chel +##フト +##cat +258 +x5 +fox +243 +##さん +car +aaa +##blog +loading +##yn +##tp +kuso +799 +si +sns +イカせるテンマ +ヒンクテンマ3 +rmb +vdc +forest +central +prime +help +ultra +##rmb +##ような +241 +square +688 +##しい +のないフロクに +##field +##reen +##ors +##ju +c1 +start +510 +##air +##map +cdn +##wo +cba +stephen +m8 +100km +##get +opera +##base +##ood +vsa +com™ +##aw +##ail +251 +なのて +count +t2 +##ᅡ +##een +2700 +hop +##gp +vsc +tree +##eg +##ose +816 +285 +##ories +##shop +alphago +v4 +1909 +simon +##ᆼ +fluke62max +zip +スホンサー +##sta +louis +cr +bas +##～10 +bc +##yer +hadoop +##ube +##wi +1906 +0755 +hola +##low +place +centre +5v +d3 +##fer +252 +##750 +##media +281 +540 +0l +exchange +262 +series +##ハー +##san +eb +##bank +##ｋ +q3 +##nge +##mail +take +##lp +259 +1888 +client +east +cache +event +vincent +##ールを +きを +##nse +sui +855 +adchoice +##и +##stry +##なたの +246 +##zone +ga +apps +sea +##ab +248 +cisco +##タ +##rner +kymco +##care +dha +##pu +##yi +minkoff +royal +p1 +への +annie +269 +collection +kpi +playstation +257 +になります +866 +bh +##bar +queen +505 +radio +1904 +andy +armani +##xy +manager +iherb +##ery +##share +spring +raid +johnson +1908 +##ob +volvo +hall +##ball +v6 +our +taylor +##hk +bi +242 +##cp +kate +bo +water +technology +##rie +サイトは +277 +##ona +##sl +hpv +303 +gtx +hip +rdquo +jayz +stone +##lex +##rum +namespace +##やり +620 +##ale +##atic +des +##erson +##ql +##ves +##type +enter +##この +##てきます +d2 +##168 +##mix +##bian +との +a9 +jj +ky +##lc +access +movie +##hc +リストに +tower +##ration +##mit +ます +##nch +ua +tel +prefix +##o2 +1907 +##point +1901 +ott +～10 +##http +##ury +baidu +##ink +member +##logy +bigbang +nownews +##js +##shot +##tb +##こと +247 +eba +##tics +##lus +ける +v5 +spark +##ama +there +##ions +god +##lls +##down +hiv +##ress +burberry +day2 +##kv +◆◆ +jeff +related +film +edit +joseph +283 +##ark +cx +32gb +order +g9 +30000 +##ans +##tty +s5 +##bee +かあります +thread +xr +buy +sh +005 +land +spotify +mx +##ari +276 +##verse +×email +sf +why +##ことて +244 +7headlines +nego +sunny +dom +exo +401 +666 +positioning +fit +rgb +##tton +278 +kiss +alexa +adam +lp +みリストを +##ｇ +mp +##ties +##llow +amy +##du +np +002 +institute +271 +##rth +##lar +2345 +590 +##des +sidebar +１５ +imax +site +##cky +##kit +##ime +##009 +season +323 +##fun +##ンター +##ひ +gogoro +a7 +pu +lily +fire +twd600 +##ッセーシを +いて +##vis +30ml +##cture +##をお +information +##オ +close +friday +##くれる +yi +nick +てすか +##tta +##tel +6500 +##lock +cbd +economy +254 +かお +267 +tinker +double +375 +8gb +voice +##app +oops +channel +today +985 +##right +raw +xyz +##＋ +jim +edm +##cent +7500 +supreme +814 +ds +##its +##asia +dropbox +##てすか +##tti +books +272 +100ml +##tle +##ller +##ken +##more +##boy +sex +309 +##dom +t3 +##ider +##なります +##unch +1903 +810 +feel +5500 +##かった +##put +により +s2 +mo +##gh +men +ka +amoled +div +##tr +##n1 +port +howard +##tags +ken +dnf +##nus +adsense +##а +ide +##へ +buff +thunder +##town +##ique +has +##body +auto +pin +##erry +tee +てした +295 +number +##the +##013 +object +psp +cool +udnbkk +16gb +##mic +miui +##tro +most +r2 +##alk +##nity +1880 +±0 +##いました +428 +s4 +law +version +##oa +n1 +sgs +docomo +##tf +##ack +henry +fc2 +##ded +##sco +##014 +##rite +286 +0mm +linkedin +##ada +##now +wii +##ndy +ucbug +##◎ +sputniknews +legalminer +##ika +##xp +2gb +##bu +q10 +oo +b6 +come +##rman +cheese +ming +maker +##gm +nikon +##fig +ppi +kelly +##ります +jchere +てきます +ted +md +003 +fgo +tech +##tto +dan +soc +##gl +##len +hair +earth +640 +521 +img +##pper +##a1 +##てきる +##ロク +acca +##ition +##ference +suite +##ig +outlook +##mond +##cation +398 +##pr +279 +101vip +358 +##999 +282 +64gb +3800 +345 +airport +##over +284 +##おり +jones +##ith +lab +##su +##いるのて +co2 +town +piece +##llo +no1 +vmware +24h +##qi +focus +reader +##admin +##ora +tb +false +##log +1898 +know +lan +838 +##ces +f4 +##ume +motel +stop +##oper +na +flickr +netcomponents +##af +##─ +pose +williams +local +##ound +##cg +##site +##iko +いお +274 +5m +gsm +con +##ath +1902 +friends +##hip +cell +317 +##rey +780 +cream +##cks +012 +##dp +facebooktwitterpinterestgoogle +sso +324 +shtml +song +swiss +##mw +##キンク +lumia +xdd +string +tiffany +522 +marc +られた +insee +russell +sc +dell +##ations +ｏｋ +camera +289 +##vs +##flow +##late +classic +287 +##nter +stay +g1 +mtv +512 +##ever +##lab +##nger +qe +sata +ryan +d1 +50ml +cms +##cing +su +292 +3300 +editor +296 +##nap +security +sunday +association +##ens +##700 +##bra +acg +##かり +sofascore +とは +mkv +##ign +jonathan +gary +build +labels +##oto +tesla +moba +qi +gohappy +general +ajax +1024 +##かる +サイト +society +##test +##urs +wps +fedora +##ich +mozilla +328 +##480 +##dr +usa +urn +##lina +##ｒ +grace +##die +##try +##ader +1250 +##なり +elle +570 +##chen +##ᆯ +price +##ten +uhz +##ough +eq +##hen +states +push +session +balance +wow +506 +##cus +##py +when +##ward +##ep +34e +wong +library +prada +##サイト +##cle +running +##ree +313 +ck +date +q4 +##ctive +##ool +##＞ +mk +##ira +##163 +388 +die +secret +rq +dota +buffet +は１ヶ +e6 +##ez +pan +368 +ha +##card +##cha +2a +##さ +alan +day3 +eye +f3 +##end +france +keep +adi +rna +tvbs +##ala +solo +nova +##え +##tail +##ょう +support +##ries +##なる +##ved +base +copy +iis +fps +##ways +hero +hgih +profile +fish +mu +ssh +entertainment +chang +##wd +click +cake +##ond +pre +##tom +kic +pixel +##ov +##fl +product +6a +##pd +dear +##gate +es +yumi +audio +##² +##sky +echo +bin +where +##ture +329 +##ape +find +sap +isis +##なと +nand +##101 +##load +##ream +band +a6 +525 +never +##post +festival +50cm +##we +555 +guide +314 +zenfone +##ike +335 +gd +forum +jessica +strong +alexander +##ould +software +allen +##ious +program +360° +else +lohasthree +##gar +することかてきます +please +##れます +rc +##ggle +##ric +bim +50000 +##own +eclipse +355 +brian +3ds +##side +061 +361 +##other +##ける +##tech +##ator +485 +engine +##ged +##ｔ +plaza +##fit +cia +ngo +westbrook +shi +tbs +50mm +##みませんか +sci +291 +reuters +##ily +contextlink +##hn +af +##cil +bridge +very +##cel +1890 +cambridge +##ize +15g +##aid +##data +790 +frm +##head +award +butler +##sun +meta +##mar +america +ps3 +puma +pmid +##すか +lc +670 +kitchen +##lic +オーフン5 +きなしソフトサーヒス +そして +day1 +future +★★★★ +##text +##page +##rris +pm1 +##ket +fans +##っています +1001 +christian +bot +kids +trackback +##hai +c3 +display +##hl +n2 +1896 +idea +さんも +##sent +airmail +##ug +##men +pwm +けます +028 +##lution +369 +852 +awards +schemas +354 +asics +wikipedia +font +##tional +##vy +c2 +293 +##れている +##dget +##ein +っている +contact +pepper +スキル +339 +##～5 +294 +##uel +##ument +730 +##hang +みてす +q5 +##sue +rain +##ndi +wei +swatch +##cept +わせ +331 +popular +##ste +##tag +p2 +501 +trc +1899 +##west +##live +justin +honda +ping +messenger +##rap +v9 +543 +##とは +unity +appqq +はすへて +025 +leo +##tone +##テ +##ass +uniqlo +##010 +502 +her +jane +memory +moneydj +##tical +human +12306 +していると +##m2 +coc +miacare +##mn +tmt +##core +vim +kk +##may +fan +target +use +too +338 +435 +2050 +867 +737 +fast +##2c +services +##ope +omega +energy +##わ +pinkoi +1a +##なから +##rain +jackson +##ement +##シャンルの +374 +366 +そんな +p9 +rd +##ᆨ +1111 +##tier +##vic +zone +##│ +385 +690 +dl +isofix +cpa +m4 +322 +kimi +めて +davis +##lay +lulu +##uck +050 +weeks +qs +##hop +920 +##ｎ +ae +##ear +～5 +eia +405 +##fly +korea +jpeg +boost +##ship +small +##リア +1860 +eur +297 +425 +valley +##iel +simple +##ude +rn +k2 +##ena +されます +non +patrick +しているから +##ナー +feed +5757 +30g +process +well +qqmei +##thing +they +aws +lu +pink +##ters +##kin +または +board +##vertisement +wine +##ien +unicode +##dge +r1 +359 +##tant +いを +##twitter +##3c +cool1 +される +##れて +##ｌ +isp +##012 +standard +45㎡2 +402 +##150 +matt +##fu +326 +##iner +googlemsn +pixnetfacebookyahoo +##ラン +x7 +886 +##uce +メーカー +sao +##ev +##きました +##file +9678 +403 +xddd +shirt +6l +##rio +##hat +3mm +givenchy +ya +bang +##lio +monday +crystal +ロクイン +##abc +336 +head +890 +ubuntuforumwikilinuxpastechat +##vc +##～20 +##rity +cnc +7866 +ipv6 +null +1897 +##ost +yang +imsean +tiger +##fet +##ンス +352 +##＝ +dji +327 +ji +maria +##come +##んて +foundation +3100 +##beth +##なった +1m +601 +active +##aft +##don +3p +sr +349 +emma +##khz +living +415 +353 +1889 +341 +709 +457 +sas +x6 +##face +pptv +x4 +##mate +han +sophie +##jing +337 +fifa +##mand +other +sale +inwedding +##gn +てきちゃいます +##mmy +##pmlast +bad +nana +nbc +してみてくたさいね +なとはお +##wu +##かあります +##あ +note7 +single +##340 +せからこ +してくたさい♪この +しにはとんとんワークケートを +するとあなたにもっとマッチした +ならワークケートへ +もみつかっちゃうかも +ワークケートの +##bel +window +##dio +##ht +union +age +382 +１４ +##ivity +##ｙ +コメント +domain +neo +##isa +##lter +5k +f5 +steven +##cts +powerpoint +tft +self +g2 +ft +##テル +zol +##act +mwc +381 +343 +もう +nbapop +408 +てある +eds +ace +##room +previous +author +tomtom +il +##ets +hu +financial +☆☆☆ +っています +bp +5t +chi +1gb +##hg +fairmont +cross +008 +gay +h2 +function +##けて +356 +also +1b +625 +##ータ +##raph +1894 +3～5 +##ils +i3 +334 +avenue +##host +による +##bon +##tsu +message +navigation +50g +fintech +h6 +##ことを +8cm +##ject +##vas +##firm +credit +##wf +xxxx +form +##nor +##space +huawei +plan +json +sbl +##dc +machine +921 +392 +wish +##120 +##sol +windows7 +edward +##ために +development +washington +##nsis +lo +818 +##sio +##ym +##bor +planet +##～8 +##wt +ieee +gpa +##めて +camp +ann +gm +##tw +##oka +connect +##rss +##work +##atus +wall +chicken +soul +2mm +##times +fa +##ather +##cord +009 +##eep +hitachi +gui +harry +##pan +e1 +disney +##press +##ーション +wind +386 +frigidaire +##tl +liu +hsu +332 +basic +von +ev +いた +てきる +スホンサーサイト +learning +##ull +expedia +archives +change +##wei +santa +cut +ins +6gb +turbo +brand +cf1 +508 +004 +return +747 +##rip +h1 +##nis +##をこ +128gb +##にお +3t +application +しており +emc +rx +##oon +384 +quick +412 +15058 +wilson +wing +chapter +##bug +beyond +##cms +##dar +##oh +zoom +e2 +trip +sb +##nba +rcep +342 +aspx +ci +080 +gc +gnu +める +##count +advanced +dance +dv +##url +##ging +367 +8591 +am09 +shadow +battle +346 +##ｉ +##cia +##という +emily +##のてす +##tation +host +ff +techorz +sars +##mini +##mporary +##ering +nc +4200 +798 +##next +cma +##mbps +##gas +##ift +##dot +##ィ +455 +##～17 +amana +##りの +426 +##ros +ir +00㎡1 +##eet +##ible +##↓ +710 +ˋ▽ˊ +##aka +dcs +iq +##ｖ +l1 +##lor +maggie +##011 +##iu +588 +##～1 +830 +##gt +1tb +articles +create +##burg +##iki +database +fantasy +##rex +##cam +dlc +dean +##you +hard +path +gaming +victoria +maps +cb +##lee +##itor +overchicstoretvhome +systems +##xt +416 +p3 +sarah +760 +##nan +407 +486 +x9 +install +second +626 +##ann +##ph +##rcle +##nic +860 +##nar +ec +##とう +768 +metro +chocolate +##rian +～4 +##table +##しています +skin +##sn +395 +mountain +##0mm +inparadise +6m +7x24 +ib +4800 +##jia +eeworld +creative +g5 +g3 +357 +parker +ecfa +village +からの +18000 +sylvia +サーヒス +hbl +##ques +##onsored +##x2 +##きます +##v4 +##tein +ie6 +383 +##stack +389 +ver +##ads +##baby +sound +bbe +##110 +##lone +##uid +ads +022 +gundam +351 +thinkpad +006 +scrum +match +##ave +mems +##470 +##oy +##なりました +##talk +glass +lamigo +span +##eme +job +##a5 +jay +wade +kde +498 +##lace +ocean +tvg +##covery +##r3 +##ners +##rea +junior +think +##aine +cover +##ision +##sia +↓↓ +##bow +msi +413 +458 +406 +##love +711 +801 +soft +z2 +##pl +456 +1840 +mobil +mind +##uy +427 +nginx +##oi +めた +##rr +6221 +##mple +##sson +##ーシてす +371 +##nts +91tv +comhd +crv3000 +##uard +1868 +397 +deep +lost +field +gallery +##bia +rate +spf +redis +traction +930 +icloud +011 +なら +fe +jose +372 +##tory +into +sohu +fx +899 +379 +kicstart2 +##hia +すく +##～3 +##sit +ra +２４ +##walk +##xure +500g +##pact +pacific +xa +natural +carlo +##250 +##walker +1850 +##can +cto +gigi +516 +##サー +pen +##hoo +ob +matlab +##ｂ +##yy +13913459 +##iti +mango +##bbs +sense +c5 +oxford +##ニア +walker +jennifer +##ola +course +##bre +701 +##pus +##rder +lucky +075 +##ぁ +ivy +なお +##nia +sotheby +side +##ugh +joy +##orage +##ush +##bat +##dt +364 +r9 +##2d +##gio +511 +country +wear +##lax +##～7 +##moon +393 +seven +study +411 +348 +lonzo +8k +##ェ +evolution +##イフ +##kk +gs +kd +##レス +arduino +344 +b12 +##lux +arpg +##rdon +cook +##x5 +dark +five +##als +##ida +とても +sign +362 +##ちの +something +20mm +##nda +387 +##posted +fresh +tf +1870 +422 +cam +##mine +##skip +##form +##ssion +education +394 +##tee +dyson +stage +##jie +want +##night +epson +pack +あります +##ppy +テリヘル +##█ +wd +##eh +##rence +left +##lvin +golden +mhz +discovery +##trix +##n2 +loft +##uch +##dra +##sse +speed +～1 +1mdb +sorry +welcome +##urn +wave +gaga +##lmer +teddy +##160 +トラックハック +せよ +611 +##f2016 +378 +rp +##sha +rar +##あなたに +##きた +840 +holiday +##ュー +373 +074 +##vg +##nos +##rail +gartner +gi +6p +##dium +kit +488 +b3 +eco +##ろう +20g +sean +##stone +autocad +nu +##np +f16 +write +029 +m5 +##ias +images +atp +##dk +fsm +504 +1350 +ve +52kb +##xxx +##のに +##cake +414 +unit +lim +ru +1v +##ification +published +angela +16g +analytics +ak +##ｑ +##nel +gmt +##icon +again +##₂ +##bby +ios11 +445 +かこさいます +waze +いてす +##ハ +9985 +##ust +##ティー +framework +##007 +iptv +delete +52sykb +cl +wwdc +027 +30cm +##fw +##ての +1389 +##xon +brandt +##ses +##dragon +tc +vetements +anne +monte +modern +official +##へて +##ere +##nne +##oud +もちろん +５０ +etnews +##a2 +##graphy +421 +863 +##ちゃん +444 +##rtex +##てお +l2 +##gma +mount +ccd +たと +archive +morning +tan +ddos +e7 +##ホ +day4 +##ウ +gis +453 +its +495 +factory +bruce +pg +##ito +ってくたさい +guest +cdma +##lling +536 +n3 +しかし +3～4 +mega +eyes +ro +１３ +women +dac +church +##jun +singapore +##facebook +6991 +starbucks +##tos +##stin +##shine +zen +##mu +tina +20℃ +1893 +##たけて +503 +465 +request +##gence +qt +##っ +1886 +347 +363 +q7 +##zzi +diary +##tore +409 +##ead +468 +cst +##osa +canada +agent +va +##jiang +##ちは +##ーク +##lam +sg +##nix +##sday +##よって +g6 +##master +bing +##zl +charlie +１６ +8mm +nb40 +##ーン +thai +##ルフ +ln284ct +##itz +##2f +bonnie +##food +##lent +originals +##stro +##lts +418 +∟∣ +##bscribe +children +ntd +yesstyle +##かも +hmv +##tment +d5 +2cm +arts +sms +##pn +##я +##いい +topios9 +539 +lifestyle +virtual +##ague +xz +##deo +muji +024 +unt +##nnis +##ᅩ +faq1 +1884 +396 +##ette +fly +64㎡ +はしめまして +441 +curry +##pop +のこ +release +##← +##◆◆ +##cast +073 +ありな +500ml +##ews +5c +##stle +ios7 +##ima +787 +dog +lenovo +##r4 +roger +013 +cbs +vornado +100m +417 +##desk +##クok +##ald +1867 +9595 +2900 +##van +oil +##ｘ +some +break +common +##jy +##lines +g7 +twice +419 +ella +nano +belle +にこ +##mes +##self +##note +jb +##ことかてきます +benz +##との +##ova +451 +save +##wing +##ますのて +kai +りは +##hua +##rect +rainer +##unge +448 +##0m +adsl +##かな +guestname +##uma +##kins +##zu +tokichoi +##price +county +##med +##mus +rmk +391 +address +vm +えて +openload +##group +##hin +##iginal +amg +urban +##oz +jobs +emi +##public +beautiful +##sch +album +##dden +##bell +jerry +works +hostel +miller +##drive +##rmin +##１０ +376 +boot +828 +##370 +##fx +##cm～ +1885 +##nome +##ctionary +##oman +##lish +##cr +##hm +433 +##how +432 +francis +xi +c919 +b5 +evernote +##uc +vga +##3000 +coupe +##urg +##cca +##uality +019 +6g +れる +multi +##また +##ett +em +hey +##ani +##tax +##rma +inside +than +740 +leonnhurt +##jin +ict +れた +bird +notes +200mm +くの +##dical +##lli +result +442 +iu +ee +438 +smap +gopro +##last +yin +pure +998 +32g +けた +5kg +##dan +##rame +mama +##oot +bean +marketing +##hur +2l +bella +sync +xuite +##ground +515 +discuz +##getrelax +##ince +##bay +##5s +cj +##イス +gmat +apt +##pass +jing +##rix +c4 +rich +##とても +niusnews +##ello +bag +770 +##eting +##mobile +１８ +culture +015 +##のてすか +377 +1020 +area +##ience +616 +details +gp +universal +silver +dit +はお +private +ddd +u11 +kanshu +##ified +fung +##nny +dx +##520 +tai +475 +023 +##fr +##lean +3s +##pin +429 +##rin +25000 +ly +rick +##bility +usb3 +banner +##baru +##gion +metal +dt +vdf +1871 +karl +qualcomm +bear +1010 +oldid +ian +jo +##tors +population +##ernel +1882 +mmorpg +##mv +##bike +603 +##© +ww +friend +##ager +exhibition +##del +##pods +fpx +structure +##free +##tings +kl +##rley +##copyright +##mma +california +3400 +orange +yoga +4l +canmake +honey +##anda +##コメント +595 +nikkie +##ルハイト +dhl +publishing +##mall +##gnet +20cm +513 +##クセス +##┅ +e88 +970 +##dog +fishbase +##! +##" +### +##$ +##% +##& +##' +##( +##) +##* +##+ +##, +##- +##. +##/ +##: +##; +##< +##= +##> +##? +##@ +##[ +##\ +##] +##^ +##_ +##{ +##| +##} +##~ +##£ +##¤ +##¥ +##§ +##« +##± +##³ +##µ +##· +##¹ +##º +##» +##¼ +##ß +##æ +##÷ +##ø +##đ +##ŋ +##ɔ +##ə +##ɡ +##ʰ +##ˇ +##ˈ +##ˊ +##ˋ +##ˍ +##ː +##˙ +##˚ +##ˢ +##α +##β +##γ +##δ +##ε +##η +##θ +##ι +##κ +##λ +##μ +##ν +##ο +##π +##ρ +##ς +##σ +##τ +##υ +##φ +##χ +##ψ +##б +##в +##г +##д +##е +##ж +##з +##к +##л +##м +##н +##о +##п +##р +##с +##т +##у +##ф +##х +##ц +##ч +##ш +##ы +##ь +##і +##ا +##ب +##ة +##ت +##د +##ر +##س +##ع +##ل +##م +##ن +##ه +##و +##ي +##۩ +##ก +##ง +##น +##ม +##ย +##ร +##อ +##า +##เ +##๑ +##་ +##ღ +##ᄀ +##ᄁ +##ᄂ +##ᄃ +##ᄅ +##ᄆ +##ᄇ +##ᄈ +##ᄉ +##ᄋ +##ᄌ +##ᄎ +##ᄏ +##ᄐ +##ᄑ +##ᄒ +##ᅢ +##ᅣ +##ᅥ +##ᅦ +##ᅧ +##ᅨ +##ᅪ +##ᅬ +##ᅭ +##ᅮ +##ᅯ +##ᅲ +##ᅳ +##ᅴ +##ᆷ +##ᆸ +##ᆺ +##ᆻ +##ᗜ +##ᵃ +##ᵉ +##ᵍ +##ᵏ +##ᵐ +##ᵒ +##ᵘ +##‖ +##„ +##† +##• +##‥ +##‧ +##
 +##‰ +##′ +##″ +##‹ +##› +##※ +##‿ +##⁄ +##ⁱ +##⁺ +##ⁿ +##₁ +##₃ +##₄ +##€ +##№ +##ⅰ +##ⅱ +##ⅲ +##ⅳ +##ⅴ +##↔ +##↗ +##↘ +##⇒ +##∀ +##− +##∕ +##∙ +##√ +##∞ +##∟ +##∠ +##∣ +##∩ +##∮ +##∶ +##∼ +##∽ +##≈ +##≒ +##≡ +##≤ +##≥ +##≦ +##≧ +##≪ +##≫ +##⊙ +##⋅ +##⋈ +##⋯ +##⌒ +##① +##② +##③ +##④ +##⑤ +##⑥ +##⑦ +##⑧ +##⑨ +##⑩ +##⑴ +##⑵ +##⑶ +##⑷ +##⑸ +##⒈ +##⒉ +##⒊ +##⒋ +##ⓒ +##ⓔ +##ⓘ +##━ +##┃ +##┆ +##┊ +##┌ +##└ +##├ +##┣ +##═ +##║ +##╚ +##╞ +##╠ +##╭ +##╮ +##╯ +##╰ +##╱ +##╳ +##▂ +##▃ +##▅ +##▇ +##▉ +##▋ +##▌ +##▍ +##▎ +##□ +##▪ +##▫ +##▬ +##△ +##▶ +##► +##▽ +##◇ +##◕ +##◠ +##◢ +##◤ +##☀ +##☕ +##☞ +##☺ +##☼ +##♀ +##♂ +##♠ +##♡ +##♣ +##♦ +##♫ +##♬ +##✈ +##✔ +##✕ +##✖ +##✦ +##✨ +##✪ +##✰ +##✿ +##❀ +##➜ +##➤ +##⦿ +##、 +##。 +##〃 +##々 +##〇 +##〈 +##〉 +##《 +##》 +##「 +##」 +##『 +##』 +##【 +##】 +##〓 +##〔 +##〕 +##〖 +##〗 +##〜 +##〝 +##〞 +##ぃ +##ぇ +##ぬ +##ふ +##ほ +##む +##ゃ +##ゅ +##ゆ +##ょ +##゜ +##ゝ +##ァ +##ゥ +##エ +##ォ +##ケ +##サ +##セ +##ソ +##ッ +##ニ +##ヌ +##ネ +##ノ +##ヘ +##モ +##ャ +##ヤ +##ュ +##ユ +##ョ +##ヨ +##ワ +##ヲ +##・ +##ヽ +##ㄅ +##ㄆ +##ㄇ +##ㄉ +##ㄋ +##ㄌ +##ㄍ +##ㄎ +##ㄏ +##ㄒ +##ㄚ +##ㄛ +##ㄞ +##ㄟ +##ㄢ +##ㄤ +##ㄥ +##ㄧ +##ㄨ +##ㆍ +##㈦ +##㊣ +##㗎 +##一 +##丁 +##七 +##万 +##丈 +##三 +##上 +##下 +##不 +##与 +##丐 +##丑 +##专 +##且 +##丕 +##世 +##丘 +##丙 +##业 +##丛 +##东 +##丝 +##丞 +##丟 +##両 +##丢 +##两 +##严 +##並 +##丧 +##丨 +##个 +##丫 +##中 +##丰 +##串 +##临 +##丶 +##丸 +##丹 +##为 +##主 +##丼 +##丽 +##举 +##丿 +##乂 +##乃 +##久 +##么 +##义 +##之 +##乌 +##乍 +##乎 +##乏 +##乐 +##乒 +##乓 +##乔 +##乖 +##乗 +##乘 +##乙 +##乜 +##九 +##乞 +##也 +##习 +##乡 +##书 +##乩 +##买 +##乱 +##乳 +##乾 +##亀 +##亂 +##了 +##予 +##争 +##事 +##二 +##于 +##亏 +##云 +##互 +##五 +##井 +##亘 +##亙 +##亚 +##些 +##亜 +##亞 +##亟 +##亡 +##亢 +##交 +##亥 +##亦 +##产 +##亨 +##亩 +##享 +##京 +##亭 +##亮 +##亲 +##亳 +##亵 +##人 +##亿 +##什 +##仁 +##仃 +##仄 +##仅 +##仆 +##仇 +##今 +##介 +##仍 +##从 +##仏 +##仑 +##仓 +##仔 +##仕 +##他 +##仗 +##付 +##仙 +##仝 +##仞 +##仟 +##代 +##令 +##以 +##仨 +##仪 +##们 +##仮 +##仰 +##仲 +##件 +##价 +##任 +##份 +##仿 +##企 +##伉 +##伊 +##伍 +##伎 +##伏 +##伐 +##休 +##伕 +##众 +##优 +##伙 +##会 +##伝 +##伞 +##伟 +##传 +##伢 +##伤 +##伦 +##伪 +##伫 +##伯 +##估 +##伴 +##伶 +##伸 +##伺 +##似 +##伽 +##佃 +##但 +##佇 +##佈 +##位 +##低 +##住 +##佐 +##佑 +##体 +##佔 +##何 +##佗 +##佘 +##余 +##佚 +##佛 +##作 +##佝 +##佞 +##佟 +##你 +##佢 +##佣 +##佤 +##佥 +##佩 +##佬 +##佯 +##佰 +##佳 +##併 +##佶 +##佻 +##佼 +##使 +##侃 +##侄 +##來 +##侈 +##例 +##侍 +##侏 +##侑 +##侖 +##侗 +##供 +##依 +##侠 +##価 +##侣 +##侥 +##侦 +##侧 +##侨 +##侬 +##侮 +##侯 +##侵 +##侶 +##侷 +##便 +##係 +##促 +##俄 +##俊 +##俎 +##俏 +##俐 +##俑 +##俗 +##俘 +##俚 +##保 +##俞 +##俟 +##俠 +##信 +##俨 +##俩 +##俪 +##俬 +##俭 +##修 +##俯 +##俱 +##俳 +##俸 +##俺 +##俾 +##倆 +##倉 +##個 +##倌 +##倍 +##倏 +##們 +##倒 +##倔 +##倖 +##倘 +##候 +##倚 +##倜 +##借 +##倡 +##値 +##倦 +##倩 +##倪 +##倫 +##倬 +##倭 +##倶 +##债 +##值 +##倾 +##偃 +##假 +##偈 +##偉 +##偌 +##偎 +##偏 +##偕 +##做 +##停 +##健 +##側 +##偵 +##偶 +##偷 +##偻 +##偽 +##偿 +##傀 +##傅 +##傍 +##傑 +##傘 +##備 +##傚 +##傢 +##傣 +##傥 +##储 +##傩 +##催 +##傭 +##傲 +##傳 +##債 +##傷 +##傻 +##傾 +##僅 +##働 +##像 +##僑 +##僕 +##僖 +##僚 +##僥 +##僧 +##僭 +##僮 +##僱 +##僵 +##價 +##僻 +##儀 +##儂 +##億 +##儆 +##儉 +##儋 +##儒 +##儕 +##儘 +##償 +##儡 +##優 +##儲 +##儷 +##儼 +##儿 +##兀 +##允 +##元 +##兄 +##充 +##兆 +##兇 +##先 +##光 +##克 +##兌 +##免 +##児 +##兑 +##兒 +##兔 +##兖 +##党 +##兜 +##兢 +##入 +##內 +##全 +##兩 +##八 +##公 +##六 +##兮 +##兰 +##共 +##兲 +##关 +##兴 +##兵 +##其 +##具 +##典 +##兹 +##养 +##兼 +##兽 +##冀 +##内 +##円 +##冇 +##冈 +##冉 +##冊 +##册 +##再 +##冏 +##冒 +##冕 +##冗 +##写 +##军 +##农 +##冠 +##冢 +##冤 +##冥 +##冨 +##冪 +##冬 +##冯 +##冰 +##冲 +##决 +##况 +##冶 +##冷 +##冻 +##冼 +##冽 +##冾 +##净 +##凄 +##准 +##凇 +##凈 +##凉 +##凋 +##凌 +##凍 +##减 +##凑 +##凛 +##凜 +##凝 +##几 +##凡 +##凤 +##処 +##凪 +##凭 +##凯 +##凰 +##凱 +##凳 +##凶 +##凸 +##凹 +##出 +##击 +##函 +##凿 +##刀 +##刁 +##刃 +##分 +##切 +##刈 +##刊 +##刍 +##刎 +##刑 +##划 +##列 +##刘 +##则 +##刚 +##创 +##初 +##删 +##判 +##別 +##刨 +##利 +##刪 +##别 +##刮 +##到 +##制 +##刷 +##券 +##刹 +##刺 +##刻 +##刽 +##剁 +##剂 +##剃 +##則 +##剉 +##削 +##剋 +##剌 +##前 +##剎 +##剐 +##剑 +##剔 +##剖 +##剛 +##剜 +##剝 +##剣 +##剤 +##剥 +##剧 +##剩 +##剪 +##副 +##割 +##創 +##剷 +##剽 +##剿 +##劃 +##劇 +##劈 +##劉 +##劊 +##劍 +##劏 +##劑 +##力 +##劝 +##办 +##功 +##加 +##务 +##劣 +##动 +##助 +##努 +##劫 +##劭 +##励 +##劲 +##劳 +##労 +##劵 +##効 +##劾 +##势 +##勁 +##勃 +##勇 +##勉 +##勋 +##勐 +##勒 +##動 +##勖 +##勘 +##務 +##勛 +##勝 +##勞 +##募 +##勢 +##勤 +##勧 +##勳 +##勵 +##勸 +##勺 +##勻 +##勾 +##勿 +##匀 +##包 +##匆 +##匈 +##匍 +##匐 +##匕 +##化 +##北 +##匙 +##匝 +##匠 +##匡 +##匣 +##匪 +##匮 +##匯 +##匱 +##匹 +##区 +##医 +##匾 +##匿 +##區 +##十 +##千 +##卅 +##升 +##午 +##卉 +##半 +##卍 +##华 +##协 +##卑 +##卒 +##卓 +##協 +##单 +##卖 +##南 +##単 +##博 +##卜 +##卞 +##卟 +##占 +##卡 +##卢 +##卤 +##卦 +##卧 +##卫 +##卮 +##卯 +##印 +##危 +##即 +##却 +##卵 +##卷 +##卸 +##卻 +##卿 +##厂 +##厄 +##厅 +##历 +##厉 +##压 +##厌 +##厕 +##厘 +##厚 +##厝 +##原 +##厢 +##厥 +##厦 +##厨 +##厩 +##厭 +##厮 +##厲 +##厳 +##去 +##县 +##叁 +##参 +##參 +##又 +##叉 +##及 +##友 +##双 +##反 +##収 +##发 +##叔 +##取 +##受 +##变 +##叙 +##叛 +##叟 +##叠 +##叡 +##叢 +##口 +##古 +##句 +##另 +##叨 +##叩 +##只 +##叫 +##召 +##叭 +##叮 +##可 +##台 +##叱 +##史 +##右 +##叵 +##叶 +##号 +##司 +##叹 +##叻 +##叼 +##叽 +##吁 +##吃 +##各 +##吆 +##合 +##吉 +##吊 +##吋 +##同 +##名 +##后 +##吏 +##吐 +##向 +##吒 +##吓 +##吕 +##吖 +##吗 +##君 +##吝 +##吞 +##吟 +##吠 +##吡 +##否 +##吧 +##吨 +##吩 +##含 +##听 +##吭 +##吮 +##启 +##吱 +##吳 +##吴 +##吵 +##吶 +##吸 +##吹 +##吻 +##吼 +##吽 +##吾 +##呀 +##呂 +##呃 +##呆 +##呈 +##告 +##呋 +##呎 +##呐 +##呓 +##呕 +##呗 +##员 +##呛 +##呜 +##呢 +##呤 +##呦 +##周 +##呱 +##呲 +##味 +##呵 +##呷 +##呸 +##呻 +##呼 +##命 +##咀 +##咁 +##咂 +##咄 +##咆 +##咋 +##和 +##咎 +##咏 +##咐 +##咒 +##咔 +##咕 +##咖 +##咗 +##咘 +##咙 +##咚 +##咛 +##咣 +##咤 +##咦 +##咧 +##咨 +##咩 +##咪 +##咫 +##咬 +##咭 +##咯 +##咱 +##咲 +##咳 +##咸 +##咻 +##咽 +##咿 +##哀 +##品 +##哂 +##哄 +##哆 +##哇 +##哈 +##哉 +##哋 +##哌 +##响 +##哎 +##哏 +##哐 +##哑 +##哒 +##哔 +##哗 +##哟 +##員 +##哥 +##哦 +##哧 +##哨 +##哩 +##哪 +##哭 +##哮 +##哲 +##哺 +##哼 +##哽 +##唁 +##唄 +##唆 +##唇 +##唉 +##唏 +##唐 +##唑 +##唔 +##唠 +##唤 +##唧 +##唬 +##售 +##唯 +##唰 +##唱 +##唳 +##唷 +##唸 +##唾 +##啃 +##啄 +##商 +##啉 +##啊 +##問 +##啓 +##啕 +##啖 +##啜 +##啞 +##啟 +##啡 +##啤 +##啥 +##啦 +##啧 +##啪 +##啫 +##啬 +##啮 +##啰 +##啱 +##啲 +##啵 +##啶 +##啷 +##啸 +##啻 +##啼 +##啾 +##喀 +##喂 +##喃 +##善 +##喆 +##喇 +##喉 +##喊 +##喋 +##喎 +##喏 +##喔 +##喘 +##喙 +##喚 +##喜 +##喝 +##喟 +##喧 +##喪 +##喫 +##喬 +##單 +##喰 +##喱 +##喲 +##喳 +##喵 +##営 +##喷 +##喹 +##喺 +##喻 +##喽 +##嗅 +##嗆 +##嗇 +##嗎 +##嗑 +##嗒 +##嗓 +##嗔 +##嗖 +##嗚 +##嗜 +##嗝 +##嗟 +##嗡 +##嗣 +##嗤 +##嗦 +##嗨 +##嗪 +##嗬 +##嗯 +##嗰 +##嗲 +##嗳 +##嗶 +##嗷 +##嗽 +##嘀 +##嘅 +##嘆 +##嘈 +##嘉 +##嘌 +##嘍 +##嘎 +##嘔 +##嘖 +##嘗 +##嘘 +##嘚 +##嘛 +##嘜 +##嘞 +##嘟 +##嘢 +##嘣 +##嘤 +##嘧 +##嘩 +##嘭 +##嘮 +##嘯 +##嘰 +##嘱 +##嘲 +##嘴 +##嘶 +##嘸 +##嘹 +##嘻 +##嘿 +##噁 +##噌 +##噎 +##噓 +##噔 +##噗 +##噙 +##噜 +##噠 +##噢 +##噤 +##器 +##噩 +##噪 +##噬 +##噱 +##噴 +##噶 +##噸 +##噹 +##噻 +##噼 +##嚀 +##嚇 +##嚎 +##嚏 +##嚐 +##嚓 +##嚕 +##嚟 +##嚣 +##嚥 +##嚨 +##嚮 +##嚴 +##嚷 +##嚼 +##囂 +##囉 +##囊 +##囍 +##囑 +##囔 +##囗 +##囚 +##四 +##囝 +##回 +##囟 +##因 +##囡 +##团 +##団 +##囤 +##囧 +##囪 +##囫 +##园 +##困 +##囱 +##囲 +##図 +##围 +##囹 +##固 +##国 +##图 +##囿 +##圃 +##圄 +##圆 +##圈 +##國 +##圍 +##圏 +##園 +##圓 +##圖 +##團 +##圜 +##土 +##圣 +##圧 +##在 +##圩 +##圭 +##地 +##圳 +##场 +##圻 +##圾 +##址 +##坂 +##均 +##坊 +##坍 +##坎 +##坏 +##坐 +##坑 +##块 +##坚 +##坛 +##坝 +##坞 +##坟 +##坠 +##坡 +##坤 +##坦 +##坨 +##坪 +##坯 +##坳 +##坵 +##坷 +##垂 +##垃 +##垄 +##型 +##垒 +##垚 +##垛 +##垠 +##垢 +##垣 +##垦 +##垩 +##垫 +##垭 +##垮 +##垵 +##埂 +##埃 +##埋 +##城 +##埔 +##埕 +##埗 +##域 +##埠 +##埤 +##埵 +##執 +##埸 +##培 +##基 +##埼 +##堀 +##堂 +##堃 +##堅 +##堆 +##堇 +##堑 +##堕 +##堙 +##堡 +##堤 +##堪 +##堯 +##堰 +##報 +##場 +##堵 +##堺 +##堿 +##塊 +##塌 +##塑 +##塔 +##塗 +##塘 +##塚 +##塞 +##塢 +##塩 +##填 +##塬 +##塭 +##塵 +##塾 +##墀 +##境 +##墅 +##墉 +##墊 +##墒 +##墓 +##増 +##墘 +##墙 +##墜 +##增 +##墟 +##墨 +##墩 +##墮 +##墳 +##墻 +##墾 +##壁 +##壅 +##壆 +##壇 +##壊 +##壑 +##壓 +##壕 +##壘 +##壞 +##壟 +##壢 +##壤 +##壩 +##士 +##壬 +##壮 +##壯 +##声 +##売 +##壳 +##壶 +##壹 +##壺 +##壽 +##处 +##备 +##変 +##复 +##夏 +##夔 +##夕 +##外 +##夙 +##多 +##夜 +##够 +##夠 +##夢 +##夥 +##大 +##天 +##太 +##夫 +##夭 +##央 +##夯 +##失 +##头 +##夷 +##夸 +##夹 +##夺 +##夾 +##奂 +##奄 +##奇 +##奈 +##奉 +##奋 +##奎 +##奏 +##奐 +##契 +##奔 +##奕 +##奖 +##套 +##奘 +##奚 +##奠 +##奢 +##奥 +##奧 +##奪 +##奬 +##奮 +##女 +##奴 +##奶 +##奸 +##她 +##好 +##如 +##妃 +##妄 +##妆 +##妇 +##妈 +##妊 +##妍 +##妒 +##妓 +##妖 +##妘 +##妙 +##妝 +##妞 +##妣 +##妤 +##妥 +##妨 +##妩 +##妪 +##妮 +##妲 +##妳 +##妹 +##妻 +##妾 +##姆 +##姉 +##姊 +##始 +##姍 +##姐 +##姑 +##姒 +##姓 +##委 +##姗 +##姚 +##姜 +##姝 +##姣 +##姥 +##姦 +##姨 +##姪 +##姫 +##姬 +##姹 +##姻 +##姿 +##威 +##娃 +##娄 +##娅 +##娆 +##娇 +##娉 +##娑 +##娓 +##娘 +##娛 +##娜 +##娟 +##娠 +##娣 +##娥 +##娩 +##娱 +##娲 +##娴 +##娶 +##娼 +##婀 +##婁 +##婆 +##婉 +##婊 +##婕 +##婚 +##婢 +##婦 +##婧 +##婪 +##婭 +##婴 +##婵 +##婶 +##婷 +##婺 +##婿 +##媒 +##媚 +##媛 +##媞 +##媧 +##媲 +##媳 +##媽 +##媾 +##嫁 +##嫂 +##嫉 +##嫌 +##嫑 +##嫔 +##嫖 +##嫘 +##嫚 +##嫡 +##嫣 +##嫦 +##嫩 +##嫲 +##嫵 +##嫻 +##嬅 +##嬉 +##嬌 +##嬗 +##嬛 +##嬢 +##嬤 +##嬪 +##嬰 +##嬴 +##嬷 +##嬸 +##嬿 +##孀 +##孃 +##子 +##孑 +##孔 +##孕 +##孖 +##字 +##存 +##孙 +##孚 +##孛 +##孜 +##孝 +##孟 +##孢 +##季 +##孤 +##学 +##孩 +##孪 +##孫 +##孬 +##孰 +##孱 +##孳 +##孵 +##學 +##孺 +##孽 +##孿 +##宁 +##它 +##宅 +##宇 +##守 +##安 +##宋 +##完 +##宏 +##宓 +##宕 +##宗 +##官 +##宙 +##定 +##宛 +##宜 +##宝 +##实 +##実 +##宠 +##审 +##客 +##宣 +##室 +##宥 +##宦 +##宪 +##宫 +##宮 +##宰 +##害 +##宴 +##宵 +##家 +##宸 +##容 +##宽 +##宾 +##宿 +##寂 +##寄 +##寅 +##密 +##寇 +##富 +##寐 +##寒 +##寓 +##寛 +##寝 +##寞 +##察 +##寡 +##寢 +##寥 +##實 +##寧 +##寨 +##審 +##寫 +##寬 +##寮 +##寰 +##寵 +##寶 +##寸 +##对 +##寺 +##寻 +##导 +##対 +##寿 +##封 +##専 +##射 +##将 +##將 +##專 +##尉 +##尊 +##尋 +##對 +##導 +##小 +##少 +##尔 +##尕 +##尖 +##尘 +##尚 +##尝 +##尤 +##尧 +##尬 +##就 +##尴 +##尷 +##尸 +##尹 +##尺 +##尻 +##尼 +##尽 +##尾 +##尿 +##局 +##屁 +##层 +##屄 +##居 +##屆 +##屈 +##屉 +##届 +##屋 +##屌 +##屍 +##屎 +##屏 +##屐 +##屑 +##展 +##屜 +##属 +##屠 +##屡 +##屢 +##層 +##履 +##屬 +##屯 +##山 +##屹 +##屿 +##岀 +##岁 +##岂 +##岌 +##岐 +##岑 +##岔 +##岖 +##岗 +##岘 +##岙 +##岚 +##岛 +##岡 +##岩 +##岫 +##岬 +##岭 +##岱 +##岳 +##岷 +##岸 +##峇 +##峋 +##峒 +##峙 +##峡 +##峤 +##峥 +##峦 +##峨 +##峪 +##峭 +##峯 +##峰 +##峴 +##島 +##峻 +##峽 +##崁 +##崂 +##崆 +##崇 +##崎 +##崑 +##崔 +##崖 +##崗 +##崙 +##崛 +##崧 +##崩 +##崭 +##崴 +##崽 +##嵇 +##嵊 +##嵋 +##嵌 +##嵐 +##嵘 +##嵩 +##嵬 +##嵯 +##嶂 +##嶄 +##嶇 +##嶋 +##嶙 +##嶺 +##嶼 +##嶽 +##巅 +##巍 +##巒 +##巔 +##巖 +##川 +##州 +##巡 +##巢 +##工 +##左 +##巧 +##巨 +##巩 +##巫 +##差 +##己 +##已 +##巳 +##巴 +##巷 +##巻 +##巽 +##巾 +##巿 +##币 +##市 +##布 +##帅 +##帆 +##师 +##希 +##帐 +##帑 +##帕 +##帖 +##帘 +##帚 +##帛 +##帜 +##帝 +##帥 +##带 +##帧 +##師 +##席 +##帮 +##帯 +##帰 +##帳 +##帶 +##帷 +##常 +##帼 +##帽 +##幀 +##幂 +##幄 +##幅 +##幌 +##幔 +##幕 +##幟 +##幡 +##幢 +##幣 +##幫 +##干 +##平 +##年 +##并 +##幸 +##幹 +##幺 +##幻 +##幼 +##幽 +##幾 +##广 +##庁 +##広 +##庄 +##庆 +##庇 +##床 +##序 +##庐 +##库 +##应 +##底 +##庖 +##店 +##庙 +##庚 +##府 +##庞 +##废 +##庠 +##度 +##座 +##庫 +##庭 +##庵 +##庶 +##康 +##庸 +##庹 +##庾 +##廁 +##廂 +##廃 +##廈 +##廉 +##廊 +##廓 +##廖 +##廚 +##廝 +##廟 +##廠 +##廢 +##廣 +##廬 +##廳 +##延 +##廷 +##建 +##廿 +##开 +##弁 +##异 +##弃 +##弄 +##弈 +##弊 +##弋 +##式 +##弑 +##弒 +##弓 +##弔 +##引 +##弗 +##弘 +##弛 +##弟 +##张 +##弥 +##弦 +##弧 +##弩 +##弭 +##弯 +##弱 +##張 +##強 +##弹 +##强 +##弼 +##弾 +##彅 +##彆 +##彈 +##彌 +##彎 +##归 +##当 +##录 +##彗 +##彙 +##彝 +##形 +##彤 +##彥 +##彦 +##彧 +##彩 +##彪 +##彫 +##彬 +##彭 +##彰 +##影 +##彷 +##役 +##彻 +##彼 +##彿 +##往 +##征 +##径 +##待 +##徇 +##很 +##徉 +##徊 +##律 +##後 +##徐 +##徑 +##徒 +##従 +##徕 +##得 +##徘 +##徙 +##徜 +##從 +##徠 +##御 +##徨 +##復 +##循 +##徬 +##微 +##徳 +##徴 +##徵 +##德 +##徹 +##徼 +##徽 +##心 +##必 +##忆 +##忌 +##忍 +##忏 +##忐 +##忑 +##忒 +##忖 +##志 +##忘 +##忙 +##応 +##忠 +##忡 +##忤 +##忧 +##忪 +##快 +##忱 +##念 +##忻 +##忽 +##忿 +##怀 +##态 +##怂 +##怅 +##怆 +##怎 +##怏 +##怒 +##怔 +##怕 +##怖 +##怙 +##怜 +##思 +##怠 +##怡 +##急 +##怦 +##性 +##怨 +##怪 +##怯 +##怵 +##总 +##怼 +##恁 +##恃 +##恆 +##恋 +##恍 +##恐 +##恒 +##恕 +##恙 +##恚 +##恢 +##恣 +##恤 +##恥 +##恨 +##恩 +##恪 +##恫 +##恬 +##恭 +##息 +##恰 +##恳 +##恵 +##恶 +##恸 +##恺 +##恻 +##恼 +##恿 +##悄 +##悅 +##悉 +##悌 +##悍 +##悔 +##悖 +##悚 +##悟 +##悠 +##患 +##悦 +##您 +##悩 +##悪 +##悬 +##悯 +##悱 +##悲 +##悴 +##悵 +##悶 +##悸 +##悻 +##悼 +##悽 +##情 +##惆 +##惇 +##惊 +##惋 +##惑 +##惕 +##惘 +##惚 +##惜 +##惟 +##惠 +##惡 +##惦 +##惧 +##惨 +##惩 +##惫 +##惬 +##惭 +##惮 +##惯 +##惰 +##惱 +##想 +##惴 +##惶 +##惹 +##惺 +##愁 +##愆 +##愈 +##愉 +##愍 +##意 +##愕 +##愚 +##愛 +##愜 +##感 +##愣 +##愤 +##愧 +##愫 +##愷 +##愿 +##慄 +##慈 +##態 +##慌 +##慎 +##慑 +##慕 +##慘 +##慚 +##慟 +##慢 +##慣 +##慧 +##慨 +##慫 +##慮 +##慰 +##慳 +##慵 +##慶 +##慷 +##慾 +##憂 +##憊 +##憋 +##憎 +##憐 +##憑 +##憔 +##憚 +##憤 +##憧 +##憨 +##憩 +##憫 +##憬 +##憲 +##憶 +##憾 +##懂 +##懇 +##懈 +##應 +##懊 +##懋 +##懑 +##懒 +##懦 +##懲 +##懵 +##懶 +##懷 +##懸 +##懺 +##懼 +##懾 +##懿 +##戀 +##戈 +##戊 +##戌 +##戍 +##戎 +##戏 +##成 +##我 +##戒 +##戕 +##或 +##战 +##戚 +##戛 +##戟 +##戡 +##戦 +##截 +##戬 +##戮 +##戰 +##戲 +##戳 +##戴 +##戶 +##户 +##戸 +##戻 +##戾 +##房 +##所 +##扁 +##扇 +##扈 +##扉 +##手 +##才 +##扎 +##扑 +##扒 +##打 +##扔 +##払 +##托 +##扛 +##扣 +##扦 +##执 +##扩 +##扪 +##扫 +##扬 +##扭 +##扮 +##扯 +##扰 +##扱 +##扳 +##扶 +##批 +##扼 +##找 +##承 +##技 +##抄 +##抉 +##把 +##抑 +##抒 +##抓 +##投 +##抖 +##抗 +##折 +##抚 +##抛 +##抜 +##択 +##抟 +##抠 +##抡 +##抢 +##护 +##报 +##抨 +##披 +##抬 +##抱 +##抵 +##抹 +##押 +##抽 +##抿 +##拂 +##拄 +##担 +##拆 +##拇 +##拈 +##拉 +##拋 +##拌 +##拍 +##拎 +##拐 +##拒 +##拓 +##拔 +##拖 +##拗 +##拘 +##拙 +##拚 +##招 +##拜 +##拟 +##拡 +##拢 +##拣 +##拥 +##拦 +##拧 +##拨 +##择 +##括 +##拭 +##拮 +##拯 +##拱 +##拳 +##拴 +##拷 +##拼 +##拽 +##拾 +##拿 +##持 +##挂 +##指 +##挈 +##按 +##挎 +##挑 +##挖 +##挙 +##挚 +##挛 +##挝 +##挞 +##挟 +##挠 +##挡 +##挣 +##挤 +##挥 +##挨 +##挪 +##挫 +##振 +##挲 +##挹 +##挺 +##挽 +##挾 +##捂 +##捅 +##捆 +##捉 +##捋 +##捌 +##捍 +##捎 +##捏 +##捐 +##捕 +##捞 +##损 +##捡 +##换 +##捣 +##捧 +##捨 +##捩 +##据 +##捱 +##捲 +##捶 +##捷 +##捺 +##捻 +##掀 +##掂 +##掃 +##掇 +##授 +##掉 +##掌 +##掏 +##掐 +##排 +##掖 +##掘 +##掙 +##掛 +##掠 +##採 +##探 +##掣 +##接 +##控 +##推 +##掩 +##措 +##掬 +##掰 +##掲 +##掳 +##掴 +##掷 +##掸 +##掺 +##揀 +##揃 +##揄 +##揆 +##揉 +##揍 +##描 +##提 +##插 +##揖 +##揚 +##換 +##握 +##揣 +##揩 +##揪 +##揭 +##揮 +##援 +##揶 +##揸 +##揹 +##揽 +##搀 +##搁 +##搂 +##搅 +##損 +##搏 +##搐 +##搓 +##搔 +##搖 +##搗 +##搜 +##搞 +##搡 +##搪 +##搬 +##搭 +##搵 +##搶 +##携 +##搽 +##摀 +##摁 +##摄 +##摆 +##摇 +##摈 +##摊 +##摒 +##摔 +##摘 +##摞 +##摟 +##摧 +##摩 +##摯 +##摳 +##摸 +##摹 +##摺 +##摻 +##撂 +##撃 +##撅 +##撇 +##撈 +##撐 +##撑 +##撒 +##撓 +##撕 +##撚 +##撞 +##撤 +##撥 +##撩 +##撫 +##撬 +##播 +##撮 +##撰 +##撲 +##撵 +##撷 +##撸 +##撻 +##撼 +##撿 +##擀 +##擁 +##擂 +##擄 +##擅 +##擇 +##擊 +##擋 +##操 +##擎 +##擒 +##擔 +##擘 +##據 +##擞 +##擠 +##擡 +##擢 +##擦 +##擬 +##擰 +##擱 +##擲 +##擴 +##擷 +##擺 +##擼 +##擾 +##攀 +##攏 +##攒 +##攔 +##攘 +##攙 +##攜 +##攝 +##攞 +##攢 +##攣 +##攤 +##攥 +##攪 +##攫 +##攬 +##支 +##收 +##攸 +##改 +##攻 +##放 +##政 +##故 +##效 +##敌 +##敍 +##敎 +##敏 +##救 +##敕 +##敖 +##敗 +##敘 +##教 +##敛 +##敝 +##敞 +##敢 +##散 +##敦 +##敬 +##数 +##敲 +##整 +##敵 +##敷 +##數 +##斂 +##斃 +##文 +##斋 +##斌 +##斎 +##斐 +##斑 +##斓 +##斗 +##料 +##斛 +##斜 +##斟 +##斡 +##斤 +##斥 +##斧 +##斩 +##斫 +##斬 +##断 +##斯 +##新 +##斷 +##方 +##於 +##施 +##旁 +##旃 +##旅 +##旋 +##旌 +##旎 +##族 +##旖 +##旗 +##无 +##既 +##日 +##旦 +##旧 +##旨 +##早 +##旬 +##旭 +##旮 +##旱 +##时 +##旷 +##旺 +##旻 +##昀 +##昂 +##昆 +##昇 +##昉 +##昊 +##昌 +##明 +##昏 +##易 +##昔 +##昕 +##昙 +##星 +##映 +##春 +##昧 +##昨 +##昭 +##是 +##昱 +##昴 +##昵 +##昶 +##昼 +##显 +##晁 +##時 +##晃 +##晉 +##晋 +##晌 +##晏 +##晒 +##晓 +##晔 +##晕 +##晖 +##晗 +##晚 +##晝 +##晞 +##晟 +##晤 +##晦 +##晨 +##晩 +##普 +##景 +##晰 +##晴 +##晶 +##晷 +##智 +##晾 +##暂 +##暄 +##暇 +##暈 +##暉 +##暌 +##暐 +##暑 +##暖 +##暗 +##暝 +##暢 +##暧 +##暨 +##暫 +##暮 +##暱 +##暴 +##暸 +##暹 +##曄 +##曆 +##曇 +##曉 +##曖 +##曙 +##曜 +##曝 +##曠 +##曦 +##曬 +##曰 +##曲 +##曳 +##更 +##書 +##曹 +##曼 +##曾 +##替 +##最 +##會 +##月 +##有 +##朋 +##服 +##朐 +##朔 +##朕 +##朗 +##望 +##朝 +##期 +##朦 +##朧 +##木 +##未 +##末 +##本 +##札 +##朮 +##术 +##朱 +##朴 +##朵 +##机 +##朽 +##杀 +##杂 +##权 +##杆 +##杈 +##杉 +##李 +##杏 +##材 +##村 +##杓 +##杖 +##杜 +##杞 +##束 +##杠 +##条 +##来 +##杨 +##杭 +##杯 +##杰 +##東 +##杳 +##杵 +##杷 +##杼 +##松 +##板 +##极 +##构 +##枇 +##枉 +##枋 +##析 +##枕 +##林 +##枚 +##果 +##枝 +##枢 +##枣 +##枪 +##枫 +##枭 +##枯 +##枰 +##枱 +##枳 +##架 +##枷 +##枸 +##柄 +##柏 +##某 +##柑 +##柒 +##染 +##柔 +##柘 +##柚 +##柜 +##柞 +##柠 +##柢 +##查 +##柩 +##柬 +##柯 +##柱 +##柳 +##柴 +##柵 +##査 +##柿 +##栀 +##栃 +##栄 +##栅 +##标 +##栈 +##栉 +##栋 +##栎 +##栏 +##树 +##栓 +##栖 +##栗 +##校 +##栩 +##株 +##样 +##核 +##根 +##格 +##栽 +##栾 +##桀 +##桁 +##桂 +##桃 +##桅 +##框 +##案 +##桉 +##桌 +##桎 +##桐 +##桑 +##桓 +##桔 +##桜 +##桠 +##桡 +##桢 +##档 +##桥 +##桦 +##桧 +##桨 +##桩 +##桶 +##桿 +##梁 +##梅 +##梆 +##梏 +##梓 +##梗 +##條 +##梟 +##梢 +##梦 +##梧 +##梨 +##梭 +##梯 +##械 +##梳 +##梵 +##梶 +##检 +##棂 +##棄 +##棉 +##棋 +##棍 +##棒 +##棕 +##棗 +##棘 +##棚 +##棟 +##棠 +##棣 +##棧 +##森 +##棱 +##棲 +##棵 +##棹 +##棺 +##椁 +##椅 +##椋 +##植 +##椎 +##椒 +##検 +##椪 +##椭 +##椰 +##椹 +##椽 +##椿 +##楂 +##楊 +##楓 +##楔 +##楚 +##楝 +##楞 +##楠 +##楣 +##楨 +##楫 +##業 +##楮 +##極 +##楷 +##楸 +##楹 +##楼 +##楽 +##概 +##榄 +##榆 +##榈 +##榉 +##榔 +##榕 +##榖 +##榛 +##榜 +##榨 +##榫 +##榭 +##榮 +##榱 +##榴 +##榷 +##榻 +##槁 +##槃 +##構 +##槌 +##槍 +##槎 +##槐 +##槓 +##様 +##槛 +##槟 +##槤 +##槭 +##槲 +##槳 +##槻 +##槽 +##槿 +##樁 +##樂 +##樊 +##樑 +##樓 +##標 +##樞 +##樟 +##模 +##樣 +##権 +##横 +##樫 +##樯 +##樱 +##樵 +##樸 +##樹 +##樺 +##樽 +##樾 +##橄 +##橇 +##橋 +##橐 +##橘 +##橙 +##機 +##橡 +##橢 +##橫 +##橱 +##橹 +##橼 +##檀 +##檄 +##檎 +##檐 +##檔 +##檗 +##檜 +##檢 +##檬 +##檯 +##檳 +##檸 +##檻 +##櫃 +##櫚 +##櫛 +##櫥 +##櫸 +##櫻 +##欄 +##權 +##欒 +##欖 +##欠 +##次 +##欢 +##欣 +##欧 +##欲 +##欸 +##欺 +##欽 +##款 +##歆 +##歇 +##歉 +##歌 +##歎 +##歐 +##歓 +##歙 +##歛 +##歡 +##止 +##正 +##此 +##步 +##武 +##歧 +##歩 +##歪 +##歯 +##歲 +##歳 +##歴 +##歷 +##歸 +##歹 +##死 +##歼 +##殁 +##殃 +##殆 +##殇 +##殉 +##殊 +##残 +##殒 +##殓 +##殖 +##殘 +##殞 +##殡 +##殤 +##殭 +##殯 +##殲 +##殴 +##段 +##殷 +##殺 +##殼 +##殿 +##毀 +##毁 +##毂 +##毅 +##毆 +##毋 +##母 +##毎 +##每 +##毒 +##毓 +##比 +##毕 +##毗 +##毘 +##毙 +##毛 +##毡 +##毫 +##毯 +##毽 +##氈 +##氏 +##氐 +##民 +##氓 +##气 +##氖 +##気 +##氙 +##氛 +##氟 +##氡 +##氢 +##氣 +##氤 +##氦 +##氧 +##氨 +##氪 +##氫 +##氮 +##氯 +##氰 +##氲 +##水 +##氷 +##永 +##氹 +##氾 +##汀 +##汁 +##求 +##汆 +##汇 +##汉 +##汎 +##汐 +##汕 +##汗 +##汙 +##汛 +##汝 +##汞 +##江 +##池 +##污 +##汤 +##汨 +##汩 +##汪 +##汰 +##汲 +##汴 +##汶 +##汹 +##決 +##汽 +##汾 +##沁 +##沂 +##沃 +##沅 +##沈 +##沉 +##沌 +##沏 +##沐 +##沒 +##沓 +##沖 +##沙 +##沛 +##沟 +##没 +##沢 +##沣 +##沥 +##沦 +##沧 +##沪 +##沫 +##沭 +##沮 +##沱 +##河 +##沸 +##油 +##治 +##沼 +##沽 +##沾 +##沿 +##況 +##泄 +##泉 +##泊 +##泌 +##泓 +##法 +##泗 +##泛 +##泞 +##泠 +##泡 +##波 +##泣 +##泥 +##注 +##泪 +##泫 +##泮 +##泯 +##泰 +##泱 +##泳 +##泵 +##泷 +##泸 +##泻 +##泼 +##泽 +##泾 +##洁 +##洄 +##洋 +##洒 +##洗 +##洙 +##洛 +##洞 +##津 +##洩 +##洪 +##洮 +##洱 +##洲 +##洵 +##洶 +##洸 +##洹 +##活 +##洼 +##洽 +##派 +##流 +##浃 +##浄 +##浅 +##浆 +##浇 +##浊 +##测 +##济 +##浏 +##浑 +##浒 +##浓 +##浔 +##浙 +##浚 +##浜 +##浣 +##浦 +##浩 +##浪 +##浬 +##浮 +##浯 +##浴 +##海 +##浸 +##涂 +##涅 +##涇 +##消 +##涉 +##涌 +##涎 +##涓 +##涔 +##涕 +##涙 +##涛 +##涝 +##涞 +##涟 +##涠 +##涡 +##涣 +##涤 +##润 +##涧 +##涨 +##涩 +##涪 +##涮 +##涯 +##液 +##涵 +##涸 +##涼 +##涿 +##淀 +##淄 +##淅 +##淆 +##淇 +##淋 +##淌 +##淑 +##淒 +##淖 +##淘 +##淙 +##淚 +##淞 +##淡 +##淤 +##淦 +##淨 +##淩 +##淪 +##淫 +##淬 +##淮 +##深 +##淳 +##淵 +##混 +##淹 +##淺 +##添 +##淼 +##清 +##済 +##渉 +##渊 +##渋 +##渍 +##渎 +##渐 +##渔 +##渗 +##渙 +##渚 +##減 +##渝 +##渠 +##渡 +##渣 +##渤 +##渥 +##渦 +##温 +##測 +##渭 +##港 +##渲 +##渴 +##游 +##渺 +##渾 +##湃 +##湄 +##湊 +##湍 +##湖 +##湘 +##湛 +##湟 +##湧 +##湫 +##湮 +##湯 +##湳 +##湾 +##湿 +##満 +##溃 +##溅 +##溉 +##溏 +##源 +##準 +##溜 +##溝 +##溟 +##溢 +##溥 +##溧 +##溪 +##溫 +##溯 +##溱 +##溴 +##溶 +##溺 +##溼 +##滁 +##滂 +##滄 +##滅 +##滇 +##滋 +##滌 +##滑 +##滓 +##滔 +##滕 +##滙 +##滚 +##滝 +##滞 +##滟 +##满 +##滢 +##滤 +##滥 +##滦 +##滨 +##滩 +##滬 +##滯 +##滲 +##滴 +##滷 +##滸 +##滾 +##滿 +##漁 +##漂 +##漆 +##漉 +##漏 +##漓 +##演 +##漕 +##漠 +##漢 +##漣 +##漩 +##漪 +##漫 +##漬 +##漯 +##漱 +##漲 +##漳 +##漸 +##漾 +##漿 +##潆 +##潇 +##潋 +##潍 +##潑 +##潔 +##潘 +##潛 +##潜 +##潞 +##潟 +##潢 +##潤 +##潦 +##潧 +##潭 +##潮 +##潰 +##潴 +##潸 +##潺 +##潼 +##澀 +##澄 +##澆 +##澈 +##澍 +##澎 +##澗 +##澜 +##澡 +##澤 +##澧 +##澱 +##澳 +##澹 +##激 +##濁 +##濂 +##濃 +##濑 +##濒 +##濕 +##濘 +##濛 +##濟 +##濠 +##濡 +##濤 +##濫 +##濬 +##濮 +##濯 +##濱 +##濺 +##濾 +##瀅 +##瀆 +##瀉 +##瀋 +##瀏 +##瀑 +##瀕 +##瀘 +##瀚 +##瀛 +##瀝 +##瀞 +##瀟 +##瀧 +##瀨 +##瀬 +##瀰 +##瀾 +##灌 +##灏 +##灑 +##灘 +##灝 +##灞 +##灣 +##火 +##灬 +##灭 +##灯 +##灰 +##灵 +##灶 +##灸 +##灼 +##災 +##灾 +##灿 +##炀 +##炁 +##炅 +##炉 +##炊 +##炎 +##炒 +##炔 +##炕 +##炖 +##炙 +##炜 +##炫 +##炬 +##炭 +##炮 +##炯 +##炳 +##炷 +##炸 +##点 +##為 +##炼 +##炽 +##烁 +##烂 +##烃 +##烈 +##烊 +##烏 +##烘 +##烙 +##烛 +##烟 +##烤 +##烦 +##烧 +##烨 +##烩 +##烫 +##烬 +##热 +##烯 +##烷 +##烹 +##烽 +##焉 +##焊 +##焕 +##焖 +##焗 +##焘 +##焙 +##焚 +##焜 +##無 +##焦 +##焯 +##焰 +##焱 +##然 +##焼 +##煅 +##煉 +##煊 +##煌 +##煎 +##煒 +##煖 +##煙 +##煜 +##煞 +##煤 +##煥 +##煦 +##照 +##煨 +##煩 +##煮 +##煲 +##煸 +##煽 +##熄 +##熊 +##熏 +##熒 +##熔 +##熙 +##熟 +##熠 +##熨 +##熬 +##熱 +##熵 +##熹 +##熾 +##燁 +##燃 +##燄 +##燈 +##燉 +##燊 +##燎 +##燒 +##燔 +##燕 +##燙 +##燜 +##營 +##燥 +##燦 +##燧 +##燭 +##燮 +##燴 +##燻 +##燼 +##燿 +##爆 +##爍 +##爐 +##爛 +##爪 +##爬 +##爭 +##爰 +##爱 +##爲 +##爵 +##父 +##爷 +##爸 +##爹 +##爺 +##爻 +##爽 +##爾 +##牆 +##片 +##版 +##牌 +##牍 +##牒 +##牙 +##牛 +##牝 +##牟 +##牠 +##牡 +##牢 +##牦 +##牧 +##物 +##牯 +##牲 +##牴 +##牵 +##特 +##牺 +##牽 +##犀 +##犁 +##犄 +##犊 +##犍 +##犒 +##犢 +##犧 +##犬 +##犯 +##状 +##犷 +##犸 +##犹 +##狀 +##狂 +##狄 +##狈 +##狎 +##狐 +##狒 +##狗 +##狙 +##狞 +##狠 +##狡 +##狩 +##独 +##狭 +##狮 +##狰 +##狱 +##狸 +##狹 +##狼 +##狽 +##猎 +##猕 +##猖 +##猗 +##猙 +##猛 +##猜 +##猝 +##猥 +##猩 +##猪 +##猫 +##猬 +##献 +##猴 +##猶 +##猷 +##猾 +##猿 +##獄 +##獅 +##獎 +##獐 +##獒 +##獗 +##獠 +##獣 +##獨 +##獭 +##獰 +##獲 +##獵 +##獷 +##獸 +##獺 +##獻 +##獼 +##獾 +##玄 +##率 +##玉 +##王 +##玑 +##玖 +##玛 +##玟 +##玠 +##玥 +##玩 +##玫 +##玮 +##环 +##现 +##玲 +##玳 +##玷 +##玺 +##玻 +##珀 +##珂 +##珅 +##珈 +##珉 +##珊 +##珍 +##珏 +##珐 +##珑 +##珙 +##珞 +##珠 +##珣 +##珥 +##珩 +##珪 +##班 +##珮 +##珲 +##珺 +##現 +##球 +##琅 +##理 +##琇 +##琉 +##琊 +##琍 +##琏 +##琐 +##琛 +##琢 +##琥 +##琦 +##琨 +##琪 +##琬 +##琮 +##琰 +##琲 +##琳 +##琴 +##琵 +##琶 +##琺 +##琼 +##瑀 +##瑁 +##瑄 +##瑋 +##瑕 +##瑗 +##瑙 +##瑚 +##瑛 +##瑜 +##瑞 +##瑟 +##瑠 +##瑣 +##瑤 +##瑩 +##瑪 +##瑯 +##瑰 +##瑶 +##瑾 +##璀 +##璁 +##璃 +##璇 +##璉 +##璋 +##璎 +##璐 +##璜 +##璞 +##璟 +##璧 +##璨 +##環 +##璽 +##璿 +##瓊 +##瓏 +##瓒 +##瓜 +##瓢 +##瓣 +##瓤 +##瓦 +##瓮 +##瓯 +##瓴 +##瓶 +##瓷 +##甄 +##甌 +##甕 +##甘 +##甙 +##甚 +##甜 +##生 +##產 +##産 +##甥 +##甦 +##用 +##甩 +##甫 +##甬 +##甭 +##甯 +##田 +##由 +##甲 +##申 +##电 +##男 +##甸 +##町 +##画 +##甾 +##畀 +##畅 +##界 +##畏 +##畑 +##畔 +##留 +##畜 +##畝 +##畢 +##略 +##畦 +##番 +##畫 +##異 +##畲 +##畳 +##畴 +##當 +##畸 +##畹 +##畿 +##疆 +##疇 +##疊 +##疏 +##疑 +##疔 +##疖 +##疗 +##疙 +##疚 +##疝 +##疟 +##疡 +##疣 +##疤 +##疥 +##疫 +##疮 +##疯 +##疱 +##疲 +##疳 +##疵 +##疸 +##疹 +##疼 +##疽 +##疾 +##痂 +##病 +##症 +##痈 +##痉 +##痊 +##痍 +##痒 +##痔 +##痕 +##痘 +##痙 +##痛 +##痞 +##痠 +##痢 +##痣 +##痤 +##痧 +##痨 +##痪 +##痫 +##痰 +##痱 +##痴 +##痹 +##痺 +##痼 +##痿 +##瘀 +##瘁 +##瘋 +##瘍 +##瘓 +##瘘 +##瘙 +##瘟 +##瘠 +##瘡 +##瘢 +##瘤 +##瘦 +##瘧 +##瘩 +##瘪 +##瘫 +##瘴 +##瘸 +##瘾 +##療 +##癇 +##癌 +##癒 +##癖 +##癜 +##癞 +##癡 +##癢 +##癣 +##癥 +##癫 +##癬 +##癮 +##癱 +##癲 +##癸 +##発 +##登 +##發 +##白 +##百 +##皂 +##的 +##皆 +##皇 +##皈 +##皋 +##皎 +##皑 +##皓 +##皖 +##皙 +##皚 +##皮 +##皰 +##皱 +##皴 +##皺 +##皿 +##盂 +##盃 +##盅 +##盆 +##盈 +##益 +##盎 +##盏 +##盐 +##监 +##盒 +##盔 +##盖 +##盗 +##盘 +##盛 +##盜 +##盞 +##盟 +##盡 +##監 +##盤 +##盥 +##盧 +##盪 +##目 +##盯 +##盱 +##盲 +##直 +##相 +##盹 +##盼 +##盾 +##省 +##眈 +##眉 +##看 +##県 +##眙 +##眞 +##真 +##眠 +##眦 +##眨 +##眩 +##眯 +##眶 +##眷 +##眸 +##眺 +##眼 +##眾 +##着 +##睁 +##睇 +##睏 +##睐 +##睑 +##睛 +##睜 +##睞 +##睡 +##睢 +##督 +##睥 +##睦 +##睨 +##睪 +##睫 +##睬 +##睹 +##睽 +##睾 +##睿 +##瞄 +##瞅 +##瞇 +##瞋 +##瞌 +##瞎 +##瞑 +##瞒 +##瞓 +##瞞 +##瞟 +##瞠 +##瞥 +##瞧 +##瞩 +##瞪 +##瞬 +##瞭 +##瞰 +##瞳 +##瞻 +##瞼 +##瞿 +##矇 +##矍 +##矗 +##矚 +##矛 +##矜 +##矢 +##矣 +##知 +##矩 +##矫 +##短 +##矮 +##矯 +##石 +##矶 +##矽 +##矾 +##矿 +##码 +##砂 +##砌 +##砍 +##砒 +##研 +##砖 +##砗 +##砚 +##砝 +##砣 +##砥 +##砧 +##砭 +##砰 +##砲 +##破 +##砷 +##砸 +##砺 +##砼 +##砾 +##础 +##硅 +##硐 +##硒 +##硕 +##硝 +##硫 +##硬 +##确 +##硯 +##硼 +##碁 +##碇 +##碉 +##碌 +##碍 +##碎 +##碑 +##碓 +##碗 +##碘 +##碚 +##碛 +##碟 +##碣 +##碧 +##碩 +##碰 +##碱 +##碳 +##碴 +##確 +##碼 +##碾 +##磁 +##磅 +##磊 +##磋 +##磐 +##磕 +##磚 +##磡 +##磨 +##磬 +##磯 +##磲 +##磷 +##磺 +##礁 +##礎 +##礙 +##礡 +##礦 +##礪 +##礫 +##礴 +##示 +##礼 +##社 +##祀 +##祁 +##祂 +##祇 +##祈 +##祉 +##祎 +##祐 +##祕 +##祖 +##祗 +##祚 +##祛 +##祜 +##祝 +##神 +##祟 +##祠 +##祢 +##祥 +##票 +##祭 +##祯 +##祷 +##祸 +##祺 +##祿 +##禀 +##禁 +##禄 +##禅 +##禍 +##禎 +##福 +##禛 +##禦 +##禧 +##禪 +##禮 +##禱 +##禹 +##禺 +##离 +##禽 +##禾 +##禿 +##秀 +##私 +##秃 +##秆 +##秉 +##秋 +##种 +##科 +##秒 +##秘 +##租 +##秣 +##秤 +##秦 +##秧 +##秩 +##秭 +##积 +##称 +##秸 +##移 +##秽 +##稀 +##稅 +##程 +##稍 +##税 +##稔 +##稗 +##稚 +##稜 +##稞 +##稟 +##稠 +##稣 +##種 +##稱 +##稲 +##稳 +##稷 +##稹 +##稻 +##稼 +##稽 +##稿 +##穀 +##穂 +##穆 +##穌 +##積 +##穎 +##穗 +##穢 +##穩 +##穫 +##穴 +##究 +##穷 +##穹 +##空 +##穿 +##突 +##窃 +##窄 +##窈 +##窍 +##窑 +##窒 +##窓 +##窕 +##窖 +##窗 +##窘 +##窜 +##窝 +##窟 +##窠 +##窥 +##窦 +##窨 +##窩 +##窪 +##窮 +##窯 +##窺 +##窿 +##竄 +##竅 +##竇 +##竊 +##立 +##竖 +##站 +##竜 +##竞 +##竟 +##章 +##竣 +##童 +##竭 +##端 +##競 +##竹 +##竺 +##竽 +##竿 +##笃 +##笆 +##笈 +##笋 +##笏 +##笑 +##笔 +##笙 +##笛 +##笞 +##笠 +##符 +##笨 +##第 +##笹 +##笺 +##笼 +##筆 +##等 +##筊 +##筋 +##筍 +##筏 +##筐 +##筑 +##筒 +##答 +##策 +##筛 +##筝 +##筠 +##筱 +##筲 +##筵 +##筷 +##筹 +##签 +##简 +##箇 +##箋 +##箍 +##箏 +##箐 +##箔 +##箕 +##算 +##箝 +##管 +##箩 +##箫 +##箭 +##箱 +##箴 +##箸 +##節 +##篁 +##範 +##篆 +##篇 +##築 +##篑 +##篓 +##篙 +##篝 +##篠 +##篡 +##篤 +##篩 +##篪 +##篮 +##篱 +##篷 +##簇 +##簌 +##簍 +##簡 +##簦 +##簧 +##簪 +##簫 +##簷 +##簸 +##簽 +##簾 +##簿 +##籁 +##籃 +##籌 +##籍 +##籐 +##籟 +##籠 +##籤 +##籬 +##籮 +##籲 +##米 +##类 +##籼 +##籽 +##粄 +##粉 +##粑 +##粒 +##粕 +##粗 +##粘 +##粟 +##粤 +##粥 +##粧 +##粪 +##粮 +##粱 +##粲 +##粳 +##粵 +##粹 +##粼 +##粽 +##精 +##粿 +##糅 +##糊 +##糍 +##糕 +##糖 +##糗 +##糙 +##糜 +##糞 +##糟 +##糠 +##糧 +##糬 +##糯 +##糰 +##糸 +##系 +##糾 +##紀 +##紂 +##約 +##紅 +##紉 +##紊 +##紋 +##納 +##紐 +##紓 +##純 +##紗 +##紘 +##紙 +##級 +##紛 +##紜 +##素 +##紡 +##索 +##紧 +##紫 +##紮 +##累 +##細 +##紳 +##紹 +##紺 +##終 +##絃 +##組 +##絆 +##経 +##結 +##絕 +##絞 +##絡 +##絢 +##給 +##絨 +##絮 +##統 +##絲 +##絳 +##絵 +##絶 +##絹 +##綁 +##綏 +##綑 +##經 +##継 +##続 +##綜 +##綠 +##綢 +##綦 +##綫 +##綬 +##維 +##綱 +##網 +##綴 +##綵 +##綸 +##綺 +##綻 +##綽 +##綾 +##綿 +##緊 +##緋 +##総 +##緑 +##緒 +##緘 +##線 +##緝 +##緞 +##締 +##緣 +##編 +##緩 +##緬 +##緯 +##練 +##緹 +##緻 +##縁 +##縄 +##縈 +##縛 +##縝 +##縣 +##縫 +##縮 +##縱 +##縴 +##縷 +##總 +##績 +##繁 +##繃 +##繆 +##繇 +##繋 +##織 +##繕 +##繚 +##繞 +##繡 +##繩 +##繪 +##繫 +##繭 +##繳 +##繹 +##繼 +##繽 +##纂 +##續 +##纍 +##纏 +##纓 +##纔 +##纖 +##纜 +##纠 +##红 +##纣 +##纤 +##约 +##级 +##纨 +##纪 +##纫 +##纬 +##纭 +##纯 +##纰 +##纱 +##纲 +##纳 +##纵 +##纶 +##纷 +##纸 +##纹 +##纺 +##纽 +##纾 +##线 +##绀 +##练 +##组 +##绅 +##细 +##织 +##终 +##绊 +##绍 +##绎 +##经 +##绑 +##绒 +##结 +##绔 +##绕 +##绘 +##给 +##绚 +##绛 +##络 +##绝 +##绞 +##统 +##绡 +##绢 +##绣 +##绥 +##绦 +##继 +##绩 +##绪 +##绫 +##续 +##绮 +##绯 +##绰 +##绳 +##维 +##绵 +##绶 +##绷 +##绸 +##绻 +##综 +##绽 +##绾 +##绿 +##缀 +##缄 +##缅 +##缆 +##缇 +##缈 +##缉 +##缎 +##缓 +##缔 +##缕 +##编 +##缘 +##缙 +##缚 +##缜 +##缝 +##缠 +##缢 +##缤 +##缥 +##缨 +##缩 +##缪 +##缭 +##缮 +##缰 +##缱 +##缴 +##缸 +##缺 +##缽 +##罂 +##罄 +##罌 +##罐 +##网 +##罔 +##罕 +##罗 +##罚 +##罡 +##罢 +##罩 +##罪 +##置 +##罰 +##署 +##罵 +##罷 +##罹 +##羁 +##羅 +##羈 +##羊 +##羌 +##美 +##羔 +##羚 +##羞 +##羟 +##羡 +##羣 +##群 +##羥 +##羧 +##羨 +##義 +##羯 +##羲 +##羸 +##羹 +##羽 +##羿 +##翁 +##翅 +##翊 +##翌 +##翎 +##習 +##翔 +##翘 +##翟 +##翠 +##翡 +##翦 +##翩 +##翰 +##翱 +##翳 +##翹 +##翻 +##翼 +##耀 +##老 +##考 +##耄 +##者 +##耆 +##耋 +##而 +##耍 +##耐 +##耒 +##耕 +##耗 +##耘 +##耙 +##耦 +##耨 +##耳 +##耶 +##耷 +##耸 +##耻 +##耽 +##耿 +##聂 +##聆 +##聊 +##聋 +##职 +##聒 +##联 +##聖 +##聘 +##聚 +##聞 +##聪 +##聯 +##聰 +##聲 +##聳 +##聴 +##聶 +##職 +##聽 +##聾 +##聿 +##肃 +##肄 +##肅 +##肆 +##肇 +##肉 +##肋 +##肌 +##肏 +##肓 +##肖 +##肘 +##肚 +##肛 +##肝 +##肠 +##股 +##肢 +##肤 +##肥 +##肩 +##肪 +##肮 +##肯 +##肱 +##育 +##肴 +##肺 +##肽 +##肾 +##肿 +##胀 +##胁 +##胃 +##胄 +##胆 +##背 +##胍 +##胎 +##胖 +##胚 +##胛 +##胜 +##胝 +##胞 +##胡 +##胤 +##胥 +##胧 +##胫 +##胭 +##胯 +##胰 +##胱 +##胳 +##胴 +##胶 +##胸 +##胺 +##能 +##脂 +##脅 +##脆 +##脇 +##脈 +##脉 +##脊 +##脍 +##脏 +##脐 +##脑 +##脓 +##脖 +##脘 +##脚 +##脛 +##脣 +##脩 +##脫 +##脯 +##脱 +##脲 +##脳 +##脸 +##脹 +##脾 +##腆 +##腈 +##腊 +##腋 +##腌 +##腎 +##腐 +##腑 +##腓 +##腔 +##腕 +##腥 +##腦 +##腩 +##腫 +##腭 +##腮 +##腰 +##腱 +##腳 +##腴 +##腸 +##腹 +##腺 +##腻 +##腼 +##腾 +##腿 +##膀 +##膈 +##膊 +##膏 +##膑 +##膘 +##膚 +##膛 +##膜 +##膝 +##膠 +##膦 +##膨 +##膩 +##膳 +##膺 +##膻 +##膽 +##膾 +##膿 +##臀 +##臂 +##臃 +##臆 +##臉 +##臊 +##臍 +##臓 +##臘 +##臟 +##臣 +##臥 +##臧 +##臨 +##自 +##臬 +##臭 +##至 +##致 +##臺 +##臻 +##臼 +##臾 +##舀 +##舂 +##舅 +##舆 +##與 +##興 +##舉 +##舊 +##舌 +##舍 +##舎 +##舐 +##舒 +##舔 +##舖 +##舗 +##舛 +##舜 +##舞 +##舟 +##航 +##舫 +##般 +##舰 +##舱 +##舵 +##舶 +##舷 +##舸 +##船 +##舺 +##舾 +##艇 +##艋 +##艘 +##艙 +##艦 +##艮 +##良 +##艰 +##艱 +##色 +##艳 +##艷 +##艹 +##艺 +##艾 +##节 +##芃 +##芈 +##芊 +##芋 +##芍 +##芎 +##芒 +##芙 +##芜 +##芝 +##芡 +##芥 +##芦 +##芩 +##芪 +##芫 +##芬 +##芭 +##芮 +##芯 +##花 +##芳 +##芷 +##芸 +##芹 +##芻 +##芽 +##芾 +##苁 +##苄 +##苇 +##苋 +##苍 +##苏 +##苑 +##苒 +##苓 +##苔 +##苕 +##苗 +##苛 +##苜 +##苞 +##苟 +##苡 +##苣 +##若 +##苦 +##苫 +##苯 +##英 +##苷 +##苹 +##苻 +##茁 +##茂 +##范 +##茄 +##茅 +##茉 +##茎 +##茏 +##茗 +##茜 +##茧 +##茨 +##茫 +##茬 +##茭 +##茯 +##茱 +##茲 +##茴 +##茵 +##茶 +##茸 +##茹 +##茼 +##荀 +##荃 +##荆 +##草 +##荊 +##荏 +##荐 +##荒 +##荔 +##荖 +##荘 +##荚 +##荞 +##荟 +##荠 +##荡 +##荣 +##荤 +##荥 +##荧 +##荨 +##荪 +##荫 +##药 +##荳 +##荷 +##荸 +##荻 +##荼 +##荽 +##莅 +##莆 +##莉 +##莊 +##莎 +##莒 +##莓 +##莖 +##莘 +##莞 +##莠 +##莢 +##莧 +##莪 +##莫 +##莱 +##莲 +##莴 +##获 +##莹 +##莺 +##莽 +##莿 +##菀 +##菁 +##菅 +##菇 +##菈 +##菊 +##菌 +##菏 +##菓 +##菖 +##菘 +##菜 +##菟 +##菠 +##菡 +##菩 +##華 +##菱 +##菲 +##菸 +##菽 +##萁 +##萃 +##萄 +##萊 +##萋 +##萌 +##萍 +##萎 +##萘 +##萝 +##萤 +##营 +##萦 +##萧 +##萨 +##萩 +##萬 +##萱 +##萵 +##萸 +##萼 +##落 +##葆 +##葉 +##著 +##葚 +##葛 +##葡 +##董 +##葦 +##葩 +##葫 +##葬 +##葭 +##葯 +##葱 +##葳 +##葵 +##葷 +##葺 +##蒂 +##蒋 +##蒐 +##蒔 +##蒙 +##蒜 +##蒞 +##蒟 +##蒡 +##蒨 +##蒲 +##蒸 +##蒹 +##蒻 +##蒼 +##蒿 +##蓁 +##蓄 +##蓆 +##蓉 +##蓋 +##蓑 +##蓓 +##蓖 +##蓝 +##蓟 +##蓦 +##蓬 +##蓮 +##蓼 +##蓿 +##蔑 +##蔓 +##蔔 +##蔗 +##蔘 +##蔚 +##蔡 +##蔣 +##蔥 +##蔫 +##蔬 +##蔭 +##蔵 +##蔷 +##蔺 +##蔻 +##蔼 +##蔽 +##蕁 +##蕃 +##蕈 +##蕉 +##蕊 +##蕎 +##蕙 +##蕤 +##蕨 +##蕩 +##蕪 +##蕭 +##蕲 +##蕴 +##蕻 +##蕾 +##薄 +##薅 +##薇 +##薈 +##薊 +##薏 +##薑 +##薔 +##薙 +##薛 +##薦 +##薨 +##薩 +##薪 +##薬 +##薯 +##薰 +##薹 +##藉 +##藍 +##藏 +##藐 +##藓 +##藕 +##藜 +##藝 +##藤 +##藥 +##藩 +##藹 +##藻 +##藿 +##蘆 +##蘇 +##蘊 +##蘋 +##蘑 +##蘚 +##蘭 +##蘸 +##蘼 +##蘿 +##虎 +##虏 +##虐 +##虑 +##虔 +##處 +##虚 +##虛 +##虜 +##虞 +##號 +##虢 +##虧 +##虫 +##虬 +##虱 +##虹 +##虻 +##虽 +##虾 +##蚀 +##蚁 +##蚂 +##蚊 +##蚌 +##蚓 +##蚕 +##蚜 +##蚝 +##蚣 +##蚤 +##蚩 +##蚪 +##蚯 +##蚱 +##蚵 +##蛀 +##蛆 +##蛇 +##蛊 +##蛋 +##蛎 +##蛐 +##蛔 +##蛙 +##蛛 +##蛟 +##蛤 +##蛭 +##蛮 +##蛰 +##蛳 +##蛹 +##蛻 +##蛾 +##蜀 +##蜂 +##蜃 +##蜆 +##蜇 +##蜈 +##蜊 +##蜍 +##蜒 +##蜓 +##蜕 +##蜗 +##蜘 +##蜚 +##蜜 +##蜡 +##蜢 +##蜥 +##蜱 +##蜴 +##蜷 +##蜻 +##蜿 +##蝇 +##蝈 +##蝉 +##蝌 +##蝎 +##蝕 +##蝗 +##蝙 +##蝟 +##蝠 +##蝦 +##蝨 +##蝴 +##蝶 +##蝸 +##蝼 +##螂 +##螃 +##融 +##螞 +##螢 +##螨 +##螯 +##螳 +##螺 +##蟀 +##蟄 +##蟆 +##蟋 +##蟎 +##蟑 +##蟒 +##蟠 +##蟬 +##蟲 +##蟹 +##蟻 +##蟾 +##蠅 +##蠍 +##蠔 +##蠕 +##蠛 +##蠟 +##蠡 +##蠢 +##蠣 +##蠱 +##蠶 +##蠹 +##蠻 +##血 +##衄 +##衅 +##衆 +##行 +##衍 +##術 +##衔 +##街 +##衙 +##衛 +##衝 +##衞 +##衡 +##衢 +##衣 +##补 +##表 +##衩 +##衫 +##衬 +##衮 +##衰 +##衲 +##衷 +##衹 +##衾 +##衿 +##袁 +##袂 +##袄 +##袅 +##袈 +##袋 +##袍 +##袒 +##袖 +##袜 +##袞 +##袤 +##袪 +##被 +##袭 +##袱 +##裁 +##裂 +##装 +##裆 +##裊 +##裏 +##裔 +##裕 +##裘 +##裙 +##補 +##裝 +##裟 +##裡 +##裤 +##裨 +##裱 +##裳 +##裴 +##裸 +##裹 +##製 +##裾 +##褂 +##複 +##褐 +##褒 +##褓 +##褔 +##褚 +##褥 +##褪 +##褫 +##褲 +##褶 +##褻 +##襁 +##襄 +##襟 +##襠 +##襪 +##襬 +##襯 +##襲 +##西 +##要 +##覃 +##覆 +##覇 +##見 +##規 +##覓 +##視 +##覚 +##覦 +##覧 +##親 +##覬 +##観 +##覷 +##覺 +##覽 +##觀 +##见 +##观 +##规 +##觅 +##视 +##览 +##觉 +##觊 +##觎 +##觐 +##觑 +##角 +##觞 +##解 +##觥 +##触 +##觸 +##言 +##訂 +##計 +##訊 +##討 +##訓 +##訕 +##訖 +##託 +##記 +##訛 +##訝 +##訟 +##訣 +##訥 +##訪 +##設 +##許 +##訳 +##訴 +##訶 +##診 +##註 +##証 +##詆 +##詐 +##詔 +##評 +##詛 +##詞 +##詠 +##詡 +##詢 +##詣 +##試 +##詩 +##詫 +##詬 +##詭 +##詮 +##詰 +##話 +##該 +##詳 +##詹 +##詼 +##誅 +##誇 +##誉 +##誌 +##認 +##誓 +##誕 +##誘 +##語 +##誠 +##誡 +##誣 +##誤 +##誥 +##誦 +##誨 +##說 +##説 +##読 +##誰 +##課 +##誹 +##誼 +##調 +##諄 +##談 +##請 +##諏 +##諒 +##論 +##諗 +##諜 +##諡 +##諦 +##諧 +##諫 +##諭 +##諮 +##諱 +##諳 +##諷 +##諸 +##諺 +##諾 +##謀 +##謁 +##謂 +##謄 +##謊 +##謎 +##謐 +##謔 +##謗 +##謙 +##講 +##謝 +##謠 +##謨 +##謬 +##謹 +##謾 +##譁 +##證 +##譎 +##譏 +##識 +##譙 +##譚 +##譜 +##警 +##譬 +##譯 +##議 +##譲 +##譴 +##護 +##譽 +##讀 +##變 +##讓 +##讚 +##讞 +##计 +##订 +##认 +##讥 +##讧 +##讨 +##让 +##讪 +##讫 +##训 +##议 +##讯 +##记 +##讲 +##讳 +##讴 +##讶 +##讷 +##许 +##讹 +##论 +##讼 +##讽 +##设 +##访 +##诀 +##证 +##诃 +##评 +##诅 +##识 +##诈 +##诉 +##诊 +##诋 +##词 +##诏 +##译 +##试 +##诗 +##诘 +##诙 +##诚 +##诛 +##话 +##诞 +##诟 +##诠 +##诡 +##询 +##诣 +##诤 +##该 +##详 +##诧 +##诩 +##诫 +##诬 +##语 +##误 +##诰 +##诱 +##诲 +##说 +##诵 +##诶 +##请 +##诸 +##诺 +##读 +##诽 +##课 +##诿 +##谀 +##谁 +##调 +##谄 +##谅 +##谆 +##谈 +##谊 +##谋 +##谌 +##谍 +##谎 +##谏 +##谐 +##谑 +##谒 +##谓 +##谔 +##谕 +##谗 +##谘 +##谙 +##谚 +##谛 +##谜 +##谟 +##谢 +##谣 +##谤 +##谥 +##谦 +##谧 +##谨 +##谩 +##谪 +##谬 +##谭 +##谯 +##谱 +##谲 +##谴 +##谶 +##谷 +##豁 +##豆 +##豇 +##豈 +##豉 +##豊 +##豌 +##豎 +##豐 +##豔 +##豚 +##象 +##豢 +##豪 +##豫 +##豬 +##豹 +##豺 +##貂 +##貅 +##貌 +##貓 +##貔 +##貘 +##貝 +##貞 +##負 +##財 +##貢 +##貧 +##貨 +##販 +##貪 +##貫 +##責 +##貯 +##貰 +##貳 +##貴 +##貶 +##買 +##貸 +##費 +##貼 +##貽 +##貿 +##賀 +##賁 +##賂 +##賃 +##賄 +##資 +##賈 +##賊 +##賑 +##賓 +##賜 +##賞 +##賠 +##賡 +##賢 +##賣 +##賤 +##賦 +##質 +##賬 +##賭 +##賴 +##賺 +##購 +##賽 +##贅 +##贈 +##贊 +##贍 +##贏 +##贓 +##贖 +##贛 +##贝 +##贞 +##负 +##贡 +##财 +##责 +##贤 +##败 +##账 +##货 +##质 +##贩 +##贪 +##贫 +##贬 +##购 +##贮 +##贯 +##贰 +##贱 +##贲 +##贴 +##贵 +##贷 +##贸 +##费 +##贺 +##贻 +##贼 +##贾 +##贿 +##赁 +##赂 +##赃 +##资 +##赅 +##赈 +##赊 +##赋 +##赌 +##赎 +##赏 +##赐 +##赓 +##赔 +##赖 +##赘 +##赚 +##赛 +##赝 +##赞 +##赠 +##赡 +##赢 +##赣 +##赤 +##赦 +##赧 +##赫 +##赭 +##走 +##赳 +##赴 +##赵 +##赶 +##起 +##趁 +##超 +##越 +##趋 +##趕 +##趙 +##趟 +##趣 +##趨 +##足 +##趴 +##趵 +##趸 +##趺 +##趾 +##跃 +##跄 +##跆 +##跋 +##跌 +##跎 +##跑 +##跖 +##跚 +##跛 +##距 +##跟 +##跡 +##跤 +##跨 +##跩 +##跪 +##路 +##跳 +##践 +##跷 +##跹 +##跺 +##跻 +##踉 +##踊 +##踌 +##踏 +##踐 +##踝 +##踞 +##踟 +##踢 +##踩 +##踪 +##踮 +##踱 +##踴 +##踵 +##踹 +##蹂 +##蹄 +##蹇 +##蹈 +##蹉 +##蹊 +##蹋 +##蹑 +##蹒 +##蹙 +##蹟 +##蹣 +##蹤 +##蹦 +##蹩 +##蹬 +##蹭 +##蹲 +##蹴 +##蹶 +##蹺 +##蹼 +##蹿 +##躁 +##躇 +##躉 +##躊 +##躋 +##躍 +##躏 +##躪 +##身 +##躬 +##躯 +##躲 +##躺 +##軀 +##車 +##軋 +##軌 +##軍 +##軒 +##軟 +##転 +##軸 +##軼 +##軽 +##軾 +##較 +##載 +##輒 +##輓 +##輔 +##輕 +##輛 +##輝 +##輟 +##輩 +##輪 +##輯 +##輸 +##輻 +##輾 +##輿 +##轄 +##轅 +##轆 +##轉 +##轍 +##轎 +##轟 +##车 +##轧 +##轨 +##轩 +##转 +##轭 +##轮 +##软 +##轰 +##轲 +##轴 +##轶 +##轻 +##轼 +##载 +##轿 +##较 +##辄 +##辅 +##辆 +##辇 +##辈 +##辉 +##辊 +##辍 +##辐 +##辑 +##输 +##辕 +##辖 +##辗 +##辘 +##辙 +##辛 +##辜 +##辞 +##辟 +##辣 +##辦 +##辨 +##辩 +##辫 +##辭 +##辮 +##辯 +##辰 +##辱 +##農 +##边 +##辺 +##辻 +##込 +##辽 +##达 +##迁 +##迂 +##迄 +##迅 +##过 +##迈 +##迎 +##运 +##近 +##返 +##还 +##这 +##进 +##远 +##违 +##连 +##迟 +##迢 +##迤 +##迥 +##迦 +##迩 +##迪 +##迫 +##迭 +##述 +##迴 +##迷 +##迸 +##迹 +##迺 +##追 +##退 +##送 +##适 +##逃 +##逅 +##逆 +##选 +##逊 +##逍 +##透 +##逐 +##递 +##途 +##逕 +##逗 +##這 +##通 +##逛 +##逝 +##逞 +##速 +##造 +##逢 +##連 +##逮 +##週 +##進 +##逵 +##逶 +##逸 +##逻 +##逼 +##逾 +##遁 +##遂 +##遅 +##遇 +##遊 +##運 +##遍 +##過 +##遏 +##遐 +##遑 +##遒 +##道 +##達 +##違 +##遗 +##遙 +##遛 +##遜 +##遞 +##遠 +##遢 +##遣 +##遥 +##遨 +##適 +##遭 +##遮 +##遲 +##遴 +##遵 +##遶 +##遷 +##選 +##遺 +##遼 +##遽 +##避 +##邀 +##邁 +##邂 +##邃 +##還 +##邇 +##邈 +##邊 +##邋 +##邏 +##邑 +##邓 +##邕 +##邛 +##邝 +##邢 +##那 +##邦 +##邨 +##邪 +##邬 +##邮 +##邯 +##邰 +##邱 +##邳 +##邵 +##邸 +##邹 +##邺 +##邻 +##郁 +##郅 +##郊 +##郎 +##郑 +##郜 +##郝 +##郡 +##郢 +##郤 +##郦 +##郧 +##部 +##郫 +##郭 +##郴 +##郵 +##郷 +##郸 +##都 +##鄂 +##鄉 +##鄒 +##鄔 +##鄙 +##鄞 +##鄢 +##鄧 +##鄭 +##鄰 +##鄱 +##鄲 +##鄺 +##酉 +##酊 +##酋 +##酌 +##配 +##酐 +##酒 +##酗 +##酚 +##酝 +##酢 +##酣 +##酥 +##酩 +##酪 +##酬 +##酮 +##酯 +##酰 +##酱 +##酵 +##酶 +##酷 +##酸 +##酿 +##醃 +##醇 +##醉 +##醋 +##醍 +##醐 +##醒 +##醚 +##醛 +##醜 +##醞 +##醣 +##醪 +##醫 +##醬 +##醮 +##醯 +##醴 +##醺 +##釀 +##釁 +##采 +##釉 +##释 +##釋 +##里 +##重 +##野 +##量 +##釐 +##金 +##釗 +##釘 +##釜 +##針 +##釣 +##釦 +##釧 +##釵 +##鈀 +##鈉 +##鈍 +##鈎 +##鈔 +##鈕 +##鈞 +##鈣 +##鈦 +##鈪 +##鈴 +##鈺 +##鈾 +##鉀 +##鉄 +##鉅 +##鉉 +##鉑 +##鉗 +##鉚 +##鉛 +##鉤 +##鉴 +##鉻 +##銀 +##銃 +##銅 +##銑 +##銓 +##銖 +##銘 +##銜 +##銬 +##銭 +##銮 +##銳 +##銷 +##銹 +##鋁 +##鋅 +##鋒 +##鋤 +##鋪 +##鋰 +##鋸 +##鋼 +##錄 +##錐 +##錘 +##錚 +##錠 +##錢 +##錦 +##錨 +##錫 +##錮 +##錯 +##録 +##錳 +##錶 +##鍊 +##鍋 +##鍍 +##鍛 +##鍥 +##鍰 +##鍵 +##鍺 +##鍾 +##鎂 +##鎊 +##鎌 +##鎏 +##鎔 +##鎖 +##鎗 +##鎚 +##鎧 +##鎬 +##鎮 +##鎳 +##鏈 +##鏖 +##鏗 +##鏘 +##鏞 +##鏟 +##鏡 +##鏢 +##鏤 +##鏽 +##鐘 +##鐮 +##鐲 +##鐳 +##鐵 +##鐸 +##鐺 +##鑄 +##鑊 +##鑑 +##鑒 +##鑣 +##鑫 +##鑰 +##鑲 +##鑼 +##鑽 +##鑾 +##鑿 +##针 +##钉 +##钊 +##钎 +##钏 +##钒 +##钓 +##钗 +##钙 +##钛 +##钜 +##钝 +##钞 +##钟 +##钠 +##钡 +##钢 +##钣 +##钤 +##钥 +##钦 +##钧 +##钨 +##钩 +##钮 +##钯 +##钰 +##钱 +##钳 +##钴 +##钵 +##钺 +##钻 +##钼 +##钾 +##钿 +##铀 +##铁 +##铂 +##铃 +##铄 +##铅 +##铆 +##铉 +##铎 +##铐 +##铛 +##铜 +##铝 +##铠 +##铡 +##铢 +##铣 +##铤 +##铨 +##铩 +##铬 +##铭 +##铮 +##铰 +##铲 +##铵 +##银 +##铸 +##铺 +##链 +##铿 +##销 +##锁 +##锂 +##锄 +##锅 +##锆 +##锈 +##锉 +##锋 +##锌 +##锏 +##锐 +##锑 +##错 +##锚 +##锟 +##锡 +##锢 +##锣 +##锤 +##锥 +##锦 +##锭 +##键 +##锯 +##锰 +##锲 +##锵 +##锹 +##锺 +##锻 +##镀 +##镁 +##镂 +##镇 +##镉 +##镌 +##镍 +##镐 +##镑 +##镕 +##镖 +##镗 +##镛 +##镜 +##镣 +##镭 +##镯 +##镰 +##镳 +##镶 +##長 +##长 +##門 +##閃 +##閉 +##開 +##閎 +##閏 +##閑 +##閒 +##間 +##閔 +##閘 +##閡 +##関 +##閣 +##閥 +##閨 +##閩 +##閱 +##閲 +##閹 +##閻 +##閾 +##闆 +##闇 +##闊 +##闌 +##闍 +##闔 +##闕 +##闖 +##闘 +##關 +##闡 +##闢 +##门 +##闪 +##闫 +##闭 +##问 +##闯 +##闰 +##闲 +##间 +##闵 +##闷 +##闸 +##闹 +##闺 +##闻 +##闽 +##闾 +##阀 +##阁 +##阂 +##阅 +##阆 +##阇 +##阈 +##阉 +##阎 +##阐 +##阑 +##阔 +##阕 +##阖 +##阙 +##阚 +##阜 +##队 +##阡 +##阪 +##阮 +##阱 +##防 +##阳 +##阴 +##阵 +##阶 +##阻 +##阿 +##陀 +##陂 +##附 +##际 +##陆 +##陇 +##陈 +##陋 +##陌 +##降 +##限 +##陕 +##陛 +##陝 +##陞 +##陟 +##陡 +##院 +##陣 +##除 +##陨 +##险 +##陪 +##陰 +##陲 +##陳 +##陵 +##陶 +##陷 +##陸 +##険 +##陽 +##隅 +##隆 +##隈 +##隊 +##隋 +##隍 +##階 +##随 +##隐 +##隔 +##隕 +##隘 +##隙 +##際 +##障 +##隠 +##隣 +##隧 +##隨 +##險 +##隱 +##隴 +##隶 +##隸 +##隻 +##隼 +##隽 +##难 +##雀 +##雁 +##雄 +##雅 +##集 +##雇 +##雉 +##雋 +##雌 +##雍 +##雎 +##雏 +##雑 +##雒 +##雕 +##雖 +##雙 +##雛 +##雜 +##雞 +##離 +##難 +##雨 +##雪 +##雯 +##雰 +##雲 +##雳 +##零 +##雷 +##雹 +##電 +##雾 +##需 +##霁 +##霄 +##霆 +##震 +##霈 +##霉 +##霊 +##霍 +##霎 +##霏 +##霑 +##霓 +##霖 +##霜 +##霞 +##霧 +##霭 +##霰 +##露 +##霸 +##霹 +##霽 +##霾 +##靂 +##靄 +##靈 +##青 +##靓 +##靖 +##静 +##靚 +##靛 +##靜 +##非 +##靠 +##靡 +##面 +##靥 +##靦 +##革 +##靳 +##靴 +##靶 +##靼 +##鞅 +##鞋 +##鞍 +##鞏 +##鞑 +##鞘 +##鞠 +##鞣 +##鞦 +##鞭 +##韆 +##韋 +##韌 +##韓 +##韜 +##韦 +##韧 +##韩 +##韬 +##韭 +##音 +##韵 +##韶 +##韻 +##響 +##頁 +##頂 +##頃 +##項 +##順 +##須 +##頌 +##預 +##頑 +##頒 +##頓 +##頗 +##領 +##頜 +##頡 +##頤 +##頫 +##頭 +##頰 +##頷 +##頸 +##頹 +##頻 +##頼 +##顆 +##題 +##額 +##顎 +##顏 +##顔 +##願 +##顛 +##類 +##顧 +##顫 +##顯 +##顱 +##顴 +##页 +##顶 +##顷 +##项 +##顺 +##须 +##顼 +##顽 +##顾 +##顿 +##颁 +##颂 +##预 +##颅 +##领 +##颇 +##颈 +##颉 +##颊 +##颌 +##颍 +##颐 +##频 +##颓 +##颔 +##颖 +##颗 +##题 +##颚 +##颛 +##颜 +##额 +##颞 +##颠 +##颡 +##颢 +##颤 +##颦 +##颧 +##風 +##颯 +##颱 +##颳 +##颶 +##颼 +##飄 +##飆 +##风 +##飒 +##飓 +##飕 +##飘 +##飙 +##飚 +##飛 +##飞 +##食 +##飢 +##飨 +##飩 +##飪 +##飯 +##飲 +##飼 +##飽 +##飾 +##餃 +##餅 +##餉 +##養 +##餌 +##餐 +##餒 +##餓 +##餘 +##餚 +##餛 +##餞 +##餡 +##館 +##餮 +##餵 +##餾 +##饅 +##饈 +##饋 +##饌 +##饍 +##饑 +##饒 +##饕 +##饗 +##饞 +##饥 +##饨 +##饪 +##饬 +##饭 +##饮 +##饯 +##饰 +##饱 +##饲 +##饴 +##饵 +##饶 +##饷 +##饺 +##饼 +##饽 +##饿 +##馀 +##馁 +##馄 +##馅 +##馆 +##馈 +##馋 +##馍 +##馏 +##馒 +##馔 +##首 +##馗 +##香 +##馥 +##馨 +##馬 +##馭 +##馮 +##馳 +##馴 +##駁 +##駄 +##駅 +##駆 +##駐 +##駒 +##駕 +##駛 +##駝 +##駭 +##駱 +##駿 +##騁 +##騎 +##騏 +##験 +##騙 +##騨 +##騰 +##騷 +##驀 +##驅 +##驊 +##驍 +##驒 +##驕 +##驗 +##驚 +##驛 +##驟 +##驢 +##驥 +##马 +##驭 +##驮 +##驯 +##驰 +##驱 +##驳 +##驴 +##驶 +##驷 +##驸 +##驹 +##驻 +##驼 +##驾 +##驿 +##骁 +##骂 +##骄 +##骅 +##骆 +##骇 +##骈 +##骊 +##骋 +##验 +##骏 +##骐 +##骑 +##骗 +##骚 +##骛 +##骜 +##骞 +##骠 +##骡 +##骤 +##骥 +##骧 +##骨 +##骯 +##骰 +##骶 +##骷 +##骸 +##骼 +##髂 +##髅 +##髋 +##髏 +##髒 +##髓 +##體 +##髖 +##高 +##髦 +##髪 +##髮 +##髯 +##髻 +##鬃 +##鬆 +##鬍 +##鬓 +##鬚 +##鬟 +##鬢 +##鬣 +##鬥 +##鬧 +##鬱 +##鬼 +##魁 +##魂 +##魄 +##魅 +##魇 +##魍 +##魏 +##魔 +##魘 +##魚 +##魯 +##魷 +##鮑 +##鮨 +##鮪 +##鮭 +##鮮 +##鯉 +##鯊 +##鯖 +##鯛 +##鯨 +##鯰 +##鯽 +##鰍 +##鰓 +##鰭 +##鰲 +##鰻 +##鰾 +##鱈 +##鱉 +##鱔 +##鱗 +##鱷 +##鱸 +##鱼 +##鱿 +##鲁 +##鲈 +##鲍 +##鲑 +##鲛 +##鲜 +##鲟 +##鲢 +##鲤 +##鲨 +##鲫 +##鲱 +##鲲 +##鲶 +##鲷 +##鲸 +##鳃 +##鳄 +##鳅 +##鳌 +##鳍 +##鳕 +##鳖 +##鳗 +##鳝 +##鳞 +##鳥 +##鳩 +##鳳 +##鳴 +##鳶 +##鴉 +##鴕 +##鴛 +##鴦 +##鴨 +##鴻 +##鴿 +##鵑 +##鵜 +##鵝 +##鵡 +##鵬 +##鵰 +##鵲 +##鶘 +##鶩 +##鶯 +##鶴 +##鷗 +##鷲 +##鷹 +##鷺 +##鸚 +##鸞 +##鸟 +##鸠 +##鸡 +##鸢 +##鸣 +##鸥 +##鸦 +##鸨 +##鸪 +##鸭 +##鸯 +##鸳 +##鸵 +##鸽 +##鸾 +##鸿 +##鹂 +##鹃 +##鹄 +##鹅 +##鹈 +##鹉 +##鹊 +##鹌 +##鹏 +##鹑 +##鹕 +##鹘 +##鹜 +##鹞 +##鹤 +##鹦 +##鹧 +##鹫 +##鹭 +##鹰 +##鹳 +##鹵 +##鹹 +##鹼 +##鹽 +##鹿 +##麂 +##麋 +##麒 +##麓 +##麗 +##麝 +##麟 +##麥 +##麦 +##麩 +##麴 +##麵 +##麸 +##麺 +##麻 +##麼 +##麽 +##麾 +##黃 +##黄 +##黍 +##黎 +##黏 +##黑 +##黒 +##黔 +##默 +##黛 +##黜 +##黝 +##點 +##黠 +##黨 +##黯 +##黴 +##鼋 +##鼎 +##鼐 +##鼓 +##鼠 +##鼬 +##鼹 +##鼻 +##鼾 +##齁 +##齊 +##齋 +##齐 +##齒 +##齡 +##齢 +##齣 +##齦 +##齿 +##龄 +##龅 +##龈 +##龊 +##龋 +##龌 +##龍 +##龐 +##龔 +##龕 +##龙 +##龚 +##龛 +##龜 +##龟 +##︰ +##︱ +##︶ +##︿ +##﹁ +##﹂ +##﹍ +##﹏ +##﹐ +##﹑ +##﹒ +##﹔ +##﹕ +##﹖ +##﹗ +##﹙ +##﹚ +##﹝ +##﹞ +##﹡ +##﹣ +##！ +##＂ +##＃ +##＄ +##％ +##＆ +##＇ +##（ +##） +##＊ +##， +##－ +##． +##／ +##： +##； +##＜ +##？ +##＠ +##［ +##＼ +##］ +##＾ +##＿ +##｀ +##ｆ +##ｈ +##ｊ +##ｕ +##ｗ +##ｚ +##｛ +##｝ +##｡ +##｢ +##｣ +##､ +##･ +##ｯ +##ｰ +##ｲ +##ｸ +##ｼ +##ｽ +##ﾄ +##ﾉ +##ﾌ +##ﾗ +##ﾙ +##ﾝ +##ﾞ +##ﾟ +##￣ +##￥ +##👍 +##🔥 +##😂 +##😎 diff --git a/nlp_data/data/loss_compara_content_sentence.bin b/nlp_data/data/loss_compara_content_sentence.bin new file mode 100644 index 0000000000000000000000000000000000000000..0776751d5627621fb32b47f80011bd5b8aa0caaf Binary files /dev/null and b/nlp_data/data/loss_compara_content_sentence.bin differ diff --git a/nlp_data/data/loss_compara_content_sentence.idx b/nlp_data/data/loss_compara_content_sentence.idx new file mode 100644 index 0000000000000000000000000000000000000000..d980da62711510c7413949e14bf74e9730c06bbb Binary files /dev/null and b/nlp_data/data/loss_compara_content_sentence.idx differ diff --git a/nlp_data/gpt2-merges.txt b/nlp_data/gpt2-merges.txt new file mode 100644 index 0000000000000000000000000000000000000000..226b0752cac7789c48f0cb3ec53eda48b7be36cc --- /dev/null +++ b/nlp_data/gpt2-merges.txt @@ -0,0 +1,50001 @@ +#version: 0.2 +Ġ t +Ġ a +h e +i n +r e +o n +Ġt he +e r +Ġ s +a t +Ġ w +Ġ o +e n +Ġ c +i t +i s +a n +o r +e s +Ġ b +e d +Ġ f +in g +Ġ p +o u +Ġa n +a l +a r +Ġt o +Ġ m +Ġo f +Ġ in +Ġ d +Ġ h +Ġan d +i c +a s +l e +Ġt h +i on +o m +l l +en t +Ġ n +Ġ l +s t +Ġ re +v e +Ġ e +r o +l y +Ġb e +Ġ g +Ġ T +c t +Ġ S +i d +o t +Ġ I +u t +e t +Ġ A +Ġ is +Ġ on +i m +a m +o w +a y +a d +s e +Ġth at +Ġ C +i g +Ġf or +a c +Ġ y +v er +u r +Ġ u +l d +Ġs t +Ġ M +' s +Ġ he +Ġ it +at ion +it h +i r +c e +Ġy ou +i l +Ġ B +Ġw h +o l +Ġ P +Ġw ith +Ġ 1 +t er +c h +Ġa s +Ġw e +Ġ ( +n d +i ll +Ġ D +i f +Ġ 2 +a g +er s +k e +Ġ " +Ġ H +e m +Ġc on +Ġ W +Ġ R +he r +Ġw as +Ġ r +o d +Ġ F +u l +at e +Ġa t +r i +p p +o re +ĠT he +Ġs e +u s +Ġp ro +Ġh a +u m +Ġa re +Ġd e +a in +an d +Ġo r +ig h +es t +is t +a b +r om +Ġ N +t h +Ġc om +Ġ G +u n +o p +0 0 +Ġ L +Ġn ot +es s +Ġe x +Ġ v +re s +Ġ E +e w +it y +an t +Ġb y +e l +o s +or t +o c +q u +Ġf rom +Ġha ve +Ġs u +i ve +ou ld +Ġs h +Ġth is +n t +r a +p e +igh t +ar t +m ent +Ġa l +u st +en d +- - +al l +Ġ O +ac k +Ġc h +Ġ le +i es +re d +ar d +â Ģ +ou t +Ġ J +Ġa b +e ar +i v +al ly +ou r +o st +g h +p t +Ġp l +as t +Ġc an +a k +om e +u d +T he +Ġh is +Ġd o +Ġg o +Ġh as +g e +' t +Ġ U +r ou +Ġs a +Ġ j +Ġb ut +Ġw or +Ġa ll +e ct +Ġ k +am e +Ġw ill +o k +Ġw he +Ġthe y +id e +0 1 +f f +ic h +p l +t her +Ġt r +. . +Ġin t +i e +u re +ag e +Ġn e +i al +a p +in e +ic e +Ġm e +Ġo ut +an s +on e +on g +ion s +Ġwh o +Ġ K +Ġu p +Ġthe ir +Ġa d +Ġ 3 +Ġu s +at ed +ou s +Ġm ore +u e +o g +ĠS t +in d +i ke +Ġs o +im e +p er +. " +b er +i z +a ct +Ġon e +Ġsa id +Ġ - +a re +Ġyou r +c c +ĠT h +Ġc l +e p +a ke +ab le +i p +Ġcon t +Ġwh ich +i a +Ġ im +Ġab out +Ġwe re +ver y +u b +Ġh ad +Ġ en +Ġcom p +, " +ĠI n +Ġu n +Ġa g +i re +ac e +a u +ar y +Ġw ould +as s +r y +Ġ âĢ +c l +o ok +e re +s o +Ġ V +ig n +i b +Ġof f +Ġt e +v en +Ġ Y +i le +o se +it e +or m +Ġ2 01 +Ġre s +Ġm an +Ġp er +Ġo ther +or d +ul t +Ġbe en +Ġl ike +as e +an ce +k s +ay s +ow n +en ce +Ġd is +ct ion +Ġan y +Ġa pp +Ġs p +in t +res s +ation s +a il +Ġ 4 +ic al +Ġthe m +Ġhe r +ou nt +ĠC h +Ġa r +Ġ if +Ġthe re +Ġp e +Ġy ear +a v +Ġm y +Ġs ome +Ġwhe n +ou gh +ac h +Ġth an +r u +on d +ic k +Ġo ver +ve l +Ġ qu +Ċ Ċ +Ġs c +re at +re e +ĠI t +ou nd +p ort +Ġal so +Ġp art +f ter +Ġk n +Ġbe c +Ġt ime +en s +Ġ 5 +op le +Ġwh at +Ġn o +d u +m er +an g +Ġn ew +-- -- +Ġg et +or y +it ion +ing s +Ġj ust +Ġint o +Ġ 0 +ent s +o ve +t e +Ġpe ople +Ġp re +Ġit s +Ġre c +Ġt w +i an +ir st +ar k +or s +Ġwor k +ad e +o b +Ġs he +Ġo ur +w n +in k +l ic +Ġ1 9 +ĠH e +is h +nd er +au se +Ġh im +on s +Ġ [ +Ġ ro +f orm +i ld +at es +ver s +Ġon ly +o ll +Ġs pe +c k +e ll +am p +Ġa cc +Ġb l +i ous +ur n +f t +o od +Ġh ow +he d +Ġ ' +Ġa fter +a w +Ġat t +o v +n e +Ġpl ay +er v +ic t +Ġc ould +it t +Ġa m +Ġf irst +Ġ 6 +Ġa ct +Ġ $ +e c +h ing +u al +u ll +Ġcom m +o y +o ld +c es +at er +Ġf e +Ġbe t +w e +if f +Ġtw o +oc k +Ġb ack +) . +id ent +Ġu nder +rou gh +se l +x t +Ġm ay +rou nd +Ġp o +p h +is s +Ġd es +Ġm ost +Ġd id +Ġad d +j ect +Ġin c +f ore +Ġp ol +on t +Ġag ain +cl ud +ter n +Ġkn ow +Ġne ed +Ġcon s +Ġc o +Ġ . +Ġw ant +Ġse e +Ġ 7 +n ing +i ew +ĠTh is +c ed +Ġe ven +Ġin d +t y +ĠW e +at h +Ġthe se +Ġp r +Ġu se +Ġbec ause +Ġf l +n g +Ġn ow +ĠâĢ ĵ +c om +is e +Ġm ake +Ġthe n +ow er +Ġe very +ĠU n +Ġse c +os s +u ch +Ġe m +Ġ = +ĠR e +i ed +r it +Ġin v +le ct +Ġsu pp +at ing +Ġl ook +m an +pe ct +Ġ 8 +ro w +Ġb u +Ġwhe re +if ic +Ġyear s +i ly +Ġd iff +Ġsh ould +Ġre m +T h +I n +Ġe v +d ay +' re +ri b +Ġre l +s s +Ġde f +Ġr ight +Ġs y +) , +l es +00 0 +he n +Ġth rough +ĠT r +_ _ +Ġw ay +Ġd on +Ġ , +Ġ1 0 +as ed +Ġas s +ub lic +Ġre g +ĠA nd +i x +Ġ very +Ġin clud +ot her +Ġim p +ot h +Ġsu b +ĠâĢ Ķ +Ġbe ing +ar g +ĠW h += = +ib le +Ġdo es +an ge +r am +Ġ 9 +er t +p s +it ed +ation al +Ġb r +Ġd own +Ġman y +ak ing +Ġc all +ur ing +it ies +Ġp h +ic s +al s +Ġde c +at ive +en er +Ġbe fore +il ity +Ġwe ll +Ġm uch +ers on +Ġth ose +Ġsu ch +Ġ ke +Ġ end +ĠB ut +as on +t ing +Ġl ong +e f +Ġth ink +y s +Ġbe l +Ġs m +it s +a x +Ġo wn +Ġpro v +Ġs et +if e +ment s +b le +w ard +Ġsh ow +Ġp res +m s +om et +Ġo b +Ġs ay +ĠS h +t s +f ul +Ġe ff +Ġg u +Ġin st +u nd +re n +c ess +Ġ ent +ĠY ou +Ġgo od +Ġst art +in ce +Ġm ade +t t +st em +ol og +u p +Ġ | +um p +Ġhe l +ver n +ul ar +u ally +Ġa c +Ġm on +Ġl ast +Ġ2 00 +1 0 +Ġst ud +u res +ĠA r +sel f +ar s +mer ic +u es +c y +Ġm in +oll ow +Ġc ol +i o +Ġm od +Ġc ount +ĠC om +he s +Ġf in +a ir +i er +âĢ Ķ +re ad +an k +at ch +e ver +Ġst r +Ġpo int +or k +ĠN ew +Ġs ur +o ol +al k +em ent +Ġus ed +ra ct +we en +Ġs ame +ou n +ĠA l +c i +Ġdiff ere +Ġwh ile +---- ---- +Ġg ame +ce pt +Ġs im +.. . +Ġin ter +e k +Ġre port +Ġpro du +Ġst ill +l ed +a h +Ġhe re +Ġwor ld +Ġth ough +Ġn um +ar ch +im es +al e +ĠS e +ĠI f +/ / +ĠL e +Ġre t +Ġre f +Ġtr ans +n er +ut ion +ter s +Ġt ake +ĠC l +Ġcon f +w ay +a ve +Ġgo ing +Ġs l +u g +ĠA meric +Ġspe c +Ġh and +Ġbet ween +ist s +ĠD e +o ot +I t +Ġe ar +Ġagain st +Ġh igh +g an +a z +at her +Ġex p +Ġo p +Ġin s +Ġg r +Ġhel p +Ġre qu +et s +in s +ĠP ro +is m +Ġf ound +l and +at a +us s +am es +Ġp erson +Ġg reat +p r +Ġs ign +ĠA n +' ve +Ġs omet +Ġs er +h ip +Ġr un +Ġ : +Ġt er +ire ct +Ġf ollow +Ġd et +ic es +Ġf ind +1 2 +Ġm em +Ġc r +e red +e x +Ġex t +ut h +en se +c o +Ġte am +v ing +ou se +as h +at t +v ed +Ġsy stem +ĠA s +d er +iv es +m in +Ġle ad +ĠB l +c ent +Ġa round +Ġgo vern +Ġc ur +vel op +an y +Ġc our +al th +ag es +iz e +Ġc ar +od e +Ġl aw +Ġre ad +' m +c on +Ġre al +Ġsupp ort +Ġ1 2 +.. .. +Ġre ally +n ess +Ġf act +Ġd ay +Ġb oth +y ing +Ġs erv +ĠF or +Ġth ree +Ġw om +Ġm ed +od y +ĠThe y +5 0 +Ġex per +t on +Ġe ach +ak es +Ġc he +Ġc re +in es +Ġre p +1 9 +g g +ill ion +Ġg rou +ut e +i k +W e +g et +E R +Ġm et +Ġs ays +o x +Ġd uring +er n +iz ed +a red +Ġf am +ic ally +Ġha pp +ĠI s +Ġch ar +m ed +v ent +Ġg ener +i ent +p le +i et +re nt +1 1 +v es +pt ion +Ġ2 0 +form ation +Ġc or +Ġoff ic +ie ld +Ġto o +is ion +Ġin f +Ġ Z +t he +o ad +Ġp ublic +Ġpro g +r ic +* * +Ġw ar +Ġp ower +v iew +Ġf ew +Ġl oc +Ġdiffere nt +Ġst ate +Ġhe ad +' ll +Ġp oss +Ġst at +re t +ant s +Ġv al +Ġis s +Ġc le +i vers +an c +Ġex pl +Ġan other +Ġ Q +Ġa v +th ing +n ce +W h +Ġch ild +Ġs ince +i red +l ess +Ġl ife +Ġde velop +itt le +Ġde p +Ġp ass +ã ĥ +Ġt urn +or n +Th is +b ers +ro ss +ĠA d +Ġf r +Ġres p +Ġsec ond +o h +Ġ / +Ġdis c +Ġ & +Ġsomet hing +Ġcomp le +Ġ ed +Ġf il +Ġmon th +a j +u c +Ġgovern ment +Ġwith out +Ġle g +Ġd ist +Ġp ut +Ġqu est +an n +Ġpro t +2 0 +Ġne ver +i ence +Ġle vel +Ġar t +Ġth ings +Ġm ight +Ġeff ect +Ġcont ro +Ġc ent +Ġ1 8 +Ġall ow +Ġbel ie +ch ool +ot t +Ġinc re +Ġfe el +Ġres ult +Ġl ot +Ġf un +ot e +Ġt y +ere st +Ġcont in +Ġus ing +Ġb ig +2 01 +Ġas k +Ġb est +Ġ ) +I N +Ġo pp +3 0 +Ġnum ber +in ess +S t +le ase +Ġc a +Ġm ust +Ġd irect +Ġg l +Ġ < +Ġop en +Ġp ost +Ġcom e +Ġse em +ord ing +Ġwe ek +ate ly +it al +Ġe l +ri end +Ġf ar +Ġt ra +in al +Ġp ri +ĠU S +Ġpl ace +Ġfor m +Ġto ld +" : +ain s +at ure +ĠTr ump +Ġst and +Ġ # +id er +ĠF r +Ġne xt +Ġs oc +Ġp ur +Ġle t +Ġl ittle +Ġh um +Ġ i +r on +1 5 +Ġ1 5 +Ġcomm un +Ġm ark +ĠThe re +Ġw r +ĠTh at +Ġin formation +w ays +Ġb us +a pp +Ġinv est +m e +Ġh ard +ain ed +e ad +Ġim port +Ġapp ro +Ġt est +Ġt ri +Ġre st +os ed +Ġf ull +Ġc are +ĠS p +Ġc ase +O N +Ġs k +Ġl ess +Ġ + +Ġpart ic +ĠP l +ab ly +u ck +is hed +ch n +b e +Ġl ist +at or +Ġto p +Ġad v +ĠB e +ru ct +Ġd em +r ation +l ing +g y +re en +g er +Ġh ome +Ġle ft +Ġbet ter +Ġd ata +Ġ1 1 +Ġatt ack +Ġpro ble +l ine +ard s +Ġbe h +r al +ĠH ow +ĠS he +ar ge +Ġ -- +: // +Ġb ro +ĠP h +at s +Ġbu ild +w w +id ed +a im +as es +en cy +Ġm ain +in ed +Ġinclud ing +Ġ { +Ġg ot +Ġint erest +Ġke ep +Ġ X +Ġe as +ain ing +Ġcl ass +âĢ ¦ +ĠN o +Ġv ar +Ġsm all +amp le +A T +Ġ ide +ĠS o +Ġre ce +Ġpol it +Ġm ov +Ġpl an +Ġper cent +iv ing +Ġc amp +Ġp ay +1 4 +s c +is ed +Ġu nt +one y +pl oy +== == +Ġdid n +ĠI nd +el s +ert ain +Ġp os +__ __ +i ver +Ġpro cess +Ġprog ram +if ied +ĠR ep +1 6 +u ro +olog y +at ter +in a +Ġn ame +ĠA ll +Ġf our +Ġret urn +v ious +b s +Ġcall ed +Ġm ove +ĠS c +ir d +Ġgrou p +Ġb re +Ġm en +Ġc ap +t en +e e +Ġd ri +le g +he re +uth or +Ġp at +Ġcur rent +id es +Ġp op +t o +ent ion +Ġal ways +Ġm il +Ġwom en +Ġ1 6 +Ġo ld +iv en +ra ph +ĠO r +r or +ent ly +Ġn ear +ĠE x +re am +s h +Ġ1 4 +Ġf ree +iss ion +st and +ĠC on +al ity +us ed +1 3 +Ġdes ign +Ġch ange +Ġch ang +Ġb o +Ġv is +em ber +Ġb ook +read y +Ġk ill +2 5 +pp ed +Ġa way +Ġab le +Ġcount ry +Ġcon st +ar n +Ġor der +A R +i or +i um +or th +1 8 +ail able +Ġs w +Ġm illion +Ġ1 3 +at ic +t ed +ĠG o +Ġo per +en g +Ġth ing +aj or +con om +ĠCom m +Ġwh y +u red +ur al +Ġs chool +b y +ĠM ar +Ġa ff +Ġd ays +Ġan n +us h +an e +I f +e g +Ġpro f +Ġhe alth +ou th +B ut +ion al +. , +Ġs ol +Ġal ready +Ġ3 0 +Ġchar act +H e +Ġf riend +E S +i ans +ic le +' d +ĠO n +Ġle ast +Ġp rom +Ġd r +Ġh ist +it her +Ġ est +i qu +1 7 +s on +Ġte ll +Ġt alk +oh n +o int +le ction +A N +Ġunt il +au gh +Ġl ater +Ġ ve +Ġv iew +end ing +iv ed +Ġwor d +w are +Ġc ost +Ġen ough +Ġg ive +ĠUn ited +Ġte chn +are nt +O R +Ġp ar +ĠD r +Ġ201 6 +r ist +er ing +Ġ Â +Ġl arge +s ide +ac y +cc ess +Ġw in +Ġimport ant +Ġ19 9 +Ġdoes n +Ġ1 7 +Ġbus iness +Ġcle ar +Ġre se +" , +ur y +Ġe qu +as ter +al f +ĠAmeric an +n ect +Ġex pect +ivers ity +Ġo cc +ĠF l +Ġk ind +Ġme an +Ġp ast +Ġde v +Ġb as +le t +ra ft +Ġor gan +Ġde l +Ġper form +Ġst ory +Ġse ason +ĠC ol +Ġcl aim +Ġc ame +Ġwith in +Ġl ine +Ġpro ject +ĠA t +Ġcontro l +end ed +ĠS y +Ġa ir +iz ation +Ġ * +le y +Ġm oney +id d +Y ou +f or +Ġfam ily +Ġm aking +Ġb it +Ġpol ice +Ġhapp en +Ġ vers +on y +u ff +ĠW hen +Ġs it +ide o +l f +is on +Ġsu re +g in +Ġapp ear +Ġl ight +Ġ es +o f +Ġw ater +Ġt imes +n ot +Ġg row +Ġcomp any +ĠT e +ow s +Ġm ar +our ce +i ol +ar m +b r +Ġex ample +Ġcon c +Ġf ore +ĠT o +p ro +E N +ri es +Ġ2 5 +ĠC an +ne y +Ġact ually +Ġe ver +ur ity +ak en +ap s +Ġt ax +Ġm ajor +am a +Ġof ten +er al +Ġhum an +Ġj ob +is ter +Ġav ailable +oc r +en n +a id +iv id +Ġrec ord +? " +Ġs ing +ĠA m +id ence +Ġnew s +st er +Ġe conom +Ġfollow ing +ĠB r +is ing +Ġh our +m ost +um ent +Ġse x +Ġdes c +Ġbec ome +ĠE d +Ġto ok +Ġha ving +Ġprodu ct +a ult +A s +ar ing +Ġme ans +Ġh op +un e +Ġch o +Ġc ertain +Ġn on +Ġde al +2 4 +le ment +oc i +en e +Ġs ide +ĠP r +ĠM ay +Ġre ason +u ed +c hed +ul ation +Ġe lect +Ġoffic ial +Ġposs ible +Ġh old +and s +ot s +Ġc ity +or ies +Ġse ver +Ġchild ren +Ġon ce +Ġact iv +l er +Ġn ight +it ions +ĠJ ohn +a pe +pl ay +Ġd one +Ġl im +Ġwork ing +ĠP res +or ld +e b +ĠC o +Ġb ody +ail s +ut es +ĠM r +Ġwhe ther +Ġa uthor +ro p +Ġpro per +Ġse en +) ; +Ġf ac +ĠS u +Ġcon d +it ing +Ġcour se +Ġ } +-------- -------- +a ign +Ġev ent +Ġen g +Ġp ot +Ġin tern +i am +Ġsh ort +em pt +ã Ĥ +ĠG od +il ar +8 0 +Ġor ig +I S +our n +ab ility +it ive +Ġd am +Ġ1 00 +Ġp ress +Ġdo ing +Ġprot ect +r ing +Ġthough t +Ġquest ion +re w +ĠW ar +Ġsever al +ĠSt ate +Ġg iven +Ġf und +ĠT w +Ġw ent +an ces +w ork +p or +m y +4 0 +Ġar g +art ment +ust om +Ġpol ic +Ġme et +Ġc reat +2 2 +ĠSt ates +Ġg ames +ra w +ut ure +Ġunder stand +ur s +ĠO b +l ish +s y +Ġm akes +Ġw on +ag on +Ġh tt +Ġl ove +ent ial +Ġcomple te +p ar +ĠI m +A L +Ġacc ount + ł +ore d +ver t +Ġ ident +Ġ201 5 +Ġother s +ĠM in +i ber +ver age +The re +ition al +d d +Ġpro b +Ġyou ng +Ġal ong +Ġacc ording +Ġy et +Ġmem bers +ĠWh at +o id +ĠM an +A nd +Ġam ong +a i +Ġem ploy +ĠR es +Ġ > +Ġinv ol +Ġl ow +a f +ĠC ar +Ġh ig +ĠO ne +ĠS ec +in ation +Ġlike ly +Ġan t +ag ed +ĠR uss +Ġb en +Ġre le +F or +b ack +ĠN ot +Ġpres ident +b all +Ġacc ess +ivid ual +ĠD em +ĠE uro +6 0 +Ġkn own +ir l +ĠG r +Ġear ly +u se +iet y +âĢ ĵ +Ġf ight +Ġs ent +Ġto day +Ġmark et +" . +Ġb ased +Ġstr ong +ur ther +Ġde b +m ber +Ġproble m +Ġde ath +Ġsoc ial +im ate +A S +ort un +Ġcamp aign +er y +C h +Ġe y +i ally +Ġm us +w h +p os +Ġ er +Ġsa f +Ġmonth s +ir on +Ġv iol +Ġf ive +Ġst re +Ġplay ers +in c +al d +y ear +a un +Ġsu ccess +Ġpres ent +ere nce +Ġ201 4 +Ġsu gg +Ġpartic ular +Ġtr y +Ġsugg est +ĠCh rist +on es +Ġpri v +2 3 +Ġc rit +Ġl and +Ġloc al +if y +2 9 +Ġa ut +E D +ĠG u +Ġm ult +Ġpolit ical +Ġask ed +Ġfor mer +it ter +ri pt +Ġcl ose +Ġp ract +ĠY ork +Ġget ting +Ġac ross +Ġcom b +Ġbelie ve +Ġ z +Ġto get +Ġtoget her +ĠC ent +ir c +Ġind ividual +ĠM c +2 7 +is k +ĠE ng +Ġf ace +Ġ2 4 +Ġval ue +Ġare a +e v +Ġw rit +ĠPres ident +Ġv ot +Ġke y +Ġm om +p ut +Ġany thing +Ġexper ience +att le +Ġm ind +a ff +om m +Ġf uture +g ed +Ġc ut +Ġto t +it ch +Ġv ideo +Ġinvest ig +Ġn et +ĠM y +r ict +i en +. ) +Ġimp ro +th ough +ward s +Ġcon nect +ĠM ed +sel ves +ens ive +m b +o ber +at ors +A n +Ġ5 0 +Ġre du +res ent +Ġab ove +Ġf re +ĠEuro pe +s w +Ġam ount +ĠA pp +Ġe ither +Ġmil it +Ġan al +Ġf ail +ĠE n +al es +Ġspec ial +Ġbl ack +I T +c her +Ġlook ing +Ġf ire +y n +Ġal most +o on +Ġstud y +Ġm iss +c hes +ro wn +Ġt re +Ġcommun ity +Ġmed ia +Ġf ood +Ġcom es +ĠUn iversity +Ġsing le +Wh at +u ly +Ġh alf +ag ue +h od +ĠRep ublic +Ġstart ed +Ġqu ick +ot o +b ook +Ġiss ue +it or +Ġel se +Ġcons ider +2 6 +ro du +Ġt aken +2 8 +9 9 +ĠW ith +Ġtr ue +Ġw a +Ġtr ad +Ġag o +Ġm ess +ie f +Ġadd ed +o ke +Ġb ad +Ġf av +3 3 +Ġsim ilar +as k +ĠD on +Ġcharact er +ort s +ĠH ouse +Ġreport ed +Ġty pe +v al +i od +ĠHow ever +Ġt arg +Ġent ire +pp ing +Ġhist ory +Ġl ive +ff ic +.... .... +ed eral +Ġtr ying +Ġdisc uss +ĠH ar +ac es +l ished +Ġse lf +os p +re st +Ġro om +el t +Ġf all +ol ution +Ġe t +Ġ x +Ġis n +Ġide a +b o +Ġs ound +ĠD ep +Ġsome one +ci ally +ull y +Ġf oc +Ġob ject +if t +ap er +Ġplay er +Ġr ather +Ġserv ice +as hing +ĠD o +ĠP art +ru g +m on +p ly +Ġm or +Ġnot hing +Ġprov ide +I C +un g +Ġpart y +Ġex ist +Ġm ag +7 0 +Ġr ul +Ġh ouse +Ġbeh ind +Ġhow ever +ĠW orld +Ġs um +Ġapp lic +Ġ ; +Ġfun ction +g r +ĠP ol +Ġfr ont +2 00 +Ġser ies +Ġt em +Ġty p +ill s +Ġo pt +Ġpoint s +Ġbel ow +itt ed +Ġspec ific +Ġ201 7 +um b +Ġr a +Ġpre vious +Ġpre t +re me +Ġc ustom +Ġcour t +ĠM e +Ġre pl +Ġwho le +g o +c er +Ġt reat +ĠA ct +Ġprob ably +Ġle arn +end er +ĠA ss +Ġvers ion +n ow +Ġche ck +ĠC al +R E +min ist +O n +our ces +Ġben ef +Ġd oc +Ġdet er +Ġen c +Ġsu per +Ġadd ress +Ġv ict +Ġ201 3 +Ġme as +t r +Ġf ield +W hen +Ġsign ific +u ge +Ġfe at +Ġcomm on +l oad +Ġbe gin +Ġbr ing +Ġa ction +er man +Ġdesc rib +Ġind ust +Ġwant ed +ri ed +m ing +Ġatt empt +4 5 +f er +Ġd ue +ress ion +# # +Ġsh all +Ġs ix +o o +Ġst ep +Ġp ub +Ġhim self +Ġ2 3 +Ġc op +Ġd est +Ġst op +A C +ib ility +Ġl ab +ic ult +Ġhour s +Ġcre ate +Ġf urther +ĠAmeric a +ĠC ity +Ġd ou +he ad +S T +ĠN orth +c ing +Ġn ational +u le +ĠIn st +Ġt aking +ĠQ u +ir t +Ġre d +Ġrese arch +v iron +ĠG e +Ġbre ak +an a +Ġsp ace +ater ial +Ġrec ent +ĠA b +Ġgener al +Ġh it +Ġper iod +Ġevery thing +ive ly +Ġph ys +Ġsay ing +an ks +Ġc ou +Ġc ult +ac ed +e al +u ation +Ġc oun +l u +Ġinclud e +Ġpos ition +ĠA fter +ĠCan ad +ĠE m +Ġim m +ĠR ed +Ġp ick +Ġcom pl +Ġm atter +re g +e xt +ang u +is c +o le +a ut +Ġcomp et +e ed +f ect +Ġ2 1 +ĠS en +ĠThe se +as ing +Ġcan not +Ġin it +Ġrel ations +ac hed +Ġb ar +Ġ4 0 +ĠT H +Ġ201 2 +Ġv ol +Ġg round +Ġsec urity +Ġup d +il t +3 5 +Ġconc ern +ĠJ ust +Ġwh ite +Ġseem s +ĠH er +pe cially +i ents +Ġann oun +Ġf ig +ight s +Ġst ri +l ike +id s +Ġs us +Ġw atch +Ġ â +Ġw ind +ĠC ont +Ġit self +Ġm ass +A l +y le +iqu e +ĠN ational +Ġab s +Ġp ack +Ġout side +Ġan im +Ġp ain +et er +Ġman ag +du ct +og n +Ġ ] +ĠSe pt +se c +o ff +ĠJ an +Ġf oot +ad es +Ġth ird +Ġm ot +Ġev idence +int on +Ġth reat +a pt +pl es +c le +Ġl o +Ġde cl +Ġit em +med i +Ġrep resent +om b +am er +Ġsignific ant +og raph +s u +Ġc al +i res +00 00 +I D +A M +Ġsim ply +Ġlong er +Ġf ile +O T +c he +S o +ate g +or g +ĠH is +Ġen er +Ġd om +Ġup on +il i +": " +Ġthem selves +Ġcom ing +Ġqu ite +Ġdiff icult +ĠB ar +il ities +re l +end s +c ial +6 4 +Ġwom an +ra p +y r +Ġne cess +ip s +Ġte xt +Ġrequ ire +Ġmilit ary +Ġre view +Ġresp ons +7 5 +Ġsub ject +Ġinst ead +Ġiss ues +Ġg en +" ," +Ġmin utes +Ġwe ap +r ay +am ed +t ime +b l +H ow +Ġc ode +ĠS m +Ġhig her +ĠSt e +r is +Ġp age +Ġstud ents +ĠIn tern +Ġmet hod +ĠA ug +ĠP er +ĠA g +Ġpolic y +ĠS w +Ġex ec +Ġac cept +um e +rib ut +Ġword s +Ġfin al +Ġchang es +ĠDem ocr +Ġfriend s +Ġres pect +Ġe p +Ġcomp an +iv il +Ġdam age +** ** +og le +viron ment +Ġne g +ent al +Ġa p +Ġtot al +iv al +! " +l im +Ġneed s +Ġag re +Ġdevelop ment +Ġa ge +ip le +2 1 +Ġresult s +ĠA f +S h +Ġg un +ĠOb ama +ro ll +Ġ @ +Ġright s +ĠB rit +Ġrun ning +Ġwas n +Ġp ort +Ġr ate +Ġpret ty +Ġtarg et +Ġsa w +Ġc irc +Ġwor ks +ic ro +al t +o ver +ww w +Th at +l ier +Ġevery one +ud e +Ġp ie +idd le +ra el +Ġr ad +Ġbl ock +Ġw alk +T o +ã ģ +n es +ĠA ust +a ul +ro te +ĠS outh +ess ion +op h +Ġshow s +Ġs ite +Ġj o +Ġr isk +cl us +l t +Ġin j +id ing +ĠS pe +Ġch all +ir m +Ġ2 2 +itt ing +st r +Ġh y +L E +ke y +Ġbe gan +at ur +ashing ton +l am +ĠD av +b it +Ġs ize +ĠP ar +3 8 +ourn al +f ace +Ġdec ision +Ġl arg +Ġj ud +re ct +Ġcontin ue +ĠO ct +ove red +ĠI nt +==== ==== +Ġp arent +ĠW ill +Ġeas y +Ġd rug +ang er +Ġs ense +Ġd i +id ay +Ġener gy +ist ic +Ġass oci +ar ter +ob al +e ks +ĠE l +ur ch +Ġg irl +o e +it le +Ġ2 8 +ĠC he +Ġrequ est +Ġso on +Ġh ost +k y +Ġst ates +om es +Ġm aterial +le x +Ġmom ent +Ġan sw +on se +Ġes pecially +Ġn orm +Ġserv ices +p ite +r an +Ġro le +4 4 +) : +Ġc red +C l +____ ____ +Ġm at +Ġl og +ĠCl inton +O U +Ġoff ice +Ġ2 6 +Ġch arg +Ġtr ack +m a +Ġhe art +Ġb all +Ġperson al +Ġbuild ing +n a +s et +b ody +ĠBl ack +Ġincre ase +itt en +Ġneed ed +3 6 +3 2 += " +Ġl ost +Ġbec ame +Ġgrou ps +ĠM us +Ġw rote +ĠP e +Ġpro p +j oy +à © +ĠWh ite +Ġde ad +. ' +Ġhtt p +Ġwe bs +O S +Ġins ide +Ġwr ong +Ġstat ement +Ġ ... +y l +Ġfil m +Ġmus ic +Ġsh are +ific ation +Ġre lease +Ġfor ward +Ġst ay +Ġcomp ut +it te +s er +Ġorig inal +Ġc ard +Ġc and +Ġd iv +at ural +Ġfav or +O M +Ġc ases +us es +Ġse ction +Ġle ave +g ing +ov ed +ĠW ashington +3 9 +ĠG l +Ġrequ ired +act ion +ap an +o or +it er +ĠK ing +Ġcount ries +ĠG erman +ll ing +Ġ2 7 +3 4 +Ġquest ions +Ġpr im +Ġc ell +Ġsh oot +Ġany one +ĠW est +Ġaff ect +ep end +Ġon line +ĠIs rael +ĠSept ember +Ġab ility +Ġcont ent +is es +Ġre ve +Ġl aun +Ġind ic +Ġfor ce +c ast +Ġso ld +av ing +f l +Ġso ft +Ġcompan ies +ce ed +Ġart icle +Ġa ud +Ġre v +Ġed uc +Ġplay ing +0 5 +Ġhe ld +ct or +Ġrele ased +Ġf ederal +3 7 +Ġad minist +Ġinter view +Ġinst all +Ġrece ived +Ġs ource +u k +P h +Ġser ious +Ġcre ated +Ġc ause +Ġim medi +Ġdef in +u el +ĠDep artment +ct ions +ĠC our +ĠN ow +z e +it es +it ution +Ġl ate +Ġspe ak +n ers +Ġleg al +ar i +ĠC or +Ġwe eks +Ġmod el +Ġp red +Ġex act +B C +ĠB y +IN G +os ing +Ġt akes +Ġreg ard +Ġopp ortun +Ġpr ice +Ġ19 8 +ĠA pr +f ully +Ġor d +Ġproble ms +ru ction +h am +ĠC ount +le ge +Ġlead ers +E T +le v +Ġde ep +olog ical +es e +h aps +ĠS ome +Ġp ers +Ġcont ract +Ġrelations hip +s p +ou d +Ġb ase +4 8 +m it +A d +anc ial +Ġcons um +Ġpot ential +Ġl angu +re m +et h +Ġrel ig +ress ed +6 6 +Ġl ink +Ġl ower +ay er +ĠJ une +Ġf em +un t +er c +ur d +Ġcont act +Ġ ill +Ġm other +Ġest ab +h tt +ĠM arch +ĠB ro +ĠCh ina +Ġ2 9 +Ġs qu +Ġprov ided +Ġa verage +as ons +Ġ201 1 +Ġex am +l in +5 5 +n ed +Ġper fect +Ġt ou +al se +u x +Ġbu y +Ġsh ot +Ġcol lect +Ġph ot +Ġplay ed +Ġsur pr +Ġofficial s +Ġsim ple +av y +Ġindust ry +Ġhand s +g round +Ġp ull +Ġr ound +Ġus er +Ġr ange +u ary +Ġpriv ate +op s +e es +Ġw ays +ĠM ich +Ġve h +Ġex cept +Ġter ms +im um +pp er +I ON +ore s +ĠDr agon +ou l +Ġd en +Ġperform ance +Ġb ill +c il +4 7 +Ġen vironment +Ġex c +ad d +Ġwor th +Ġp ict +Ġch ance +Ġ201 8 +b or +Ġspe ed +ict ion +Ġal leg +ĠJ apan +at ory +re et +Ġm atch +ĠI I +Ġst ru +ord er +Ġst e +Ġl iving +Ġst ruct +in o +Ġse par +her n +Ġresp onse +Ġen joy +Ġv ia +A D +um ents +ace book +Ġmem ber +ib r +iz ing +Ġto ol +ĠM on +ĠWh ile +h ood +ĠA ng +ĠD ef +Ġoff er +T r +a ur +Ġturn ed +ĠJ uly +d own +an ced +Ġrec ently +ĠE ar +Ġc e +ĠSt ar +ĠC ong +rough t +Ġbl ood +Ġhop e +Ġcom ment +ain t +Ġar ri +il es +Ġpartic ip +ough t +ri ption +0 8 +4 9 +Ġg ave +Ġse lect +Ġkill ed +sy ch +Ġgo es +i j +Ġc oll +Ġimp act +at ives +ĠS er +0 9 +ĠAug ust +Ġb oy +d e +ĠD es +Ġf elt +U S +Ġexpect ed +Ġim age +ĠM ark +cc ording +o ice +E C +ĠM ag +en ed +h old +ĠP ost +Ġpre vent +N o +Ġinvol ved +Ġey es +Ġquick ly +A t +un k +Ġbeh av +Ġ ur +Ġl ed +c ome +e y +Ġcand id +Ġear lier +Ġfoc us +et y +P ro +led ge +ix ed +ill ed +Ġpop ular +A P +Ġset t +l ight +Ġvar ious +in ks +Ġlevel s +Ġro ad +ell ig +ab les +he l +itte e +ĠG ener +y pe +Ġhe ard +ic les +Ġm is +Ġus ers +ĠS an +Ġimpro ve +Ġf ather +Ġse arch +The y +v il +Ġprof ess +Ġkn ew +Ġl oss +Ġev ents +6 5 +Ġb illion +0 7 +0 2 +ĠNew s +ĠA M +Ġco ver +w here +ens ion +Ġb ott +Ġare as +en ces +op e +ĠTw itter +a el +Ġget s +ĠGo ogle +Ġs n +i ant +Ġv ote +Ġnear ly +Ġinclud ed +Ġrec ogn +z z +m m +al ed +Ġhappen ed +0 4 +Ġh ot +Ġwho se +Ġc ivil +Ġsu ff +o es +it iz +ĠSy ri +Ġresp ond +Ġh on +Ġfeat ures +Ġeconom ic +ĠApr il +r im +Ġtechn ology +Ġo ption +ag ing +Ġpur ch +R e +Ġl at +ch ie +is l +Ġrec omm +u f +Ġtr aining +Ġeffect s +Ġf ast +Ġ201 0 +Ġocc ur +Ġwebs ite +Ġem ail +Ġs ens +e ch +Ġo il +Ġinf lu +Ġcurrent ly +ĠS ch +ĠAd d +Ġgo al +Ġsc ient +Ġcon v +1 00 +em y +Ġdec ided +Ġtra vel +Ġm ention +L L +0 3 +Ġe lection +Ġph one +Ġlook s +Ġsit uation +Ġc y +Ġh or +b ed +ĠCour t +a ily +av es +Ġqu ality +ĠCom p +w ise +Ġt able +Ġst aff +ĠW ind +et t +Ġtri ed +ide red +Ġadd ition +Ġb ox +Ġl ack +ar ily +Ġw ide +Ġm id +Ġbo ard +ys is +Ġant i +h a +Ġd ig +en ing +Ġd ro +C on +6 8 +Ġsl ow +b ased +se qu +Ġp ath +E x +ak er +Ġwork ed +Ġp en +Ġeng ine +Ġlook ed +ĠSu per +ĠS erv +Ġvict im +U n +Ġproper ty +Ġint rodu +Ġexec ut +ĠP M +L e +Ġcol or +ĠM ore +Ġ6 0 +Ġnet work +Ġd ate +c ul +id ge +Ġext ra +3 1 +Ġs le +6 7 +Ġw ond +Ġreport s +j ust +ĠAust ral +Ġcap ital +Ġen s +Ġcomm and +Ġallow ed +Ġpre p +Ġca pt +h ib +Ġnum bers +ch an +Ġf air +m p +om s +Ġre ach +W ith +t ain +Ġbro ad +Ġcou ple +ec ause +ly ing +ĠF eb +Ġsc reen +Ġl ives +Ġpri or +ĠCong ress +A r +Ġappro ach +Ġe mer +ar ies +ĠD is +s erv +ĠN e +Ġbu ilt +c ies +Ġre pe +Ġrul es +for ce +ĠP al +Ġfin ancial +Ġcons idered +ĠCh ar +n ces +ĠI S +Ġb rought +Ġb i +i ers +ĠS im +O P +Ġproduct s +Ġvis it +Ġdoc ument +Ġcon duct +Ġcomplete ly +in ing +ĠCal if +ib ly +Ġwr itten +ĠT V +em ents +Ġd raw +O ne +Ġpub lished +Ġsec ret +r ain +he t +ĠF acebook +ond ay +ĠU p +Ġsex ual +Ġth ous +ĠP at +Ġ ess +Ġstand ard +Ġar m +g es +ect ion +Ġf ell +Ġfore ign +an i +ĠFr iday +Ġreg ular +in ary +Ġincre ased +Ġus ually +Ġdem on +Ġd ark +Ġadd itional +ro l +ĠO f +Ġprodu ction +! ! +und red +Ġintern ational +id ents +ĠF ree +rou p +Ġr ace +Ġm ach +Ġh uge +A ll +le ar +ove mber +Ġto wn +Ġatt ention +ĠO ff +y ond +ĠThe n +f ield +Ġter ror +ra z +ĠB o +Ġmeet ing +ĠP ark +Ġar rest +Ġf ear +Ġa w +ĠV al +or ing +' , +Ġext reme +ar r +Ġwork ers +A fter +Ġ3 1 +n et +am ent +Ġdirect ly +Ġpop ulation +ub e +ĠOct ober +ĠI N +ĠJan uary +5 9 +ĠDav id +Ġc ross +ce mber +ĠF irst +Ġmess age +ir it +Ġn ation +Ġp oll +is ions +Ġansw er +n y +is ode +Ġcar ry +ĠRuss ia +Ġhe ar +eng th +ro y +Ġn atural +in ally +Ġdo g +m itted +Ġtr ade +Ġsub st +Ġmult iple +ĠAf ric +Ġf ans +Ġs ort +Ġgl obal +ic ation +ĠW ed +ar a +Ġa chie +Ġlangu age +ve y +Ġt al +Ġnecess ary +Ġdet ails +Ġs en +ĠS und +ĠRe g +ĠR ec +0 6 +Ġs il +ress ive +Ġmed ical +un ch +orn ia +Ġu nd +f ort +oc ks +ĠM onday +ues day +c raft +7 7 +ur t +Ġ ver +ĠH ill +Ġrece ive +Ġmor ning +es tern +Ġb ank +Ġs at +ir th +ĠH igh +Ġdev ice +ĠTH E +ĠCent er +Ġsaf e +Ġp le +ĠCanad a +Ġsystem s +Ġass ist +Ġsur v +Ġb attle +ĠS oc +vert is +S he +Ġp aper +Ġgrow th +Ġc ast +S c +Ġpl ans +ll ed +Ġpart s +Ġw all +Ġmove ment +Ġpract ice +im ately +Ġdis play +Ġsomet imes +om p +ĠP aul +ĠY es +k ing +5 8 +o ly +Ġs on +Ġav oid +ok es +ĠJ ew +Ġto wards +as c +Ġ // +ĠK ore +Ġtalk ing +Ġcor rect +Ġsp ent +ic ks +i able +e ared +Ġter m +Ġwant s +om ing +Ġ ut +Ġdou b +Ġfor ces +Ġp lease +6 9 +ĠN ovember +at form +ond on +Ġon es +Ġimmedi ately +ĠRuss ian +ĠM et +Ġde g +Ġparent s +C H +ĠAmeric ans +al y +ĠM od +Ġsh own +Ġcond itions +Ġst uff +Ġre b +ĠY our +Ġinclud es +n own +ĠS am +Ġexper ien +m ission +ĠE ven +augh t +Ġannoun ced +ĠRepublic an +Ġdeter min +Ġdescrib ed +ĠCount y +( ) +Ġdo or +Ġchang ed +Ġne igh +ĠH ere +Ġcle an +Ġp an +ĠDe cember +ĠEurope an +ir ing +ap ter +Ġcl ub +ĠT uesday +Ġp aid +ĠN et +Ġattack s +Ġcharact ers +Ġal one +Ġdirect or +d om +Ġ3 5 +Ġl oad +Ġr out +ĠCalif ornia +Ġfin ally +Ġr ac +Ġcont r +Ġexact ly +res h +p ri +ĠIs lam +Ġn ature +Ġcare er +Ġlat est +Ġcon vers +ĠS l +p ose +ci ent +ĠIn c +iv ity +8 8 +ĠA tt +ĠM or +nes day +Ġwe ight +k en +Ġnot e +Ġteam s +Ġ \ +air s +ĠG reen +Ġh undred +on ent +Ġstre ng +Ġcons ist +ic ated +Ġreg ul +Ġl ic +ast ic +Ġt en +urs day +ellig ence +ous ly +ĠU K +B I +Ġcost s +Ġind epend +ĠA P +Ġnorm al +Ġh om +Ġob vious +Ġs we +Ġst ar +Ġread y +ac her +Ġimp lement +g est +Ġs ong +ĠG et +ĠL ab +Ġinterest ing +us ing +Ġg iving +ĠSund ay +Ġet c +Ġm iddle +Ġrem ember +r ight +os ition +ut ions +Ġm ax +4 6 +Ġyour self +Ġdem and +Ġtreat ment +Ġd anger +ĠC ons +Ġgu y +ĠBrit ish +Ġphys ical +Ġrel ated +Ġrem ain +Ġcould n +Ġref er +Ġc itiz +b ox +EN T +bo ard +Ġin n +I G +er o +ĠSt reet +osp ital +ren ch +cher s +Ġst ra +O L +ag er +ĠA N +Ġeas ily +I A +en ge +in y +Ġcl os +ock ed +Ġus es +ĠC oun +I m +u ild +? ? +m ore +Ġan g +Ġwr ite +ol ute +5 7 +Ġlead er +Ġread ing +< / +Ġaut om +est s +4 3 +Ġleg isl +ĠG old +Ġdesign ed +ĠS T +ĠLe g +a res +Ġbe aut +ĠT ex +Ġappear s +Ġstru gg +ĠR om +Ġ 00 +Ġcho ice +Ġparticular ly +ĠF rom +op er +ĠL ondon +ann ed +Ġallow s +ob ile +Ġdiffere nce +âĢ ¢ +ĠV iew +ĠWed nesday +Ġal though +Ġrel ative +Ġapplic ation +ate ver +Ġare n +Ġmy self +Ġim ag +Ġdis e +Ġsoc iety +Ġfre qu +ĠEng lish +Ġpo or +ĠD ay +Ġwrit ing +Ġse ven +Ġstart ing +Ġb ud +Ġpr int +ĠTr ans +uf act +ĠSt ud +n ew +Ġcr im +Ġg ives +Ġco ol +a e +i ance +ĠGener al +Ġthink ing +Ġsa ve +Ġlim ited +ĠPart y +Ġmean ing +p en +ow ers +ĠJ ack +E M +Ġn ice +ru pt +Ġg as +Ġe ight +Ġfe et +Ġeff ort +Ġ ign +ic it +B l +co in +Ġop in +Ġbr ain +Wh ile +he st +ĠTh ursday +Ġwould n +augh ter +Ġtou ch +le ments +Ġstud ies +Ġcent er +c ont +or ge +Ġcomput er +Ġinvestig ation +P l +or ks +Ġ200 8 +Ġincre asing +Ġst ore +Ġcom ments +Ġb al +m en +Ġdo ll +Ġl iber +Ġw ife +Ġlaw s +atur day +it ness +Ġmod ern +ĠS k +Ġadminist ration +Ġopportun ity +Ġs al +Ġpower ful +M y +Ġclaim s +ĠEar th +ord s +Ġt itle +Ġes c +n ame +N ot +om en +Ġbe yond +Ġc amer +Ġse ll +it ute +ear ch +Ġapp l +im ent +4 2 +ĠAr t +Ġun f +Ġviol ence +ur g +ĠE ast +Ġcomp ared +Ġopt ions +Ġthrough out +Ġv s +ig r +. [ +ac hes +7 8 +Ġfil es +F L +E L +ar ian +ĠJ ames +ĠA ir +an ch +Ġdet ail +Ġpie ce +P S +Ġn amed +Ġeduc ation +Ġdri ve +Ġitem s +Ġstud ent +ic ed +: : +ic o +Ġth row +Ġsc ene +Ġcomple x +Ġ200 9 +Ġpre c +ĠB re +7 9 +Ġcon cept +Ġstat us +am ing +Ġd ied +Ġknow ledge +Ġbegin ning +O D +ru ary +Ġcertain ly +Ġgu ys +Ġsl ight +in n +ound s +Ġf ine +Ġf at +ic ations +Ġper haps +ĠA nt +Ġinc ome +Ġhtt ps +Ġmajor ity +port s +st on +Ġgreat er +Ġfe ed +ent ially +Ġsaf ety +Ġun ique +and om +Ġg one +Ġshow ed +Ġhist or +Ġcoun ter +i us +id a +Ġlead ing +i pe +Ġs end +ĠDon ald +er ve +Ġdef ense +ines e +Ġy es +ĠF ire +ĠMus lim +ra q +Ġcontin ued +os h +Ġprov ides +Ġpr ison +ĠP re +Ġhapp y +Ġeconom y +Ġtr ust +ag s +ĠG ame +Ġweap ons +um an +ĠC le +it ation +Ġanal ysis +ĠT imes +Ġsc ience +- > +Ġfig ure +Ġdis app +ent y +Ġsoft ware +Ġu lt +Ġoffic ers +N ew +I s +Ġrem ains +ĠInd ia +Ġp sych +ri ef +Ġc at +es c +Ġob serv +Ġst age +ĠD ark +Ġent er +ch ange +Ġpass ed +Ġdes pite +ĠO ut +Ġmov ie +r s +Ġv oice +m ine +ĠPl ay +Ġto ward +ĠT er +Ġreg ion +Ġval ues +or ters +Ġm ount +Ġoffic er +ĠO ther +b an +Ġh ous +w ood +ro om +I V +ĠS un +se e +ĠO ver +ro g +9 0 +Ġl ay +ĠT ur +a wn +Ġpress ure +ĠS ub +Ġbook s +ed om +ĠS and +A A +ag o +Ġre asons +f ord +Ġactiv ity +U T +N ow +ĠSen ate +ce ll +n ight +Ġcall s +in ter +Ġlet ter +ĠR ob +ĠJ e +Ġcho ose +ĠL aw +G et +B e +Ġro b +Ġtyp es +Ġpl atform +Ġqu arter +R A +ĠT ime +Ġmay be +ĠC r +9 5 +p re +Ġmov ing +Ġl if +Ġgo ld +Ġs om +Ġpat ients +Ġtr uth +ĠK e +ur ance +ant ly +m ar +Ġchar ge +ĠG reat +Ġce le +---------------- ---------------- +Ġro ck +ro id +an cy +Ġcred it +a ud +B y +ĠE very +Ġmov ed +ing er +rib ution +Ġn ames +Ġstra ight +ĠHe alth +ĠW ell +Ġfe ature +Ġr ule +Ġsc he +in ated +ĠMich ael +ber g +4 1 +il ed +b and +Ġcl ick +ĠAng el +on ents +Â Ń +ĠI raq +ĠS aturday +Ġa ware +p art +Ġpat tern +O W +ĠL et +Ġgr ad +ign ed +Ġassoci ated +Ġst yle +n o +i ation +a ith +il ies +Ġst ories +ur ation +Ġindividual s +ĠâĢ ¦ +m iss +ĠAss oci +ish ing +ab y +Ġsum mer +ĠB en +Ġ3 2 +Ġar ch +ut y +ĠTex as +h ol +Ġfull y +Ġm ill +Ġfollow ed +ĠB ill +ĠInd ian +ĠSec ret +ĠB el +ĠFeb ruary +Ġjob s +Ġseem ed +ĠGo vern +i pped +Ġreal ity +Ġl ines +Ġp ark +Ġmeas ure +ĠO ur +I M +Ġbro ther +Ġgrow ing +Ġb an +Ġest im +Ġc ry +ĠS chool +Ġme chan +ĠO F +ĠWind ows +Ġr ates +ĠO h +Ġpos itive +Ġcult ure +ist ics +ic a +Ġh ar +y a +ite ly +i pp +Ġm ap +en cies +ĠWill iam +I I +ak ers +5 6 +ĠM art +ĠR em +Ġal tern +it ude +Ġco ach +row d +D on +Ġk ids +Ġj ournal +Ġcor por +Ġf alse +Ġwe b +Ġsle ep +Ġcont ain +Ġst o +Ġb ed +iver se +ĠR ich +ĠCh inese +Ġp un +Ġme ant +k nown +Ġnot ice +Ġfavor ite +a ven +Ġcond ition +Ġpur pose +) ) +Ġorgan ization +Ġchall eng +Ġman ufact +Ġsus p +ĠA c +Ġcrit ic +un es +uc lear +Ġm er +vent ion +Ġ8 0 +Ġm ist +ĠU s +ĠT or +htt p +ol f +Ġlarg er +Ġadv ant +Ġrese ar +Ġact ions +m l +Ġke pt +Ġa im +, ' +c ol +Ġbenef its +if ying +Ġact ual +ĠIntern ational +Ġveh icle +Ġch ief +Ġeff orts +ĠLe ague +ĠM ost +Ġwa it +Ġad ult +Ġover all +Ġspe ech +Ġhigh ly +Ġfem ale +Ġer ror +Ġeffect ive +5 4 +Ġenc our +w ell +Ġfail ed +Ġcons erv +Ġprogram s +Ġt rou +Ġa head +5 00 +vertis ement +I P +ĠF ound +p ir +Ġ % +Ġcr ime +and er +Ġloc ation +ĠI ran +Ġbehav ior +az ing +Ġr are +Ġem b +Ġca used +Ġsh ip +Ġact ive +Ġcont ribut +Ġg reen +Ġac qu +Ġref lect +ven ue +Ġf irm +Ġb irth +] . +Ġclear ly +Ġem ot +Ġag ency +ri age +Ġmem ory +9 8 +S A +ĠSe e +ac ing +C C +Ġbig gest +Ġr ap +Ġbas ic +Ġb and +e at +Ġsus pect +ĠM ac +Ġ9 0 +m ark +ist an +Ġsp read +am s +k i +as y +ra v +ĠR ober +Ġdemon str +r ated +Ġabs olute +Ġpl aces +Ġim pl +ibr ary +Ġc ards +Ġdest roy +Ġv irt +ve re +Ġapp eared +y an +p oint +Ġbe g +Ġtem per +s pe +ant ed +ear s +ĠD irect +Ġl ength +Ġbl og +am b +Ġint eg +Ġres ources +ac c +if ul +Ġsp ot +Ġfor ced +Ġthous ands +ĠMin ister +Ġqu al +ĠF rench +at ically +Ġgener ally +Ġdr ink +Ġth us +I L +od es +Ġappro pri +ĠRe ad +Ġwh om +Ġey e +Ġcol lege +Ġ4 5 +ire ction +Ġens ure +Ġapp arent +id ers +Ġrelig ious +Ġmin or +ol ic +Ġt ro +ĠWh y +rib ute +m et +Ġprim ary +Ġdevelop ed +Ġpe ace +Ġsk in +st e +av a +Ġbl ue +Ġfam ilies +Ġ ir +Ġapp ly +Ġin form +ĠSm ith +C T +i i +Ġlim it +Ġres ist +........ ........ +um n +Ġconf lic +Ġtw e +ud d +ĠT om +Ġl iter +qu e +b on +Ġha ir +Ġevent ually +Ġp us +Ġhelp ed +Ġag g +or ney +ĠApp le +Ġf it +ĠS ur +Ġpre m +Ġs ales +Ġsecond s +Ġstreng th +Ġfeel ing +¿ ½ +Ġt our +Ġknow s +o om +Ġex erc +Ġsom ew +ï ¿½ +> > +Ġsp okes +Ġide as +Ġreg ist +so ft +ĠD el +ĠP C +Ġpro pos +Ġlaun ch +Ġbott om +T H +ĠP lease +v est +it z +ĠIn ter +Ġsc ript +Ġr at +ar ning +Ġ il +ĠJ er +ĠA re +Ġwh atever +ok en +ci ence +Ġmod e +Ġag ree +Ġs ources +Ġinit ial +Ġrest rict +Ġwond er +us ion +## ## +ĠS il +vil le +Ġb urn +t w +as ion +Ġ £ +Ġn or +u ing +Ġre ached +Ġs un +Ġc ateg +ig ration +Ġc ook +Ġprom ot +Ġm ale +Ġcl imate +Ġf ix +Ġalleg ed +U R +all ed +Ġim ages +C ont +ot a +Ġschool s +i os +Ġd rop +Ġst ream +ĠM o +Ġprevious ly +al ing +Ġp et +Ġdou ble +Ġ( @ +ann el +Ġdef ault +t ies +Ġr ank +ĠD ec +ĠCoun cil +Ġweap on +Ġst ock +Ġanal y +ĠSt r +Ġpict ure +ĠPol ice +f erence +Ġcent ury +Ġcitiz ens +Ġon to +Ġexp and +Ġhe ro +ĠS ol +Ġw ild +Ġupd ate +Ġcustom ers +r ont +d ef +Ġl ik +Ġcrim inal +ĠChrist ian +S P +7 6 +Ġle aving +Ġother wise +ĠD ist +Ġbas is +5 2 +5 3 +ic ip +ĠB er +Ġrecomm end +Ġfl oor +Ġc rowd +ol es +Ġ7 0 +Ġcent ral +ĠE v +Ġd ream +Ġdown load +Ġconf ir +ĠTh om +Ġwind ow +Ġhapp ens +Ġun it +Ġt end +Ġs pl +Ġbec omes +Ġfight ing +Ġpred ict +ĠP ress +ĠP ower +Ġhe avy +ak ed +Ġf an +or ter +ate gy +B A +iz es +Ġsp end +H ere +Ġ200 7 +Ġad op +ĠH am +Ġfoot ball +ĠP ort +od ay +5 1 +amp ions +Ġtrans fer +h t +Ġ3 8 +ter m +ac ity +Ġb ur +] , +tern al +r ig +b ut +Ġthere fore +ĠB ecause +res p +re y +Ġm ission +S ome +Ġnot ed +Ġass um +Ġdise ase +Ġed it +Ġprog ress +r d +ĠB rown +oc al +Ġadd ing +Ġra ised +ĠAn y +Ġt ick +Ġsee ing +ĠPe ople +Ġagre ement +Ġser ver +Ġw at +Ġdeb ate +Ġsupp osed +il ing +Ġlarg est +Ġsuccess ful +ĠP ri +ĠDemocr atic +Ġj ump +ĠSyri a +Ġown ers +Ġoff ers +Ġshoot ing +Ġeff ic +se y +Ġha ven +ver se +te red +ĠL ight +im al +ĠB ig +Ġdef end +Ġbe at +Ġrecord s +% ) +Ġsc en +Ġemploy ees +Ġdev ices +he m +Ġcom mer +ĠM ex +Ġbenef it +ĠPro f +Ġil leg +Ġsur face +ĠAl so +Ġh arm +ing ly +w ide +ĠA lex +Ġsh ut +ĠC ur +Ġl ose +p m +Ġchall enge +se mb +Ġst ation +Ġint elligence +Ġacc ur +ĠFl or +Ġrequ ires +ĠM al +b um +Ġh ospital +Ġsp irit +Ġoff ered +Ġprodu ce +ĠComm un +Ġcreat ing +Ġcr is +s pect +Ġend ed +Ġd aily +Ġvot ers +land s +i as +i h +on a +Ġsm art +ĠOff ice +ĠL ord +ri al +ĠIntern et +Ġcirc um +Ġextreme ly +' . +Ġopin ion +ĠM il +Ġg ain +B S +ĠF in +y p +Ġuse ful +Ġbud get +Ġcom fort +is f +Ġback ground +el ine +Ġep isode +Ġen emy +Ġtri al +Ġestab lish +d ate +ĠC ap +Ġcontin ues +Ġshow ing +ĠUn ion +w ith +Ġpost ed +ĠSy stem +Ġe at +ri an +Ġr ise +ĠGerman y +il s +Ġsign ed +Ġv ill +Ġgr and +m or +ĠEng land +Ġproject s +um ber +Ġconf erence +z a +Ġrespons ible +ĠAr ab +Ġlearn ed +âĢĶ âĢĶ +i pping +ĠGe orge +O C +Ġreturn ed +ĠAustral ia +Ġb rief +Q u +Ġbr and +ill ing +ab led +Ġhig hest +Ġtr ain +ĠComm ission +wh ile +Ġn om +cept ion +Ġm ut +ĠBl ue +Ġinc ident +v ant +8 6 +ĠI D +Ġn uclear +7 4 +ĠL ike +ĠR E +ĠM icro +l i +m ail +Ġcharg es +8 9 +Ġad just +ad o +Ġear th +N A +Ġpr ices +P A +Ġd raft +Ġrun s +Ġcandid ate +ens es +Ġmanag ement +ĠPh il +ĠM iss +Ġte ach +g ram +Ġunderstand ing +a it +ic ago +A dd +ĠE p +sec ut +Ġsepar ate +Ġinst ance +Ġe th +Ġun less +**** **** +ĠF ore +in ate +Ġoper ations +S p +Ġf aith +g ar +ĠCh urch +ron ic +Ġconf ig +os ure +Ġactiv ities +Ġtrad itional +Ġ3 6 +Ġd irection +Ġmach ine +Ġsur round +Ġp ush +un ction +ĠE U +Ġeas ier +Ġarg ument +G B +Ġm icro +Ġsp ending +iz ations +Ġthe ory +ad ow +Ġcall ing +ĠL ast +Ġd er +Ġinflu ence +Ġcomm it +Ġph oto +Ġun c +ist ry +g n +ast e +ack s +Ġdis p +ad y +d o +ĠG ood +Ġ ` +Ġw ish +Ġreve aled +Âł Âł +l ig +Ġen force +ĠComm ittee +Ġche m +Ġmil es +Ġinterest ed +Ġsol ution +ic y +in ct +Ġ- > +ĠD et +Ġrem oved +Ġcomp ar +e ah +Ġpl ant +ĠS ince +Ġachie ve +Ġadvant age +Ġslight ly +b ing +Ġpl aced +u nder +201 5 +ĠM ad +Ġt im +os es +Ġc ru +ĠR ock +Ġmost ly +Ġneg ative +Ġset ting +Ġprodu ced +Ġm ur +Ġconnect ion +ĠM er +Ġdri ver +Ġexecut ive +Ġass ault +Ġb orn +ĠV er +t ained +Ġstruct ure +Ġredu ce +Ġdec ades +Ġd ed +u ke +ĠM any +idd en +Ġle ague +S e +Ġjo in +Ġdis co +Ġd ie +c ks +act ions +Ġass ess +ag n +Ġgo als +our s +I R +Ġsen ior +ill er +m od +ip ment +oc ol +u y +ĠQ ue +Ġpart ies +ir gin +Ġle arning +it able +Ġstre et +Ġcamer a +A pp +Ġsk ills +b re +c ious +Ġcele br +ĠFr anc +Ġexist ing +Ġwill ing +l or +Ġ id +ĠSp ace +Ġcrit ical +ĠL a +ortun ately +Ġser ve +Ġc old +Ġspec ies +T S +Ġanim als +ĠB ay +Ġold er +ĠU nder +est ic +ĠT re +Ġte acher +Ġpre fer +v is +Ġth read +ĠM att +Ġmanag er +ãĥ » +Ġprofess ional +ĠV ol +Ġnot es +The se +ul a +Ġf resh +ent ed +u zz +ed y +clus ion +ĠR el +Ġdoub t +E O +Ġopen ed +ĠB it +Ad vertisement +Ġgu ess +ĠU N +Ġse qu +Ġexpl ain +ott en +Ġatt ract +ak s +Ġstr ing +Ġcont ext +oss ible +ĠRepublic ans +Ġsol id +Ġc ities +Ġask ing +Ġr andom +u ps +ur ies +ar ant +dd en +g l +ĠFlor ida +Ġdep end +ĠSc ott +Ġ3 3 +Ġi T +ic on +Ġmention ed +Ġ2 000 +Ġclaim ed +Ġdefin itely +ul f +Ġc ore +Ġopen ing +ĠCon st +wh ich +ĠT ra +A G +7 2 +Ġbelie ved +ad a +Ġ4 8 +ĠSec urity +yr ight +ĠP et +ĠL ou +Ġhold ing +======== ======== +Ġ ice +Ġb row +Ġauthor ities +h ost +w ord +Ġsc ore +ĠD iv +Ġcell s +Ġtrans l +Ġneigh bor +Ġrem ove +u ct +Ġdist rict +ĠA ccording +Ġwor se +Ġconcern s +Ġpresident ial +Ġpolic ies +ĠH all +7 3 +Ġh us +A Y +Ġ200 6 +ĠJ ud +Ġindepend ent +ĠJust ice +ili ar +pr int +igh ter +Ġprotect ion +z en +Ġsu dden +h ouse +ĠJ es +P R +ĠIn f +Ġb ul +Ġ _ +ĠServ ice +ĠP R +Ġstr ategy +ff ect +Ġgirl s +Ġmiss ing +oy al +ĠTe am +ul ated +Ġd at +Ġpolit ics +ab or +A ccording +Ġspe ll +Ġg raph +ort hern +T C +A b +Ġlab or +is her +Ġk ick +ĠiT unes +Ġstep s +pos es +Ġsmall er +E n +ber t +Ġro ll +Ġresear chers +Ġcl osed +Ġtrans port +Ġlaw y +________ ________ +ĠCh icago +Ġas pect +Ġn one +Ġmar riage +9 6 +Ġe lements +ĠF re +ĠS al +Ġd ram +F C +t op +e qu +Ġhe aring +Ġsupport ed +Ġtest ing +co hol +Ġmass ive +Ġst ick +Ġgu ard +is co +ph one +F rom +How ever +Ġb order +Ġcop y +ograph y +l ist +7 1 +Ġown er +cl ass +ru it +r ate +ĠO nce +Ġdig ital +Ġt ask +ER S +Ġinc red +t es ++ + +ĠFr ance +Ġb reat +ow l +Ġiss ued +ĠW estern +Ġdet ect +Ġpart ners +Ġsh ared +ĠC all +Ġcan cer +ac he +rib e +Ġexpl ained +Ġhe at +{ " +Ġinvest ment +ĠB ook +Ġw ood +Ġtool s +ĠAl though +Ġbelie f +Ġcris is +Ġg e +ĠM P +Ġoper ation +ty pe +~ ~ +g a +Ġcont ains +ant a +Ġexp ress +ĠG roup +ĠJ ournal +k a +Ġam b +ĠUS A +Ġfind ing +Ġfund ing +h ow +Ġestab lished +ide os +Ġdeg ree +Ġdanger ous +ang ing +Ġfre edom +pp ort +out hern +Ġch urch +Ġc atch +ĠTw o +Ġpres ence +ĠGu ard +U p +Ġauthor ity +ĠPro ject +Ġbut ton +Ġcon sequ +Ġval id +Ġwe ak +Ġstart s +Ġref erence +ĠM em +" ) +U N +or age +ĠO pen +Ġcol lection +y m +g ency +Ġbeaut iful +ro s +Ġtell s +Ġwa iting +n el +Ġprov iding +ĠDemocr ats +Ġd aughter +Ġm aster +Ġpur poses +ĠJapan ese +Ġequ al +Ġturn s +Ġdoc uments +Ġwatch ing +R es +Ġr an +201 4 +Ġre ject +ĠKore a +Ġvictim s +Le vel +ere nces +Ġw itness +Ġ3 4 +Ġre form +com ing +Ġocc up +Ġc aught +Ġtra ffic +ad ing +Ġmod els +ar io +Ġserv ed +Ġb atter +u ate +ĠSecret ary +Ġagre ed +Ġtr uly +yn am +ĠR et +Ġun its +ĠRes earch +h and +az ine +ĠM ike +Ġvar iety +ot al +Ġam azing +Ġconfir med +Ġentire ly +Ġpurch ase +Ġe lement +Ġc ash +Ġdeter mine +D e +Ġc ars +ĠW all +â ĸ +Ġview s +Ġdrug s +Ġdep artment +ĠSt ep +u it +Ġ3 9 +as ure +ĠCl ass +Ġc overed +ĠB ank +Ġme re +u ana +Ġmult i +Ġm ix +Ġun like +lev ision +Ġsto pped +Ġs em +ĠG al +ul es +Ġwe l +ĠJohn son +l a +Ġsk ill +Ġbec oming +ri e +Ġappropri ate +f e +ell ow +ĠPro t +ul ate +oc ation +Ġweek end +od ies +Ġsit es +Ġanim al +ĠT im +Ġsc ale +Ġcharg ed +Ġinst ruct +ill a +Ġmethod s +Ġc ert +Ġjud ge +ĠH el +Ġdoll ars +Ġstand ing +ĠS qu +Ġdeb t +l iam +Ġdri ving +ĠS um +ĠEd ition +Ġal bum +and on +I F +ĠU k +6 3 +ad er +Ġcommer cial +es h +ĠGovern ment +Ġdisc overed +Ġout put +ĠHill ary +ĠCar ol +Ġ200 5 +Ġab use +anc ing +Ġsw itch +Ġann ual +T w +Ġst ated +ag ement +in ner +Ġdem ocr +Ġres idents +Ġallow ing +Ġfact ors +od d +Ġf uck +em ies +Ġoccur red +ot i +Ġn orth +ĠP ublic +Ġinj ury +Ġins urance +C L +oll y +ã Ģ +Ġrepe ated +Ġar ms +ang ed +Ġconst ruction +Ġf le +P U +ic ians +Ġfor ms +ĠMc C +ant ic +Ġm ental +p ire +Ġequ ipment +Ġf ant +Ġdiscuss ion +Ġregard ing +k in +ar p +Ġch air +og ue +Ġpro ceed +ĠI d +O ur +Ġmur der +M an +Ġ4 9 +as p +Ġsupp ly +Ġin put +Ġwe alth +liam ent +Ġpro ced +or ial +ĠSt at +ĠN FL +hen s +ĠInst itute +Ġput ting +ourn ament +et ic +Ġloc ated +Ġk id +er ia +r un +Ġpr inc +Ġ ! +go ing +ĠB et +Ġcl ot +Ġtell ing +Ġprop osed +i ot +or ry +Ġfund s +g ment +ĠL ife +Ġb aby +ĠB ack +Ġsp oke +Im age +Ġear n +ĠA T +g u +Ġex change +ĠL in +ov ing +Ġp air +M ore +az on +Ġarrest ed +Ġkill ing +c an +ĠC ard +y d +Ġident ified +Ġm obile +Ġthan ks +ony m +ĠF orm +Ġhundred s +ĠCh ris +ĠC at +Ġtre nd +h at +ĠA v +om an +Ġelect ric +ĠW il +S E +O f +Ġrest aur +ot ed +Ġtr ig +Ġn ine +Ġb omb +Wh y + ¯ +Ġco verage +Ġapp eal +ĠRober t +ĠS up +Ġfin ished +Ġfl ow +Ġdel iver +Ġcal cul +Ġphot os +Ġph il +Ġpie ces +Ġapp re +k es +Ġr ough +D o +Ġpart ner +Ġconcern ed +Ġ3 7 +ĠG en +C ol +ct ors +Ġ= > +st ate +Ġsuggest ed +ĠFor ce +C E +Ġher self +ĠPl an +w orks +o oth +ren cy +Ġcor ner +Ġhus band +Ġintern et +ĠA ut +em s +os en +ĠAt l +g en +Ġbal ance +6 2 +Ġsound s +te xt +Ġar r +ov es +Ġmill ions +Ġrad io +Ġsat isf +ĠD am +M r +G o +S pe +Ġcomb at +r ant +ĠG ree +Ġf uel +Ġdist ance +Ġtest s +Ġdec re +ĠE r +Ġman aged +D S +Ġt it +Ġmeas ures +ĠL iber +Ġatt end +as hed +ĠJ ose +ĠN ight +d it +ĠN ov +ĠE nd +out s +Ġgener ation +Ġadv oc +y th +Ġconvers ation +ĠS ky +act ive +ce l +ri er +ĠFr ank +Ġg ender +Ġcon cent +Ġcar ried +and a +ĠV irgin +Ġarri ved +ic ide +ad ed +Ġfail ure +Ġmin imum +le ts +Ġwor st +Ġkeep ing +Ġint ended +Ġilleg al +Ġsub sc +Ġdetermin ed +Ġtri p +Y es +Ġra ise +Ġ ~ +Ġfeel s +Ġpack age +ĠJ o +h i +201 6 +re al +Ġf ra +Ġsy mb +M e +uck y +p ret +ĠK h +ĠEd it +ĠWe b +em ic +ĠCol or +Ġjust ice +I nt +Ġfar m +ck now +" > +el ess +Ġredu ced +Ġ5 00 +x x +ĠR ad +ĠW ood +Ġcl in +Ġhy p +il er +ur a +k ins +8 5 +6 1 +ĠThe ir +ĠM ary +Ġs an +Ġno vel +ĠWh o +Ġcap acity +Ġimp ossible +Ġpl ays +Ġmin ister +ij uana +ic ate +ĠS et +Ġf ram +Ġ ing +Ġcommun ities +ĠF BI +it a +Ġb on +Ġstr ateg +Ġinterest s +l ock +g ers +m as +ĠAN D +Ġconflic t +Ġrequire ments +Ġs ac +Ġoper ating +in i +rel ated +Ġcomm itted +Ġrelative ly +Ġs outh +¯ ¯ +Ġaff ord +Ġident ity +Ġdec isions +Ġacc used +pl ace +Ġvict ory +o ch +i at +N ame +C om +t ion +ed s +Ġsee k +Ġt ight +ĠIm ages +Ġinit i +Ġhum ans +Ġfam iliar +Ġaud ience +Ġintern al +vent ure +Ġs ides +ĠT O +Ġd im +Ġcon clud +Ġapp oint +Ġenforce ment +ĠJ im +ĠAssoci ation +Ġcircum st +ĠCanad ian +Ġjo ined +Ġdiffere nces +ĠL os +Ġprot est +Ġtw ice +w in +Ġgl ass +ars h +ĠAr my +Ġexp ression +Ġdec ide +Ġplan ning +an ia +Ġhand le +ĠMicro soft +ĠN or +Ġmax imum +ĠRe v +Ġse a +Ġev al +Ġhel ps +re f +Ġb ound +Ġm outh +Ġstand ards +Ġcl im +ĠC amp +ĠF ox +cl es +Ġar my +ĠTe chn +ack ing +x y +S S +Ġ4 2 +Ġbu g +ĠUk rain +ĠM ax +ĠJ ones +ĠSh ow +l o +Ġplan et +Ġ7 5 +Ġwin ning +Ġf aster +Ġspe ct +Ġbro ken +T R +Ġdef ined +Ġhealth y +Ġcompet ition +htt ps +ĠIs land +ĠF e +Ġannoun ce +ĠC up +ĠInst ead +Ġcl ient +Ġposs ibly +se ction +ock et +l ook +Ġfin ish +Ġcre w +Ġres erv +Ġed itor +Ġh ate +Ġs ale +Ġcontro vers +Ġp ages +w ing +Ġnum er +Ġopp osition +Ġ200 4 +Ġref uge +Ġfl ight +Ġap art +ĠL at +A meric +ĠAfric a +Ġapplic ations +ĠPal est +ĠB ur +Ġg ar +ĠSoc ial +Ġup gr +Ġsh ape +Ġspe aking +ans ion +a o +ĠS n +Ġwor ry +ĠBrit ain +P lease +rou d +Ġh un +Ġintrodu ced +Ġd iet +I nd +ĠSec ond +Ġfun ctions +ut s +ĠE ach +ĠJe ff +Ġst ress +Ġaccount s +Ġgu arant +ĠAn n +ed ia +Ġhon est +Ġt ree +ĠAfric an +ĠB ush +} , +Ġs ch +ĠOn ly +Ġf if +ig an +Ġexerc ise +ĠEx p +Ġscient ists +Ġlegisl ation +ĠW ork +ĠS pr +à Ĥ +ĠH uman +Ġ è +Ġsur vey +Ġr ich +ri p +Ġmain tain +Ġfl o +Ġleaders hip +st ream +ĠIslam ic +Ġ 01 +ĠCol lege +Ġmag ic +ĠPr ime +Ġfig ures +201 7 +ind er +x ual +ĠDe ad +Ġabsolute ly +Ġfour th +Ġpresent ed +resp ond +rib le +Ġal cohol +at o +ĠD E +por ary +Ġgr ab +Ġvar i +Ġqu ant +ĠPh oto +Ġpl us +r ick +ar ks +Ġaltern ative +Ġp il +Ġappro x +th at +Ġobject s +ĠR o +ĠAnd roid +Ġsignificant ly +ĠR oad +k ay +R ead +av or +Ġa cknow +ĠH D +ĠS ing +O r +ĠM ont +Ġun s +pro f +Ġneg oti +ĠAr ch +ik i +Ġte levision +ĠJew ish +Ġcomm ittee +Ġmot or +Ġappear ance +Ġs itting +Ġstri ke +ĠD own +com p +ĠH ist +Ġf old +ac ement +ĠLou is +Ġbel ong +ĠâĢ ¢ +Ġm ort +Ġprep ared +Ġ6 4 +ĠM aster +Ġind eed +ĠD en +Ġre nt +T A +our ney +ar c +S u +9 7 +Ġadv ice +Ġchang ing +Ġlist ed +Ġlaun ched +is ation +ĠP eter +is hes +Ġl ived +ĠM el +ĠSup reme +ĠF ederal +Ġ) ; +ruct ure +Ġset s +Ġphil os +u ous +Ġ ł +Ġappl ied +ĠN OT +Ġhous ing +ĠM ount +Ġo dd +Ġsu st +D A +ffic ient +Ġ ? +ol ved +Ġp owers +Ġth r +Ġrem aining +ĠW ater +L C +Ġca uses +ãģ ® +Ġman ner +ad s +Ġsuggest s +Ġend s +stand ing +f ig +ĠD un +id th +Ġg ay +Ġter min +ĠAngel es +M S +Ġscient ific +Ġco al +ap ers +b ar +ĠThom as +Ġsy m +ĠR un +th is +P C +igr ants +Ġmin ute +ĠDist rict +cell ent +Ġle aves +Ġcomple ted +am in +Ġfoc used +Ġmon itor +Ġveh icles +M A +ĠM ass +ĠGr and +Ġaffect ed +itution al +Ġconst ruct +Ġfollow s +Ġt on +re ens +Ġh omes +ĠE xt +ĠLe vel +r ast +ĠI r +Ġel im +Ġlarge ly +ĠJ oe +Ġvot es +all s +Ġbusiness es +ĠFound ation +ĠCent ral +Ġy ards +Ġmaterial s +ul ner +Ġgu ide +Ġclos er +um s +Ġsp orts +ed er +J ust +Ġtax es +8 4 +ĠO ld +Ġdec ade +ol a +Ġv ir +Ġdro pped +Ġdel ay +it ect +Ġsec ure +ste in +le vel +Ġtre ated +Ġfil ed +ain e +Ġv an +Ġm ir +Ġcol umn +ict ed +e per +Ġro t +Ġcons ult +Ġent ry +Ġmar ijuana +ĠD ou +Ġapparent ly +ok ing +clus ive +Ġincre ases +an o +Ġspecific ally +Ġte le +ens ions +Ġrelig ion +ab ilities +Ġfr ame +ĠN ote +ĠLe e +Ġhelp ing +Ġed ge +ost on +Ġorgan izations +à ĥ +ĠB oth +hip s +Ġbig ger +Ġbo ost +ĠSt and +Ġro w +ul s +ab ase +Ġr id +L et +are n +ra ve +Ġst ret +P D +Ġv ision +Ġwe aring +Ġappre ci +Ġa ward +ĠU se +Ġfact or +w ar +ul ations +) ( +Ġg od +Ġter rit +Ġpar am +ast s +8 7 +Ġen emies +ĠG ames +F F +Ġacc ident +W ell +ĠMart in +T ER +Ġat h +ĠHe ll +Ġfor g +Ġve ter +ĠMed ic +f ree +Ġst ars +Ġexp ensive +Ġac ad +ra wn +ĠW he +Ġl ock +Ġform at +Ġsold iers +s m +Ġag ent +Ġrespons ibility +or a +ĠS cience +Ġrap id +Ġt ough +ĠJes us +Ġbelie ves +M L +Ġwe ar +le te +Ãĥ ÃĤ +ĠD ri +Ġcomm ission +ĠB ob +O h +ap ed +Ġwar m +ÃĥÃĤ ÃĥÃĤ +Ġ200 3 +ort ion +Ġhas n +ust er +Ġun ivers +ĠI ll +Ġk ing +olog ies +9 4 +ĠT em +ĠM os +Ġpat ient +ĠMex ico +ce an +ĠDe ath +ĠSand ers +y ou +ĠC ast +ĠComp any +pt y +Ġhappen ing +F P +ĠB attle +Ġb ought +A m +M od +U s +ut ers +ĠC re +ĠTh ose +Ġ4 4 +is er +Ġs oul +ĠT op +ĠHar ry +ĠA w +Ġse at +ff ee +Ġrev olution +Ġ( " +ĠD uring +et te +Ġr ing +Ġoff ensive +Ġreturn s +Ġv ideos +Ġdis cl +Ġfam ous +en ced +ĠS ign +ĠR iver +Ġ3 00 +P M +ĠB us +ĠC H +Ġcandid ates +ard en +Ġpercent age +Ġvis ual +Ġthan k +Ġtrou ble +ner gy +Ġ200 1 +Ġpro ve +ash ion +Ġen h +ĠL ong +U M +Ġconnect ed +Ġposs ibility +O ver +Ġexper t +Ġl ibrary +art s +ĠDirect or +Ġfell ow +9 2 +ir ty +Ġd ry +Ġsign s +ĠL ove +Ġqu iet +f oot +Ġp ure +ĠH un +Ġf illed +ph as +ĠE lect +end ment +ĠEx pl +Ġun able +n s +m o +Ġv ast +ob e +Ġident ify +app ing +ĠCarol ina +g ress +Ġpro te +Ġf ish +Ġcircumst ances +raz y +ĠPh ot +Ġb odies +ĠM ur +Ġdevelop ing +ĠA R +Ġexperien ced +Ġsubst ant +ĠBo ard +es ome +Ġdom estic +Ġcomb ined +ĠP ut +Ġchem ical +ĠCh ild +Ġpo ol +ĠC y +Ġe gg +c ons +st ers +Ġh urt +Ġmark ets +Ġconserv ative +Ġsupp orters +Ġag encies +id el +O b +ur b +Ġ4 3 +ĠDef ense +y e +ĠA p +du le +Ġtemper ature +Ġconduct ed +ĠCh ief +Ġpull ed +Ġf ol +L ast +ont o +os is +V ER +D es +ĠP an +F irst +Ġadv ance +Ġlic ense +r ors +ĠJ on +Ġimag ine +Ġhe ll +Ġf ixed +Ġinc or +os ite +ĠL og +ick en +] : +Ġsurpr ise +h ab +Ġc raft +ol t +ĠJ ul +Ġd ial +Ġrele vant +Ġent ered +Ġlead s +ĠA D +ĠCle an +Ġpict ures +ess or +Ġal t +Ġpay ing +P er +ĠMark et +Ġupd ates +am ily +ĠT ype +ĠH ome +Ġ5 5 +semb ly +rom e +8 3 +Ġgreat est +Ġhe ight +Ġhe av +ain ts +Ġlist en +as er +ĠS H +Ġcap able +ac le +Ġpers pect +in ating +Ġoff ering +ry pt +ĠDe velop +ab in +r c +Ġbr ight +al ty +ar row +Ġsupp l +ind ing +ack ed +gy pt +ĠAn other +p g +ĠVirgin ia +ĠL u +Ġpl anned +Ġp it +Ġswe et +T ype +ĠD i +Ġtyp ically +ĠFranc isco +Ġpro spect +ĠD an +Ġte en +re es +Ġsc hed +Ġh ol +Ġsc r +Ġlot s +l ife +Ġnews p +Ġfor get +ĠN one +ĠM iddle +ĠR yan +ed d +Ġse vere +Ġsu it +ll er +9 3 +Ġcor respond +Ġexpl os +u ations +Ġfl ag +g ame +r id +Ġpr in +ĠD ata +Ġde ploy +ĠEn ter +su it +gh an +ĠM en +Ġthough ts +Ġmat ters +Ġad apt +ĠA ri +Ġf ill +Ġfor th +Ġs am +Ġ4 1 +Ġpay ment +ĠH or +Ġsp ring +du c +Ġl osing +Ġbring ing +F O +al a +Ġdist ribution +he red +b our +ĠIsrael i +om a +Ġcomb ination +Ġpl enty +V E +C an +ĠH aw +Ġper man +ĠSpe cial +Ġto w +Ġsee king +Ġexam ples +Ġclass es +c r +Ġbe er +Ġmov es +ĠI P +ĠK n +Ġpan el +E ven +Ġproper ly +Ġr is +Ġpl ug +Ġestim ated +E very +Ġdef ensive +ag raph +Ġpre gn +Ġinst it +ĠV ict +Ġvol ume +Ġpos itions +Ġl inks +ĠPro gram +ĠWe ek +ag ues +Ġtrans form +k er +ĠC EO +Ġc as +Ġopp onent +Ġtwe et +ĠC ode +Ġsh op +Ġf ly +Ġtal ks +Ġb ag +Ph one +Ġa id +Ġpl ants +Ġ6 5 +Ġatt orney +ar ters +qu est +ĠMag ic +Ġbeg ins +Ġmy ster +Ġenvironment al +Ġst orage +N N +Ġm arg +Ġs ke +Ġmet al +ell y +Ġord ered +Ġrem ained +Ġl oved +Ġprom pt +Ġupd ated +Ġexper ts +Ġwalk ing +Ġan cient +Ġperform ed +AT E +Ġne ither +i ency +Ġmanufact ure +ĠP ak +Ġselect ed +Ġm ine +Ġult imately +Ġexpl an +Ġlab el +ĠServ ices +ribut ed +Tr ump +Ġsy n +ĠU lt +S C +Ġme at +Ġg iant +ĠW ars +ĠO N +Ġad m +Ġinter pret +Ġeven ing +Ġev il +ĠB oston +ĠW ild +Ġ à +ĠBit coin +ĠAm azon +D r +ĠIn formation +Ġobvious ly +Ġadv anced +Ph oto +ol ar +Ġwe ather +Ġsymb ol +Ġso le +Ġpot entially +ost er +Ġorig inally +m un +3 00 +az e +ess ions +Ġde ck +Ġst ood +Ġyou th +ĠB ern +R ep +ĠT est +Ġbas ically +ot ic +Ġinvol ve +ol it +ly n +S ee +Ġair craft +Ġconf irm +E W +Ġmess ages +ĠRich ard +Ġk it +Ġpro hib +Ġv ulner +is ters +Ġexist ence +Ġturn ing +ĠS P +Ġdes ire +Ġfl at +Ġm ent +se ason +ang es +Ġneighbor hood +ĠL ake +AT ION +Ġpoint ed +b ur +Ġinn ov +uc ks +U L +Ġprofess or +Ġexp ressed +A B +ic ious +Ġ200 2 +ĠDe v +Ġs ession +Ġb are +s en +Ġdis s +ĠC ath +ĠP ass +ĠP oint +Ġdo ctor +or row +ail ed +ĠR ub +ĠD C +ĠChar l +p erson +Ġwrit er +igh ters +ure au +Ġob lig +Ġrecord ed +Ġbro ke +Ġord ers +il ty +Ġmot ion +in ity +l aw +ad ium +Ġimm igration +Ġcontr ast +Ġb att +Ġex cellent +Ġtechn ical +am i +Ġt un +Ġcl oud +ĠY ear +ge on +Ġcre ation +Ġstr ange +Ġa uth +Ġfor t +b orn +Ġext ent +ĠT oday +ĠCl ub +Ġr ain +Ġs ample +Ġaccept ed +Ġt act +Ġf ired +ĠS on +Ġstand s +Ġb oot +Ġ4 7 +Ġstat ements +Ġvers ions +Ġse lling +ound ed +Ġ199 0 +Ġwere n +ĠW atch +Ġexper iment +P ost +Ġret ail +ul ed +In st +un te +ãĥ ¼ +Ġdep art +Ġb ond +i very +om pl +Ġre action +ĠSyri an +ĠP ac +app ed +ani el +D P +Ġres olution +Ġre act +Ġappro ved +on om +m ond +ĠO ffic +-- - +Ġrepl ace +Ġt ack +Ġsp ort +Ġch ain +Ġemer gency +r ad +ĠPalest in +Ġ4 6 +Ġautom atically +Ġrout e +Ġp al +Ġb anks +ĠPar is +ĠMed ia +ro ad +ic ing +i xt +ist ed +Ġg rew +Ġco ord +ĠW here +om in +Ġsub s +� � +Ġ ± +Ġcorpor ate +Ġse lection +n oon +ĠRep ort +c s +clud ing +ord ers +anc he +ĠIt s +Ġslow ly +ĠE gypt +ĠA cc +Ġcol le +iqu es +E X +Ġattempt s +ur l +ĠC ross +Ġfind ings +ĠS C +ĠO R +Ġind ex +ens ity +ĠW ay +ĠL and +Ġsh ock +d is +Ġd ynam +Ġc art +m osp +S ince +i est +ĠB oy +Ġst orm +ĠCont in +201 3 +he w +il it +Ġess ential +iqu id +O ther +ive red +Ġreason able +A ct +Ġsub sequ +ĠP ack +ĠF ort +Ġconsider ing +Ġun iversity +l og +Ġmar ried +Ġill ust +ĠTr ue +£ ı +Ġnumer ous +rast ructure +Ġserious ly +Ġrefer red +u a +Ġconsist ent +on na +ĠRe al +ru ption +ci ples +Ġfact s +9 1 +ot es +er g +The n +Ġacc ompl +N ote +Ġre venue +Ġpass ing +Ġm al +e en +ĠY et +Ġg ather +ter day +ew ork +ĠA uthor +P e +Ġopt im +Ġr ub +Ġè £ı +Ġun known +st one +Ġun ion +ol ve +Ġopportun ities +Ġbrow ser +ĠW al +ĠC ost +Ġreport ing +st s +p et +Ġs and +Ġsudden ly +Ġsurpr ising +ĠV R +Ġsomew hat +ĠB as +ult ure +iz z +ĠC D +Ġchalleng es +Ġsett ings +Ġexperien ces +ĠF ull +Ġcan n +Ġrece iving +ES T +Ġj oint +Ġcult ural +Ġa st +8 2 +as tern +ce ived +ĠC ru +Ġb ull +p ired +am m +Ġfac ing +p ower +Ġb oss +ĠH ol +Ġinst r +Ġincreasing ly +Ġsh ift +Ġstre ets +ĠWilliam s +ab b +Ġl ie +Ġl augh +ĠC a +P L +Ġadult s +Ġcustom er +Ġob tained +Ġsupport ing +ht ml +f ire +Ġdetail ed +Ġpick ed +ĠR ight +ld er +E E +st ood +ĠK im +Ġw ire +Ġs ight +Ġdevelop ers +Ġpers ons +Ġs ad +Ġc up +Ġwar ning +Ġboy s +l ong +Ġb ird +f o +Ġw al +Ġobserv ed +Ġz one +iven ess +Ġch annel +c ript +Ġref used +ĠAg ain +Ġsu c +Ġspokes man +ĠRe f +r ite +ou ston +ãĥ ³ +ĠS her +Ġact s +ĠN ame +Ġstrugg le +ar ry +omet imes +Ġdisc rim +H T +Ġcateg ory +Ġreal ize +Ġemploy ee +ĠAf ghan +en ger +Ġgun s +ĠSte ve +ĠM ot +ĠO l +ok ed +Ġth ick +Ġfair ly +ill y +Ġsur ve +ĠM at +we ight +â Ķ +Ġtro ops +Ġag ents +Ġbatter y +Ġmot iv +à ¡ +S ec +d en +o very +L S +Ġfl u +Ġconf ident +ĠO per +Ġem pty +Ġp hen +Ġse ctor +Ġexc ited +Ġrem ote +ap h +o en +Ġdestroy ed +Ġmor al +ĠH P +ĠR on +Ġd ress +ĠB at +Ġl it +ĠM S +Ġa f +H L +r um +is ms +Ġshould n +Ġsym pt +ĠTor onto +het ic +Ġcar bon +Ġinstall ed +Ġviol ent +Ġsol ar +j a +Ġpract ices +Ġr ide +ĠP enn +Ġimpro ved +Ġaud io +Ġbehav i +ĠP S +Ġe ating +D ata +ĠRe view +p ass +cl aim +u ated +ang ers +c hen +Ġproper ties +Ġany where +An other +Ġbl ow +ĠJack son +Ġp roud +Ġplan e +l ines +Ġsqu are +Ġpro of +ans as +Ġtalk ed +m akers +Ġs ister +Ġhold s +Ġres ident +Ġ= = +Ġresist ance +Ġspl it +Ġpro secut +Ġconf idence +res ents +Ġcut s +Ġexcept ion +Ġz ero +Get ty +Ġcop yright +Ġtot ally +orm al +ific ations +ĠAustral ian +Ġs ick +Ġ1 50 +Ġhouse hold +Ġfe es +Ġdri vers +og en +ĠN Y +Ġnecess arily +Ġregul ations +ear ing +s l +Ġperspect ive +c are +ic ial +H is +Ġesc ape +Ġsurpr ised +ĠV an +ur rent +Ġv ac +8 1 +ĠTh us +Ġem phas +ĠCh ampions +ĠI ce +Ġn arr +Ġhead s +Ġca using +b el +f ortunately +ĠM a +Ġtarg ets +ci pl +Ġafter noon +Ġadd s +ĠMay be +ĠF our +ess ed +ple te +Ġus ual +ch o +ing u +Ġwith d +ĠE nergy +ĠE conom +O O +Ġart icles +Ġinj ured +Ġman age +Ġexpl ains +Ġdi agn +R ec +at ures +Ġlink ed +Ġdiscuss ed +Ġexpl o +Ġocc asion +ath an +Ġopp osite +Ġfac es +Ġden ied +ĠK night +Ġn ut +Ġapprox imately +Ġdisapp oint +onym ous +ĠB est +ĠL o +ĠH y +ĠA ff +Ġvot ing +an while +ĠII I +Ġinstit utions +ag ram +ĠD aily +Ġdr ag +Ġnear by +Ġgu ilty +Ġcon ver +P re +s hip +Ġre ward +Ġphilos oph +ĠS S +u gh +Ġapp s +f riend +Ġu pper +Ġad vert +Ġs now +Ġfr ust +Ġour selves +F r +ĠD ie +amp ion +Ġdis miss +Ġc ere +Ġsign al +f rom +Ġ ). +Ġ5 2 +Ġcr imes +it ors +est ival +use um +Ġcoun cil +ĠS aud +M ay +ĠG un +ic ian +et her +Ġsu fficient +ĠH en +so le +Ġhistor ical +ĠF ar +ĠT urn +Ġp in +Ġsuc ceed +m at +ly mp +Ġtrad ition +ĠO k +Ġc ro +Ġdesc ription +al le +Ġsk y +T e +Ġwide ly +Ġw ave +Ġdefin ition +ĠJew s +Ġcy cle +Ġref ere +Ġbr ings +us al +Ġal ive +Ġfrequ ently +Ġint ention +ĠCont rol +l v +y stem +Ġpriv acy +g ent +ren ce +ĠQu est +ĠChrist mas +Ġr ail +Ġco oper +Ġtest ed +ĠC apt +as ks +Ġcomfort able +Ġdel ivered +sc ape +Ġdep th +ĠG OP +Ġwrit es +Ġass ets +Ġsa v +im ents +Ġtrans ition +Ġart ist +ĠL ook +Ġl ob +Ġcomp onents +ar ity +Ġwalk ed +Ġro ot +Ġparticip ants +Ġnot iced +Ġres c +Ġn av +ĠAd minist +d a +ut ral +pl ate +Ġimport ance +Ġass ert +ious ly +c ription +Ġinj uries +ĠChe ck +Ġregist ered +Ġint ent +Ġmiss ed +ograph ic +Ġsent ence +oun ter +Ġassist ance +ev in +Ġdat abase +Ġbuild ings +Ġclass ic +Ġth inks +ĠOh io +P r +ug g +Ġfe e +p an +Ġeffect ively +Ġfac ility +Ġbe ar +Ġch apter +Ġdog s +ĠCol umb +Ġl atter +it ial +Ġad mitted +T V +ĠGe org +Ġpost s +\ \ +Ġlawy er +Ġequ ival +Ġm and +Ġcontro lled +ĠW alk +ĠAnd rew +Ġmen u +am ental +Ġprotect ed +v a +Ġadminist r +or al +Ġre in +ĠS ar +Ġamount s +Ġn ative +ĠM oon +Ġrep resents +Ġab andon +Ġcarry ing +Ġt ank +m ary +Ġdecl ared +T ube +Ġh at +Ġpun ish +el lect +m es +Ġun iverse +ĠR od +ph y +Ġinf rastructure +Ġ5 1 +Ġopp osed +ow nt +c a +ĠM ake +Ġhard ware +Ġco ffee +R el +b al +w orld +ĠS af +ĠSe a +in als +Ġown ed +Ġh all +ers ion +Ġdescrib e +ĠP ot +Ġport ion +Ġat mosp +Ġgovern ments +Ġdep ending +Ġoff ense +Ġtr ick +aw a +ĠL ine +ĠV is +ĠH ard +ĠOr ig +ĠCl ick +Ġdes k +ĠVal ley +ĠS ov +Ġmov ies +Ġrem ark +Ġm ail +Ġcons cious +Ġrul ing +ĠR ights +Ġmed ic +he nt +ĠW omen +> < +Ġrepl aced +ĠP rem +ĠTh anks +Ġre new +ĠB all +if orm +Ġsh ots +C omm +Ġar med +Ġconst ant +Ġt aste +Ġreal ized +Ġbu ff +Ġm o +Ġeffic ient +M ost +or ation +if ies +Ġcommun ication +Ġfl ood +Ġconsequ ences +Ġany way +ig g +ĠG M +ĠTh ank +Ġ iron +Ġev olution +ĠC op +tw itter +Ġ9 5 +Ġrelationship s +ad el +ĠYou ng +Ġpropos al +ay ers +uild ing +ĠH ot +OR E +c os +Ġcoll abor +P G +ax y +Ġknow ing +Ġsupport s +ow ed +Ġcontrol s +Ġmere ly +um er +Ġath let +Ġf ashion +p ath +Ġg ift +Ġer a +AN D +Ġkind s +ĠKore an +Ġleg it +ul ous +Ġess entially +Ġthe rap +n ic +Ġsuff ered +Ġh ur +Ġprom ise +Ġex cess +Ġover w +Ġpr ime +ĠH ouston +er ry +ĠM s +R S +201 2 +Ġst ores +ĠO lymp +Ġj ourney +Al though +S ub +ĠE duc +ĠCh apter +Ġrequest s +Ġconsum ers +Ġt iny +Ġis ol +ĠF air +b a +ĠY OU +Ġcr ash +ce ler +Ġemot ional +Ġgood s +Ġelect ed +Ġmod er +ĠLin ux +Ġbl ocks +Ġis land +ĠSoc iety +Ġelect ions +Ġbroad cast +Ġche ap +Ġn ations +Ġse asons +4 00 +Ġwas te +ĠS at +Ġfield s +em ploy +Ġprof ile +Ġauth ors +AL L +ĠG ra +w est +ĠT y +Ġdeath s +Ġv acc +Ġfor med +Ġd u +Ġon going +ĠMuslim s +el f +ig ure +Ġass ume +ĠUkrain e +w ater +Ġco ast +Ġvot ed +g or +ĠA S +ĠMich igan +az a +ĠAr m +i ro +Ġf lex +as ters +' ' +Ġwel come +ar l +Ġloc ations +ig ation +ĠF il +Ġbu ying +Ġarch itect +Ġhard er +ĠC ub +Ġinter face +Ġrestaur ant +Ġdisco ver +Ġex ceed +Ġfav our +ger y +Ġd uty +Ġp itch +ad or +ĠM ach +b oy +Ġrespond ed +Ġext ended +her s +M any +ra id +if er +ĠIn s +S er +Ġmed ium +s he +ĠS ports +Ġmag azine +ut ation +Ġlim its +ĠG all +Ġex ternal +raz il +Ġyoung er +t le +Ġrem ind +ĠC ON +Ġimmedi ate +Ġh idden +Ġvol unte +Ġsim pl +od cast +Ġph ase +d r +Ġpl ot +Ġexp osure +R I +og rap +v in +an ish +ĠAc ad +ĠEng ine +Ġexp ansion +ĠP ay +Y our +Ġpus hed +ĠE ll +ĠHe ad +Ġmarket ing +ĠA C +k et +Ġh its +Ġg ro +ĠA ge +ĠSc ot +] [ +Ġst im +Ġi Phone +Ī Ĵ +Ġn arrow +ĠGet ty +ĠTur key +Ġperfect ly +Ġen able +ut ch +Ġprec ise +Ġreg ime +Ġsh if +Ġcomp ens +g un +d iv +Ġch osen +ĠK en +An y +Ġtre es +Ġrecomm ended +ĠR en +u able +ĠH T +F ollow +E G +ĠH and +ĠK enn +Ġarg uments +Ġex ists +Ġb ike +ĠCons erv +Ġbre aking +ĠG ar +Ġc razy +Ġvirt ual +ay lor +ix el +Ġ19 80 +Ġper mission +ĠSer ies +Ġconsum er +Ġclose ly +c alled +Ġ5 4 +Ġhop es +Ġar ray +ĠW in +ĠLab our +Ġsp ons +ĠI re +Ġp ow +Ġread ers +Ġemploy ment +Ġcreat ure +Ġresult ing +Ġaccur ate +Ġmom ents +Ġarg ued +Ġp ed +D uring +Ġ5 3 +ĠT al +Ġs ought +Ġsuff ering +Ġ icon +le e +Ġ( $ +al ian + ° +Ġp ra +Ġbon us +( " +k o +Ġact ing +D E +f all +Ġcompar ison +Ġsm ooth +ĠN AS +u pp +ĠJose ph +ep ing +ĠT ake +ĠM id +Ġs ending +f ast +ĠF all +Ġdeal ing +us er +ĠOr gan +C o +Ġatt ached +Ġse es +% . +Ġtyp ical +AR T +Ġfind s +ĠAs ia +um in +ĠC ore +ĠE nt +in ent +u ce +ĠBl ood +ĠN ever +Ġem ails +Ġhigh light +Ġconf ront +at us +ut ed +Ġun us +Ġtop ic +ĠAd am +Ġb le +at i +Ġunder stood +S et +st ruct +T P +Ġm ob +a a +ĠSt art +pect ed +se ll +Ġded icated +ĠC A +u an +Ġsong s +esc ription +Ġte ch +Ġr ape +Ġas ide +Ġgr ant +Ġ5 6 +s ub +Ġarg ue +Ġcont aining +Ġsche dule +Ġliber al +Ġpublic ly +Ġheav ily +ĠU t +in er +ĠS ection +ĠC are +we et +l s +D is +âĶ Ģ +ĠF ollow +B ack +ĠI T +Ġb es +j i +ĠH it +est ed +Ġevery body +ĠSw ed +Ġfem in +Ġfac ilities +Ġcon ven +C omp +ĠO S +c ore +Ġan x +Ġdiv ision +ĠC am +ĠSt an +m ates +Ġexpl ore +pl om +Ġsh ares +pl oad +an es +Ġide al +et ers +ĠB ase +Ġpl astic +Ġdist inct +ĠNet work +ĠSe attle +Ġtrad ing +ens us +int end +Ġex hib +Ġinit ially +ĠF ood +Ġthous and +ĠBus iness +act er +Ġpar agraph +Ġrough ly +Ġw ww +Ġcreat ive +ĠCon f +Ġconsum ption +Ġfil ms +ag an +Ġob tain +Ġt all +Ġt or +Ġacknow led +Ġg rown +al o +K E +Ġ4 00 +end ers +t aining +U G +Ġsu icide +Ġwat ched +ĠL ist +al i +re hens +Ġsurround ing +Ġp ip +Ġf lying +ĠJ ava +ord an +Ġserv ing +in ations +p ost +Ġsh o +A v +Ġj ail +z y +Ġ199 9 +Ġ< / +Ġliter ally +ĠS ir +Ġexp osed +Ġl ies +st ar +Ġb at +Ġear ned +ĠD ig +Ġspec ified +ĠSe ason +Ġdeg rees +Don ald +Ġcent re +Ġsh aring +Ġwin ter +ĠC O +C he +Ġ Î +M P +Ġun w +Ġfew er +ĠM ir +Ġsomew here +ĠK ey +Ġattack ed +ĠK ir +Ġdom ain +Ġstrong er +Ġ9 9 +Ġpen alty +I d +Sc ript +Ġdecl ined +Ġne ck +Ġfra ud +Ġcur rency +Ġr ising +R C +âĢ¦ âĢ¦ +H z +Ġt ab +Ġtal ent +n am +ĠN BA +Ġvill age +Ġleg s +ĠN ext +E d +Ġac id +Ġhy d +8 00 +Ġinvol ving +ĠIm age +ĠBe fore +F l +Ġyes terday +S ource +Ġterror ist +Ġsu p +Ġsy nt +ĠSaud i +Ġw est +Ġr u +b urg +Ġvis ible +Ġstru ck +r ison +Ġaw esome +Ġd rawn +Ġansw ers +ĠG irl +ĠR am +Ġthreat s +Ġdef eat +os it +Ġv ent +atur ally +Americ an +end a +ĠH oly +Ġr um +% , +c ase +ĠHist ory +ĠYou Tube +Ġsit uations +ĠD NA +S te +Ġsa ved +It em +Ġrec ip +olog ist +Ġfac ed +Ġel ig +O nce +ĠL i +u h +Ġmist ake +ĠDiv ision +ĠB ell +Ġsympt oms + ® +Ġdom in +Ġfall ing +Ġend ing +as hes +Ġmat ches +ĠOn line +Ġexplan ation +D ef +red it +Ġany more +ĠT otal +ĠF OR +us hed +Ġlet ters +Ġris ks +ĠO K +Ġreported ly +: \ +Ġpl ate +Ġsubject s +Ġattempt ed +if ier +ian a +Ġunlike ly +ĠTh ough +um a +ĠIn vest +ĠPr in +ic an +ĠD ar +ĠColor ado +au g +Ġve get +a os +ri a +Ġshe l +Ġmark ed +Ġ( ) +Ġsp r +p o +ĠL ink +Ġdef e +ĠJ r +Ġthem e +Ġpass ion +ĠP en +Ġinf o +iz er +Ġsh it +ĠC ivil +ap se +c re +Ġpo ly +Ġcomp onent +ĠChar les +ĠIre land +ĠPro v +Ġdo ctors +Ġgr anted +Ġpain t +Ġhon or +Ġsm oke +Ġpay ments +Ġprim arily +ĠKing dom +r ich +ate ll +Ġde als +Ġsched uled +Ġfund amental +Ġprote in +Ġnewsp aper +Ġcl ients +yth on +ĠD ate +h us +Ġfeed back +Ġstret ch +Ġc ock +Ġhot el +ĠQue en +Ġsu gar +Ġj u +Ġmil k +Ġappro val +ĠL ive +Ġequival ent +ef ully +Ġins ert +z ona +Ġext ension +d ri +J ohn +Ġacc omp +S m +ĠF und +Ġconst antly +Ġ` ` +Ġgener ated +ĠA ction +ĠP sych +ĠT ri +Ġrecogn ize +Ġv ary +ph a +ĠR a +d f +et ch +ĠSov iet +Tw o +Ġpattern s +Ġprof ession +an ing +T ime +ĠL im +Ġcol ors +ĠA z +ĠT R +Ġinf ect +Ġphen omen +Ġshe ll +Al so +Ġput s +Ġdel ivery +Ġbro wn +Ġprocess ing +Ġlight s +ess age +ĠBro ok +ĠA ud +l ation +Ġindust rial +L ike +ĠB razil +rou s +ES S +ĠL uc +Ġsome how +Ġ8 5 +Ġpro port +Ġpolit icians +Ġindic ate +Ġh ole +Ġtechn iques +Ġcompet itive +Ġph r +Ġv o +ist ent +ĠD ream +Ġcamp us +Ġaspect s +Ġhelp ful +Ġsh ield +or se +Ġtrig ger +m al +Ġ5 8 +Ġt ort +Ġperson ally +Ġt ag +Ġkeep s +ĠV ideo +Ġben ch +Ġg ap +a ire +Ġe ast +Ġrec overy +per ial +Ġprof it +ĠM ic +Ġ5 7 +Ġcol on +Ġstrong ly +st yle +Ġalleg ations +h an +Ġrep orters +j o +r ine +arg et +and al +Ġ0 3 +Ġfl ash +tr ans +Ġstr ict +Ġpark ing +ĠPak istan +Ġl i +Ġwe ird +ĠE ric +Ġreg ions +ĠJ un +Ġint ellect +ĠW H +od ing +rib utes +up id +ĠT it +Ġf inger +or ia +Ġe lev +ĠF ield +Ġcon clusion +; ; +Ġfeel ings +Ġext ensive +Ġm ixed +Ġne uro +v y +Ġhar ass +ĠC irc +ou ch +Ġterrit ory +Ġsuccess fully +M ar +Ġing red +Ġoverw hel +Ġl ayer +V iew +Ġall ies +ill ance +ĠTh ree +Ġb unch +Ġnorm ally +Ġnet works +Ġsac r +ĠC IA +b les +Ġch ose +Ġopp onents +Ġregard less +Ġfr anch +Ġpre f +ĠP o +Ġbr idge +ann a +ĠSil ver +Ġw age +p age +ri or +Ġrad ical +ĠL ittle +Ġman ip +Ġsecret ary +Ġg ang +D R +F A +Ġdec ent +ĠSp irit +Ġun cle +ĠDevelop ment +Ġinvest ors +Ġwall s +Ġpub lish +Ġgener ate +iss ions +c ar +Ġprom ote +Ġcut ting +Ġche st +Ġdrink ing +Ġcollect ed +Ġ7 2 +Ġhop ing +Ġem br +gor ith +Ġwar ned +Ġinstruct ions +O G +ĠD id +ĠAg ency +Ġg ear +Ġcritic ism +ĠF urther +Ġut il +ann y +R ed +Ġcoun sel +ĠAs ian +Ġredu ction +p ool +Ġteach ing +Ġdeep ly +i y +Ġestim ates +Ġcho ices +Ġperman ent +in em +ke l +Ġf asc +p se +f ile +ĠL ow +ĠP erson +Ġt ournament +st al +Ġm el +U ST +ĠR ay +az i +V al +Ġcont ained +ĠH olly +Ġw ake +Ġreve al +Ġprocess es +ĠIS IS +Ġ0 9 +Ġbl ind +Ġste el +ĠB ad +Ġcare fully +app y +ro it +Ġg aming +Ġhous es +ĠC oll +Ġtr uck +er m +Ġsc ored +Ġocc as +ret urn +b ound +v ar +Ġsh arp +Ġaf raid +ĠE X +am ber +c ific +Ġsche me +N C +ĠPol it +Ġdecl ine +Ġ199 8 +Ġpus hing +Ġposs ession +Ġpriv ile +Ġteacher s +Ġy ield +H A +ĠDav is +it led +#### #### +Ġr ig +ĠD aniel +ac on +Ġh ide +ut en +Ġcolle agues +Ġprin ciples +Ġl oud +Ġs in +ĠDem on +Ġst one +Ġ0 2 +Ġt aught +Ġter rible +Ġst uck +ĠPol icy +te en +Ġimplement ation +ĠB BC +ĠAP I +Ġwhe el +all as +Ġch ampions +ol ars +play er +Ġrepeated ly +ĠSt ill +Ġlik es +ast y +es ter +ĠCath olic +R L +Ġb ath +Ġno ise +t itle +Ġn orthern +P art +Ġmag n +Ġf ab +ĠAs h +Ġdis pl +Ġtick et +Ġm urd +Ġalong side +ĠMus ic +Ġr iver +ĠSte el +ĠC L +ĠPl ayer +ĠM ult +ow ing +re p +s ize +Ġt ur +ĠGeorg ia +isc al +ra ction +Ġc able +Ġ5 9 +Ġw ins +Ġup coming +Ġsurv ive +Ġins pired +ĠEduc ation +Ġstat istics +ĠF oot +iam i +Ġy ellow +ĠP age +. - +ĠH as +Ġur ban +Ġa x +es sel +\ " +Ġquarter back +Ġreg ister +ĠLab or +Ġab ilities +ĠF amily +Ġvar iable +ĠPr ice +Ġcont em +Ġth in +ĠE qu +d ata +Ġg otten +Ġconst it +Ġas ks +Ġt ail +Ġexc iting +ĠE ffect +ĠSp anish +Ġencour age +ins on +ĠA h +Ġcommit ment +C S +Ġr ally +Ġ: : +Ġsubs id +Ġsp in +Ġcapt ured +201 8 +Ġinn oc +Ġalleged ly +ĠC ome +Ġart ists +ĠN umber +Ġelect ronic +Ġreg ional +ap es +Ġw ra +Ġmy th +pr ise +ĠM iller +ĠC reat +ĠEp isode +b ell +Ġdirect ed +Ġext ract +Ġs orry +Ġv ice +ag ger +ĠSu pport +Ġ6 6 +ĠI ron +Ġwonder ful +Ġg ra +N et +ion e +E ng +Ġsh ips +ik es +ĠK evin +it ar +Ġactiv ists +tr ue +ĠAri zona +ent h +ĠDes pite +ĠS E +Ġha bit +ern el +Ġin qu +Ġab ortion +Ġv oid +Ġexpl icit +Ġeng aged +Ġang ry +Ġr ating +Ġfr ag +b ro +ick ing +d ev +Ġwor ried +Ġob ser +Ġap artment +ĠG T +Ġest ate +ĠConst itution +em on +ĠS now +Ġcount y +Ġdis ag +ĠStep hen +Ġimm igrants +w ind +ĠN ations +Ġfol ks +O ut +Ġg all +Ġtarget ed +Ġst ead +ĠB on +ĠL ib +Ġinform ed +Ġ12 0 +ch ain +idel ines +or ough +Ġdri ven +Ġregular ly +Ġbas ket +Ġprinc iple +oc ument +Ġst un +ib ilities +ĠRom an +ĠAb out +Ġal ert +Ġdemocr acy +Ġrepresent ed +H S +c ers +p arent +Ar t +p ack +Ġdi plom +re ts +ĠN O +Ġcapt ure +ĠAd v +Ħ ¢ +Ġannounce ment +ĠL ear +Ġh ook +Ġpur s +ĠS uch +ĠC amer +Ġrefuge es +ĠV e +P ol +Ġrecogn ized +l ib +Ġhad n +A ss +Ġpil ot +us hing +Ġreturn ing +Ġtra il +ĠSt one +Ġrout ine +Ġcour ts +Ġdes per +Ġfriend ly +ĠIt aly +Ġpl ed +Ġbreat h +Ġstud io +N S +Ġimp ressive +ĠAfghan istan +Ġf ing +Ġd ownt +ink ing +ĠR og +i ary +col or +se x +ar on +Ġf ault +ĠN ick +D own +ĠR ose +ĠS outhern +X X +is odes +L ist +6 00 +Ġout come +er r +Ġelse where +Ġret ire +Ġp ounds +ĠGl obal +Pe ople +Ġcommun ications +Ġlo an +Ġrat io +ĠEm pire +Ġg onna +Ġinv ent +D F +Ġ19 70 +ĠComm on +p at +Ġprom ised +Ġd inner +ĠH om +Ġcreat es +Ġoper ate +ver ty +ĠJ ordan +et ime +Ġsust ain +R eg +Ġincred ible +im a +Ġwar rant +Ġm m +A tt +Ġlaw suit +Ġreview s +it ure +ĠS ource +l ights +ĠF ord +Ġ6 3 +g roup +st ore +Ġfeat ured +Ġfore ver +Ġpo verty +ĠP op +ĠC NN +az z +ab is +ach ing +Ġl aid +ĠSu pp +Ġfil ter +en a +ĠCommun ity +Ġcreat ures +u ction +ĠR oyal +Ġassoci ation +ĠCon nect +ĠBr ad +âĸ Ī +l ers +the re +ĠG i +Ġval uable +AC K +ĠT aylor +Ġl iquid +ĠAtt orney +ĠCar l +ĠF inal +ag a +ĠWil son +B ecause +ĠProf essor +ak a +Ġincred ibly +r ance +! ) +R ef +s k +Ġsol utions +Ġatmosp here +Ġbl ame +um es +ĠN ob +C A +um ps +r ical +ĠPut in +ĠD est +or ic +ĠP A +Ġrespect ively +w an +Ġfif th +â Ħ¢ +ĠC ry +Ġgovern or +res ident +Ġpurch ased +Ġh ack +Ġint ense +ob s +Ġorig in +Ġdef ine +Ġcare ful +** * +Ġshould er +Cl ick +Ġt ied +Ġdest ruction +ou red +Ġno body +Ġh o +ĠEx per +Ġt ip +" ; +Ġtechn ique +Ġj ur +ĠP ok +b ow +Ġleg end +Ġacc ord +Ġbus y +ĠInt el +Ġh ang +ak i +. ] +âĢĶâĢĶ âĢĶâĢĶ +Ġsur gery +Ġrep rodu +Ġun iform +Ġscen es +c ode +Ġ6 2 +l isher +ĠH ave +ph ia +Ġcry pt +Ġrec on +Ġsc ream +Ġadop ted +Ġsc ores +N e +ĠIt alian +in cluding +B O +Ġindic ated +Ġent ertain +G u +T ext +i el +Ġtw enty +Ġeng age +off s +ĠPac ific +Ġsm ile +Ġperson nel +Ġto ler +Ġdo ors +Ġt one +Ġmach ines +Ġent ering +ten ance +C O +ĠJer sey +Ġfore st +Ġhor se +Ġcompl aint +ĠSpr ing +y o +ĠPl us +ed ing +ĠRet urn +qu arters +ial s +c ow +Ġacad emic +Ġf ruit +Ġ199 6 +og ether +Ġw ine +Ġpur su +ĠSte ven +Ġlic ens +Wh o +Ġclot hes +re ction +Ġsqu ad +Ġst able +Ġr aw +z ens +St ar +ut ies +anc er +Ġke ys +ĠM u +Ġcompl icated +ig er +ĠTe xt +Ġabs or +Ġ6 8 +Ġfun ny +Ġrel ief +ĠL ew +ĠC ook +Ġch art +Ġdraw ing +G E +Ġmod ule +ĠB ull +I LL +Ġs alt +0000 0000 +il le +Ġres ource +aw ay +adel phia +ĠB ru +Ġ6 7 +Ġsome body +Ġparticip ate +Ġro se +we red +Ġmus cle +Ġcons ent +Ġcontin uing +ĠGuard ian +ĠOr der +reg on +Ġre ar +Ġprov ision +Ġlik ed +ri ent +Ġb ra +Tr ans +Ġmeet ings +Ġto x +Ġcon vent +Ġaut o +Ġrec ording +ĠSo ft +00 1 +ĠR oll +Ġprogram ming +Ġp ic +Ġprov ed +Ġst ab +ĠA st +Ġca ption +ul ating +ĠAtt ack +Ġnew ly +Ġ199 7 +f r +Ġdis cipl +ĠGree k +Ġed ition +ĠDo es +ĠB ox +if le +ack et +Ġpass es +Ġgu est +Ġac celer +it als +U D +Ġaut hent +ĠR est +ov al +t a +u ine +Ġarm or +ĠT own +Ġcomp at +Ġinc hes +Des pite +Ġass ign +he rent +Ġprep are +ĠM eg +oc key +Ġdep ends +Ġtrack s +w atch +Ġl ists +ĠN orthern +Ġal ter +re c +ĠE astern +Ġcond em +Ġevery where +? ' +Ġaff ili +Ġf ought +": {" +Ġm ac +it arian +Ġsc ope +ĠA L +aw s +ar ms +Ġqu e +Ġenjoy ed +nes ota +Ġagg ressive +ĠSt ory +ĠI V +Ġrec ipe +Ġrare ly +ĠMed ical +val ue +ang el +ay ing +omet hing +Ġsub section +Ġs outhern +Ġfrequ ency +re te +roll ed +ult s +ĠN ic +Ġbeh alf +Ġsequ ence +ab et +Ġcontrovers ial +Ġcomp rom +Ġwork er +Ġmain ly +Ġal gorith +ĠM ajor +or ce +g ender +Ġorgan ized +Ġf ake +Ġconclud ed +ĠE D +ĠEx ec +r age +Ġch ances +ber ry +ĠTr ad +Ġconfig uration +Ġwithd raw +Ġf ro +ud es +ĠBro ther +ĠB rian +Ġtri es +Ġsam ples +Ġb id +ĠGold en +Ġphot ograph +if est +ĠD O +ĠPar liament +******** ******** +R em +Ġcont est +Ġsign ing +p x +ĠZ eal +âĶĢ âĶĢ +E ar +Ġex it +Be fore +ĠCor por +n ull +mon th +Ġrac ial +ott ed +ĠV eg +ĠRe uters +Ġsw ord +ps on +ĠRom ney +a ed +Ġt rib +Ġin ner +Ġprot ocol +ĠB i +ĠM iami +ever al +p ress +Ġsh ipping +ĠAm endment +ĠHow ard +con nect +ĠD isc +ĠJ ac +iam ond +ĠThere fore +s es +ĠPrin cess +ĠUS B +ĠAn th +Ġsurve illance +Ġap olog +Ġ6 1 +ow a +Ġf ulf +j s +Ġl uck +ust ed +Ġ § +n i +Ġant icip +em an +Ġwin ner +Ġsil ver +ll a +ic ity +Ġunus ual +Ġcr ack +Ġt ies +e z +Ġpract ical +Ġprov ince +ĠPl ace +Ġprior ity +IC E +Ġdescrib es +Ġbr anch +F orm +ask a +miss ions +b i +Ġp orn +ĠTur k +Ġent hus +Ġf ighters +Ġ0 8 +ĠDet roit +Ġfound ation +av id +A re +Ġjud gment +cl ing +Ġsol ve +ĠDes ign +W here +hes is +ĠT ro +a fter +Ġne utral +ĠPalestin ian +ĠHolly wood +Ġadv is +ĠN on +y es +ol is +Ġrep utation +Ġsm ell +Ġb read +ĠB ul +ĠBe ach +Ġclaim ing +Ġgen etic +Ġtechn ologies +Ġupgr ade +row s +Ġdevelop er +ĠJ osh +ĠDis ney +erv ed +ip al +Ġun ex +Ġbare ly +t hen +ĠP ub +Ġill ness +et ary +ĠB al +Ġp atch +Ġbut t +Ġst upid +ĠD og +ĠD allas +f ront +ie ce +Ġprot ests +Ġch at +oen ix +Ġw ing +Ġpar liament +Ġ7 7 +ose xual +Ġre nder +pt ions +ĠCo ast +os a +ĠG reg +h op +ĠMan agement +Ġbit coin +Ġrec over +Ġincor por +or ne +ĠUs ing +Ġpre ced +Ġthreat ened +Ġspirit ual +ĠE vent +ĠF red +Ġadvert ising +Ġimprove ments +ĠC ustom +Ġer rors +Ġsens itive +ĠN avy +Ġcre am +L ook +Ġex clusive +Ġcomp rehens +Ġde leg +Ġcon ce +Ġrem em +Ġstruct ures +Ġst ored +N D +Ġ1 000 +U P +ĠB udd +A F +w oman +ĠAcad emy +ð Ł +se a +Ġtem porary +Ab out +es ters +Ġtick ets +Ġposs ess +in ch +o z +Ġl a +Ġcontract s +Ġun p +Ġc ig +ĠK at +ult ural +as m +Ġmount ain +ĠCapt ain +St ep +m aking +ĠSp ain +Ġequ ally +Ġl ands +at ers +Ġreject ed +er a +im m +ri x +C D +Ġtrans action +g ener +less ly +Ġ| | +Ġc os +ĠHen ry +Ġprov isions +Ġg ained +Ġdirect ory +Ġra ising +ĠS ep +ol en +ond er +Ġcon sole +in st +Ġb om +Ġunc ertain +1 50 +ock ing +Ġmeas ured +Ġpl ain +Ġse ats +Ġd ict +S L +af e +Ġest imate +iz on +at hered +Ġcontribut ed +Ġep isodes +omm od +G r +AN T +Ġ6 9 +G ener +Ġ2 50 +vious ly +rog en +Ġterror ism +Ġmove ments +ent le +oun ce +ĠS oul +Ġpre v +ĠT able +act s +ri ors +t ab +Ġsuff er +Ġn erv +Ġmain stream +ĠW olf +Ġfranch ise +b at +Ġdem ands +Ġag enda +Ġdo zen +Ġclin ical +iz ard +ĠO p +t d +Ġvis ited +ĠPer haps +Ġact or +Ġde lic +Ġcont ribute +Ġin ject +ĠE s +ac co +Ġlist ening +Ġcon gress +epend ent +Ġprem ium +Ġ7 6 +ĠIr ish +Ġass igned +ĠPh ys +Ġworld wide +Ġnarr ative +ot ype +m ont +b ase +ĠB owl +ĠAdminist ration +Ġrel ation +ĠE V +C P +Ġco vers +Ġ7 8 +Ġcert ific +Ġgr ass +Ġ0 4 +pir acy +ir a +Ġengine ering +ĠM ars +Ġun employ +ĠFore ign +st ract +Ġv en +Ġst eal +Ġrepl ied +Ġult imate +Ġtit les +d ated +Ġj oy +a us +Ġhy per +ak u +Ġoffic ially +ĠPro duct +Ġdifficult y +per or +Ġresult ed +rib ed +l ink +wh o +~~ ~~ +ĠSpe ed +ĠV iet +W ind +ĠBar ack +Ġrestrict ions +ĠSh are +Ġ199 5 +ition ally +Ġbeaut y +op t +Ġm aps +ĠC R +ĠN ation +ĠCru z +W ill +Ġelectric ity +Ġor g +Ġb urd +Ġviol ation +Ġus age +Ġper mit +ĠCh ron +ĠF ant +Ġn aturally +Ġ0 7 +Ġth rown +ĠAw oken +Ġal ien +ĠHer o +ĠK ent +ĠR ick +ri ke +Ġp ace +}, {" +G L +Ġpo ison +ĠT ower +Ġform al +al ysis +Ġgen uine +Ġk il +a ver +Ġproced ure +ĠPro p +intend o +ĠM ain +as ant +Ġtr ained +G ame +ĠL oad +ĠM A +Ġcru cial +Ġle ts +ĠF R +Ġch ampion +1 01 +ĠCon ference +Ġwrit ers +Ġconnect ions +Ġo kay +ir ms +ĠR and +Ġenc ounter +ĠB uff +Ġachie ved +Ġche cks +isc ons +Ġassist ant +Ġwhen ever +ĠA ccess +ĠU r +b in +Ġcl ock +is p +op her +Ġb orrow +Ġm ad +Ġperson ality +on ly +IS T +ab ama +Ġg ains +Ġcommon ly +Ġter r +Ġhyp ot +Ġre ly +Ġt iss +iscons in +Ġrid ic +f unction +ĠO regon +Ġun com +r ating +el and +ĠN C +Ġm oon +ann on +Ġvulner able +ut ive +³³ ³³ +ĠRad io +Ġw estern +se ct +ĠT ony +Ġocc urs +ĠO s +ĠH on +Ã Ń +Ġv essel +ĠScot land +Ġdiscrim ination +Ġsubsequ ent +st ring +Ġfant asy +ĠSh adow +Ġtest im +W E +it i +r as +Ġbo at +Ġmar ks +Ġord inary +Ġre n +Ġrepresent ative +Ġpet ition +Ġ7 3 +Ġad venture +Ġign ore +ĠPhil adelphia +ĠS av +V P +Ġfact ory +Ġt asks +Ġdep ression +z ed +................ ................ +ĠSt orm +Ġc ogn +Ġelig ible +Ġredu cing +v ia +Ġ0 5 +Ġstri king +Ġdoll ar +h o +O V +Ġinstr ument +Ġphilosoph y +ĠMo ore +ĠA venue +Ġrul ed +ĠFr ont +IN E +ĠM ah +Ġscen ario +ĠNAS A +Ġen orm +Ġdeb ut +Ġte a +T oday +Ġabs ence +S im +Ġh am +le ep +Ġt ables +ĠHe art +M I +K e +re qu +V D +m ap +Ġchair man +Ġp ump +Ġrapid ly +v i +Ġsubstant ial +E P +d es +ch ant +ili pp +ĠS anta +ri ers +anche ster +L oad +ĠC ase +Ġsa ving +Ġ7 4 +ĠA FP +er ning +oun ced +ĠMin nesota +ĠW as +Ġrec ru +Ġassess ment +ĠB ron +U E +Ġdynam ic +Ġf urn +ul ator +Ġprop ag +h igh +Ġacc ommod +Ġst ack +ĠS us +w rit +Ġre ven +ĠGod d +ĠZeal and +ab s +Ġbr ut +Ġper pet +h ot +Ġhard ly +ĠB urn +ãĤ ¹ +Ġst y +Ġtrans actions +Ġg ate +Ġsc reens +Ġsub mitted +Ġ1 01 +Ġlangu ages +ugh t +em en +Ġfall s +Ġc oc +Ĥ ¬ +Ġstri kes +p a +Ġdel iber +ĠI M +Ġrel ax +ann els +ĠSen ator +Ġext rem +Ġ} , +ĠDe b +Ġbe ll +Ġdis order +c ut +Ġi OS +Ġl ocked +Ġem issions +Ġshort ly +" ] +ĠJud ge +ĠS ometimes +Ġr ival +Ġd ust +Ġreach ing +F ile +¯¯ ¯¯ +ino is +ĠJ ason +Ġs atell +are t +Ġst ations +Ġag ric +ĠTechn ology +com es +ĠUn fortunately +ĠChild ren +Ġappl ies +ast ed +Ġan ger +ail ability +ĠDam age +Ġcomp are +ĠStand ard +Ġaim ed +ĠB a +angu age +Ġreg ulation +Ġj ury +Ġair port +Ġse ctions +ĠPr ince +em ed +Ġmedic ine +Ġh itting +Ġsp ark +ol ves +Ġad s +St ate +Ġfood s +Ġrepl acement +Ġch icken +Ġlow est +Ġmind s +Ġinvol ves +u i +Ġarr ang +Ġproced ures +ĠWh ich +ivers ary +Ġb ills +Ġimprove ment +Ġin ev +Ġexpect ations +Ġintellect ual +Ġsp aces +Ġmechan ism +2 50 +bre ak +ĠZ e +ĠT enn +ĠB alt +Ġbar rel +Ġstat ic +man n +Pol ice +Ġt ips +Ġhand ling +c us +od ed +il ton +ir y +Ġjournal ists +our se +Ġcom ic +Ġnom ine +IT Y +Ġvers us +Ġlo op +Ġsur f +ĠInd ust +ĠHun ter +Ġbelief s +is an +Ġset up +Ġbre w +im age +Ġcomput ers +f ol +} ," +ĠMed al +Ġtax p +Ġdisplay ed +Ġg rav +Ġf iscal +M on +ĠMos cow +ĠK ong +ĠCent re +Ġcamer as +ĠMr s +ĠH ay +Ġa ver +ĠK elly +p y +Ġrequire ment +Ġent itled +omb ie +Ġsh adow +ag ic +ĠA k +Ġel ite +Ġdiv ided +Ġhead ing +Ġcop ies +Ġloss es +Ġv it +k ed +ĠB ry +Ġan s +ĠSte am +Ġrep orter +he im +ĠIt em +Ġsuper ior +d on +ere nt +à ¶ +Ġtherap y +Ġpe ak +ĠMod el +Ġl ying +Ġg am +z er +r itten +Ġrespons es +Ġconsider ation +ĠB ible +Ġl oyal +Ġinst ant +Ġp m +ĠFore st +à ¼ +Ġext end +Ġconv icted +Ġfound er +Ġconv in +ĠO ak +che ck +Ġsch olars +p ed +Ġover se +T op +c ount +ĠAr k + · +Ġ0 6 +ĠL A +m d +ĠLat in +im ental +ĠC PU +Ġsubst ance +Ġminor ity +Ġmanufact uring +E r +ocol ate +Ġatt ended +ĠMan ager +r ations +Ġappreci ate +om y +GB T +id ency +B L +Ġguarant ee +pos ition +Ġo cean +clud e +Ġhead ed +Ġt ape +Ġlo ose +Ġlog ic +Ġpro ven +Ġsp ir +Ġad mit +is a +Ġinvestig ate +Ġ199 4 +sy lv +ĠL ost +c est +Ġ7 1 +Ġrequest ed +Ġwind ows +ĠPok é +ĠWith out +M et +Ġbehavi our +Ġread er +Ġh ung +ĠKe ep +Ġro les +Ġimplement ed +Ġbl ank +Ġserv es +ĠJ ay +Ġc ited +ĠF riend +prof it +ap on +Ġrep air +it em +arr ass +Ġcrit ics +ad i +ĠF ather +Ġsh out +Ġf ool +Ġ8 8 +Ġprodu cing +Ġl ib +Ġround s +Ġcirc le +Ġpre par +Ġsub mit +Ġn ic +mor row +ãĥ « +U nder +Ġv ital +ater n +Ġpass word +Ġpublic ation +Ġprom inent +Ġspeak s +Ġb ars +Ġde eper +ĠM ill +port ed +Ġw id +Ġbut ter +Ġsm oking +Ġindic ates +K ey +rop ri +ĠF ile +all ing +ast ing +ĠR us +Ġad j +Ġ7 9 +av al +Ġpres um +bur gh +on ic +Ġf ur +Ġpoll s +ik a +Ġsecond ary +Ġmon ster +ig s +ĠCur rent +E vent +Ġowners hip +end ar +Ġarri ve +ĠT ax +Ġn ull +ĠPri v +Ġth ro +Ġk iss +c at +Ġup set +ang le +it ches +ect or +olog ists +ĠGal axy +Ġcor ruption +Ġh int +ent er +ĠH ospital +Ġgreat ly +Ġbeg un +es y +Ġso il +ĠAnt on +Ġmain tenance +ãĥ © +Ġdo zens +Ġhuman ity +ĠAl abama +Ġr om +w orth +ap ing +sylv ania +l ah +Ġg athered +G A +Ġattack ing +f ound +ĠSqu are +Ġar bit +ict ions +ĠW isconsin +Ġd ance +ĠS aint +arch y +Ġbase ball +Ġcontribut ions +Ġliter ature +Ġex ha +per ty +t est +Ġb ab +Ġcontain er +let ter +Ġfall en +Ġwebs ites +Ġbott le +ĠS ac +Ġbre ast +ĠP L +Ġveter an +Ġinterview s +ĠA le +Ġb anned +eng ers +ĠRev olution +in th +Ġconc erning +IV E +Ġexp enses +ĠMatt hew +ĠColumb ia +d s +ist ance +Ġent ity +.. ." +Ġrel iable +Ġpar alle +ĠChrist ians +Ġopin ions +Ġin du +l ow +Ġcompet e +Ġth orough +Ġemploy ed +Ġestablish ment +ig en +ĠC ro +Ġlawy ers +ĠSt ation +T E +ĠL ind +ĠP ur +it ary +Ġeffic iency +âĢ IJ +ĠL y +Ġm ask +Ġdis aster +Ġag es +ER E +es is +ĠH old +Ġcas ual +b led +Ġen abled +ĠEn vironment +ĠInt elligence +i per +ĠM ap +ĠB E +Ġemer ged +is dom +Ġc abin +Ġregist ration +Ġfing ers +Ġro ster +Ġfram ework +ĠDo ctor +et ts +Ġtransport ation +Ġaware ness +H er +Ġattempt ing +O ff +ĠSt ore +ÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤ +ĠK now +Ġdef ence +Ġsc an +ĠT en +ĠCh air +ĠP H +ĠAtl anta +Ġfuck ing +Ġans wered +b n +ĠK ar +Ġcateg ories +Ġr ational +Ġc ust +Ġrob ot +Ġcorrect ly +Ġg if +Ġgraph ics +m ic +Ġground s +ĠO pp +i ate +Ġdist ributed +Ġsan ctions +Ġchalleng ing +ut o +Ġingred ients +Ġinv ited +Ġfound ed +ĠRe qu +d ed +Ġb owl +Ġbrother s +ĠH a +I O +Ġw ages +im ore +oc ial +Ġse ed +ative ly +Ġaddress es +ĠI owa +ab eth +Ġatt itude +is d +ch ild +Ġm ole +Ġdisco very +y ard +B r +Ġ8 2 +Ġsuppl ies +ell ing +Ġdist ingu +C R +Ġre cept +Ġ vert +Ġsw im +b ec +d oor +ĠY eah +Ġg al +Ġinter act +ĠE SP +ĠC S +amp s +Ġconvin ced +Ġobject ive +Ġdis h +ĠPhot os +l ad +Ġdownt own +o il +in ction +Ġto morrow +ĠC OM +Ġsurv ival +sh ot +Ġsett lement +C ons +ĠX box +int erest +ĠS M +arg o +en ess +Ġeth nic +b ered +M in +ĠT ok +Ġinc ent +ĠComm and +Ġmain tained +Ġbreak s +br idge +at ar +ag g +ĠF inally +un icip +ĠO nt +le ft +Ġrecogn ition +Ġ* / +ĠP ers +Ġwe lf +Ġaddress ed +ĠK ansas +Ġvir us +Ġwhere as +Ġp apers +ram s +ĠMin istry +Ġple asure +Ġacqu ired +Ġd uration +j pg +Ġcal m +ĠN HL +Ġburn ing +Ġfold er +ick ed +ĠP y +ĠIll inois +Cl ass +ĠGodd ess +Ġperform ing +Ġwelf are +j ar +In ter +Ġl in +Ġenh ance +Ġnot ion +f are +yp es +ĠAre a +Ġcann abis +ĠDie go +f s +ĠM anchester +com m +in ite +Ġcover ing +ĠS ound +Ġ19 60 +Ġ8 4 +e lect +z ing +Ġcitiz en +Ġph ones +Ġr aid +Ġign ored +ĠOb ject +Ġu pload +c ard +Ġmod ified +Ġroom s +ia h +r ange +he ast +ach us +Ġsuggest ing +âĢ ĭ +gr ade +E l +Ġclot hing +Ġr h +ĠH an +un ity +en cing +ĠAust in +sec ution +t ra +d em +ĠQ ual +Ġhe aven +Ġst ages +Ġw edd +pl us +ific ial +ĠIm m +ĠH o +iet ies +Ġphr ase +Ġbr ill +act ory +Ġprov iders +Ġsil ence +Ġa er +ĠA I +ĠAd venture +Ġplatform s +Ġdemonstr ated +Ġinter f +ing ton +Ġr aces +Ġgr ade +ult ane +ĠTh rough +f alse +Ġb ow +ĠA B +Ġfl avor +Ġhistor ic +g ov +Ġcol our +Ġview ed +ĠEm ail +el come +Ġinter vention +Ġd iversity +Ġperiod s +Ġre verse +ĠV ery +Ġqu ote +ĠLe ft +th rough +Ġsc rew +Ġland ing +Ġp ill +Ġw et +Ġprot esters +Ġrepe at +av ed +er k +Ġsal ary +ĠPenn sylvania +St ill +Ġmay or +Ġkit chen +Ġfeat uring +ĠM useum +ĠT ournament +ĠF al +Ġser vers +U C +Ġany body +im g +ĠTr ade +ixt ure +the less +Ġfin ance +Ġcl osing +ĠPat ri +i ac +ab el +Ġ> > +or ous +Ġf irms +sc reen +un a +Ġemb arrass +ul se +Ġlet ting +Ġth rew +ile y +Ġch annels +l an +ĠVeg as +Ġse ar +Ġfant astic +ar re +uzz le +ĠD er +Th ose +Ġsw ing +Ġshe et +ind ex +co ver +og an +Ġvari ables +ĠTe ch +Ġsp oken +ac hel +ĠD a +ĠMount ain +Ġload ed +Ġfoot age +vers ion +Ġun l +ĠPh oenix +Ġthrow ing +Ġf iring +Ġtrack ing +Ġw idth +Ġstrugg ling +ro oms +ot ion +Ġmonth ly +ĠSer ver +Ġegg s +op en +M C +Ġ199 3 +Ġh ired +Ġstay ed +ĠAll en +Ġst ro +Ġ9 8 +st ep +ĠTurk ish +Ġfab ric +ist ing +ĠD om +Ġd ates +Ġpr on +Ġbasket ball +Ġl ucky +ĠArab ia +Ġassum ed +est y +Ġaff airs +Ġgl ad +ĠInd eed +ĠF A +ĠW ord +Ġjo ining +if ice +p read +ir ts +ĠSe lect +Ġpop ulations +aw are +Ġn ose +Ġcompl aints +st art +Ġsc oring +Th anks +Ġmin ing +Ġvisit ors +S H +Ġdam aged +Ġcharacter istics +ĠP ent +D C +Ġ8 3 +ĠS ix +r ates +Ġfl ags +ĠB rew +d og +M ark +// // +Ġexec ution +Ġj oke +ph ones +Ġtestim ony +Ġob st +Q L +ĠC ut +Ġstud ied +ĠN intendo +ick et +ĠN BC +Ġl ad +ĠB ra +ĠM oh +Ġk ernel +Ġoverwhel ming +Ġag ed +Ġapplic able +ĠC ond +Ġroad s +ĠBl ock +m ade +od ge +Ġcomm ands +Ġoff ices +vel and +Ġt ut +Ġrece iver +ĠF ro +Ġsho pping +Ġi P +ĠSt re +ĠA BC +Ġentertain ment +ĠB ow +ort ed +M c +Ġread s +gr ad +ĠCol lect +Ġâ ĪĴ +ĠCap ital +eder ation +Ġemploy er +Ġinvolve ment +Ġanx iety +al ia +Ġro of +ĠAm ong +ĠDemocr at +Ġstat s +ĠV ill +Ġconst itutional +Ġrefer ring +itt y +Ġtack le +out ube +Ġback ed +ĠH ong +ĠBro ad +Ġe le +ĠO tt +Ġ199 2 +h our +achus etts +C al +Ġdefe ated +Ġ8 1 +es p +Ġseem ingly +w as +ĠJ enn +ĠK urd +Ġg ene +Ġdisc ount +R et +EC T +( ); +Ġclub s +Ġs id +ĠM arsh +Che ck +Ġp p +ĠE ag +ides pread +Ġbe ings +F T +Ġintrodu ction +ĠCh ange +AR D +Ġ1 10 +ad ows +ier ce +Ġme al +a uthor +ĠB ang +lah oma +Ġr anks +201 1 +?? ?? +m ax +Ġcoll apse +Ġop ens +Ġe cho +Ġs oph +Ġrac ist +Ġenorm ous +Ġw aves +Ġt ap +Ġcomprehens ive +. -- +ĠR oy +Ġfarm ers +Rel ated +a ired +ron es +ĠC rim +Ġproport ion +Ġdesign s +Ġnegoti ations +Ġvirt ually +ĠBat man +Ġwar n +Ġlegit imate +m ate +Ġcon vention +, , +net ic +ĠS D +Ġconsist ently +Ġcompens ation +Ġpunish ment +Ġy e +Ġt ie +ĠB ureau +ir lf +ĠB u +ĠA ren +ĠPh ilipp +Ġkn ife +Ġmem ories +ĠR oss +Ġang le +Ġ8 6 +ĠTh under +Ġre nd +ĠT our +Ġcount s +s ung +ĠIm p +Ġeduc ational +Ġaccess ible +C OM +Ġd rew +y er +G l +am ine +OR T +O B +I B +m aster +Ġtri als +og y +h ar +ĠTr ust +Ġprefer red +irlf riend +ĠN ev +Ġb in +Ġc ow +P age +Ġsign ature +ĠB L +7 00 +Ġret ired +Ġby tes +Ġneigh b +ĠLeg end +Ġdev ast +Ġsuspect ed +is ons +ĠPoké mon +sc ale +Ġcap abilities +Ġre vel +Ġche ese +d y +igr ant +Ġfail ing +b its +ĠHer oes +ĠG host +ĠS cient +Ġappoint ed +ur i +Ġinst itution +Ġexpand ed +g reg +Ġmonitor ing +Ġp odcast +Ġcoal ition +Ġ9 6 +J o +Ġst olen +ĠS ab +Ġstop s +Ġhol iday +Ġint r +C ar +Bl ack +ĠL GBT +Ġwar ming +ĠAnd erson +Ġ8 9 +Ġprodu cer +M ed +Ġaccur acy +ĠMar vel +iz abeth +ĠPat rick +m ony +Ġmin i +ac les +Ġover t +the y +Ġmembers hip +ĠV en +Ġex ch +Ġrem oval +ĠD ave +T Y +m ad +ĠF ind +Ġad equ +Ġe c +Ġte eth +Ġemot ion +Ġper m +Ġsole ly +d b +Ġextra ord +IG HT +c al +Ġgu idelines +Ġd ying +Ġsusp ended +ĠPrem ier +ĠAnth ony +el ve +Ġd ad +ĠE th +ĠFoot ball +Ġabandon ed +Ġ< < +Ġm arch +Ġhor ror +âĢ¦ " +Ġchild hood +Ġcampaign s +Ġl unch +ĠAl bert +bl ock +âĸĪ âĸĪ +ound ing +Ġb one +or gan +ad ers +ĠFl ash +ĠDri ve +Ġton ight +Ġw ars +ĠF L +Ġform ation +con st +New s +Ġcom pe +or ious +ĠSt aff +Ġdiscuss ions +ĠProt ection +ĠJ am +Ġcrit eria +Ġinstall ation +Ġaccompl ish +iz za +Ġpub lisher +Ġresc ue +ĠT ry +U LL +ĠS om +ĠH op +ore t +th s +ord on +Ġp ocket +ĠIn v +Down load +ĠCr ime +Ġb ene +ĠGu ide +ĠAs sembly +Ġparam eters +I E +ĠAlex ander +Ġconc ert +ĠSc he +Ġsh oes +Ġvis iting +Ġrec all +Ġb ub +Ġr ural +Ġconc rete +ĠR os +N ext +R uss +Ġlo ans +ĠSh ield +Ġtre m +hem at +k g +ĠHar ris +is ition +ĠM ove +ĠF C +Ġf ate +ĠCh o +Ġt ired +Ġprinc ipal +h ist +ien ces +ath y +Ġse vent +Ġm ood +Ġstrateg ic +Ġdise ases +Ġfor um +Ġtem por +Ġhead quarters +P ar +ig e +fl ix +Ġgu itar +Ġ9 4 +On ly +Ġrele ases +ro ph +================ ================ +Ġ6 00 +ĠContin ue +ig ate +ĠC rit +sy stem +Ġdis abled +Ġunex pected +ith ub +Ġuncle ar +ĠE st +Ġcontr ad +Ġstrateg ies +vent ures +Ġpass age +AM E +Ġimpro ving +Ġreve als +Ġdecre ase +ov a +Ġann oy +ĠSh ort +ĠL ibrary +Ġcy ber +n ell +ĠH ur +ĠC B +Ġphot ograp +U I +Ġs ed +G e +Ġ8 7 +Ġd iverse +Ġencour aged +Ġcons piracy +Ġbird s +Ġoper ator +Ġhand ful +Ġclass ified +? ) +Ġdram atic +Ġinvestig ators +it o +Ġw idespread +ĠR oom +-------------------------------- -------------------------------- +Ġcollect ive +Ġjournal ist +St ring +Ġtemper atures +il a +Ġgu id +Ġins pect +Ġmiss ile +ĠMay or +Ġman ual +Ġsim ultane +Ġrat ings +Ġsu ck +Ġ9 7 +Ġunivers al +Ġph arm +Ġdis rupt +ian o +A V +Ġf t +Ġstat ist +old s +ĠWalk er +ph p +Ġunder t +ĠL as +ish op +nt il +res hold +ĠWhe ther +M s +Ġden y +ĠCl oud +Ġprov ider +Ġsurv iv +ĠUp date +h as +Ġmist akes +ch arge +pl ed +r ity +Ġn ode +ĠMass achusetts +ool s +lic ation +Ġf ails +em ale +or i +back s +Ġsh irt +Ġ' ' +ĠN AT +Ġwat ers +els on +Ġe ase +Ġsc ar +Ġcont ents +m ind +Ġcont ribution +Ġsh r +Ġhand ed +Ġst ability +Ġtra ve +E m +Ġmir ror +12 3 +Ġwe igh +Ġf iction +ou ver +ist ant +r ition +ĠF ed +Ġphys ically +Ġst ake +ĠArt icle +ĠAr c +ĠLew is +ĠM ind +Ġdemonstr ate +Ġprof its +v ision +om ic +ol id +Ġbatt les +Ġdri ves +Ġeas tern +ĠS ony +!! ! +ar ation +v ard +ĠG L +port ation +Ġ9 2 +Ġlaw makers +Ġprotect ing +ĠE PA +Ġy eah +Ġsh ame +ol ph +e ven +x it +Ġatt ach +Ġrepresent ing +Ġob s +ĠUt ah +iff s +ĠFre edom +à ³ +A K +Ġinc idents +it age +Ġview ers +c d +Ġm ouse +Ġcl ar +Ġaccord ance +Ġb ot +c or +ĠSum mer +he ld +Ġinnoc ent +Ġiniti ative +ol s +________________ ________________ +Ġsp ots +p ace +Ġconvent ional +Ġcorpor ations +Ġblock ed +H D +at tered +Ġref ers +Ġbu ck +ĠDig ital +12 0 +Ġtop ics +T F +Ä ģ +br id +re ement +Ġunder lying +ĠM ember +Ġinvestig ating +Ġpregn ancy +Ġtouch down +ĠB and +ĠCall er +Ġinst ances +P P +w a +G ood +Ġ199 1 +ĠC old +Ġfear s +Ġrem arks +Ĩ Ĵ +at al +Ġm it +Ġexper iments +i pt +Col or +ind u +Up date +Ġ9 3 +A g +Ġ å +anc ouver +B oth +Ġjud ges +Ob ject +Ġst ere +umb n +Ġparticip ation +ĠSt ars +ĠJ ere +Ġweek ly +ĠB an +Ġconvers ations +ĠP itt +u z +ĠIndian a +ĠK ick +Ġinf ection +Ġhero es +Ġsett led +Ġstri p +Ġh al +Ġd ump +ĠS ci +Ġl es +Ġref erences +ĠU RL +ĠBr idge +Ġwant ing +For ce +Ġex clus +Me anwhile +m n +Ġg entle +m aker +sen al +ĠG ro +ou ri +ĠR ain +ĠAll iance +Ġl ift +el a +S D +ĠCle veland +Ġrank ed +Ġst adium +Ġdead ly +ä ¸ +Ġr iding +ar ia +ĠAr mor +Ġdocument ation +ĠGree ce +ree k +Ġl ens +ĠS a +Ġg ross +ĠE mer +ag ers +ĠD ub +ĠR h +ĠAM D +Ġarri val +Ġdes ert +Ġsupp lement +ĠRes p +Ġkn ee +Ġmarg in +f ont +og g +201 0 +ĠP ir +ĠP rom +iv als +Ġint ake +Ġdifferent ly +ug s +Ġb its +clud ed +Ġsearch ing +ĠD u +um ble +Ġfunction al +ĠBalt imore +ĠC ould +Ġdes ired +Ġcirc uit +ĠL yn +ĠG O +ĠF alse +re pre +' : +alt ies +Ġmin im +Ġdro ve +ĠSh ould +Ġh ip +Ġpro s +Ġut ility +ĠN ature +ĠM ode +P resident +o pp +r at +form ance +Ġconcent ration +Ġf ont +ĠB ud +Ġam id +Ġre vers +ĠM L +B ar +Ġinter action +Ġjur isd +Ġspell s +d ep +f il +Ġcivil ians +ut ter +ĠCo oper +ĠBel ow +Ġent rance +Ġcon vert +Ġcontrovers y +ow ered +Ġcontr ary +Ġar c +ĠExec utive +ĠOffic er +Ġpack ages +Ġprog ressive +w idth +Ġreserv ed +v ol +ĠSam sung +Ġprint ed +Ġcent ers +Ġintrodu ce +ĠKenn edy +Ġodd s +Ġsure ly +Ġindepend ence +Ġpass engers +repre ne +ĠBe h +Ġl oves +ĠESP N +Ġfac ilit +Ġident ical +Ġdo ct +Ġpartners hip +con f +ĠH ide +Ġconf used +ĠC ow +M en +Ġw rest +ĠIraq i +Ġh oles +ĠStud ies +Ġpregn ant +h ard +Ġsign als +I X +Ġpull ing +Ġgrad uate +Ġnomine e +D ate +Ġper mitted +Ġâ Ĥ¬ +ĠOk lahoma +St art +Ġauthor ized +Ġal arm +ĠC os +v an +Ġgener ations +c ular +Ġdr agon +ĠSoft ware +ĠEd ward +Ġcontro ller +S en +ge red +ĠV ik +Ġappro ached +Th ank +Ġcan ce +Ġform ula +ĠSm all +Ġweak ness +Ġr amp +it udes +j ud +Ġbrill iant +Ġacc us +s ource +Ġ8 00 +ĠE vil +S w +Ġhom eless +we ek +i ens +r ics +ĠTh ird +T O +Ġorgan ic +Ġpresent ation +ag h +ĠDown load +v ation +Ġas sembly +or able +hold ers +ĠBern ie +ĠHel p +Ġt ong +ĠF ight +Ġbe ach +B ook +ĠL ic +Ġr ush +ĠR ound +ou p +ĠMar x +Ġcalcul ated +ĠDe vil +ĠSar ah +Ġoccasion ally +Ġbul let +Av ailable +g ate +Ġ9 1 +Ġh osp +Ġprom ises +ĠH IV +ĠSt adium +ĠSt ock +ĠCorpor ation +g age +N G +ĠC redit +Ġs ne +ib l +Ġacc um +s uch +Ġterror ists +Ġconscious ness +ĠZ h +Ġdram a +ool a +pir ation +Ġlab our +ĠN in +Ġut ter +Ġdemocr atic +Ġass ass +il ation +Ġg est +Ġab road +Ġmet ab +Ġs orts +Ġfl av +U B +Ġm g +ĠNot hing +ĠO d +Ġmus ical +200 9 +Ġdro ps +oc ated +ater al +0000 00 +Ġg re +Ġequ ality +Ġburd en +Ġv ig +ĠLe ader +-------- ---- +Ġcere mony +Ġf ighter +Ġact ors +Ġ æ +am an +F i +Ġal ign +put er +Ġe lder +ĠN SA +Ġrepresent ation +ĠOnt ario +IT H +usal em +Ġharass ment +itz er +Ġsy mp +Ġbox es +ĠD R +Ġman ifest +at re +Ġ ^ +Ġd ies +le ton +Ġmiss ions +et he +Ġres olve +Ġfollow ers +Ġas c +Ġk m +l ord +am med +Ġsil ent +ĠAssoci ated +Ġtim ing +Ġprison ers +ĠK ings +ĠF ive +Ġtow er +Ġappro aches +Ġprecise ly +Ġb ureau +ĠM other +ĠI ss +Ġkey board +it ual +Ġfund ed +Ġstay ing +Ġpsych ological +Ġm ile +ĠLe on +ĠBar b +w ill +Ġw ider +ĠAtl antic +Ġt ill +ĠR ome +ro t +Ġaccomp an +Ġfl our +ac o +W orld +ĠExp ress +ĠY u +C or +Ġple ased +part y +Ġpoint ing +Ġinf lation +Ġro y +Ġ ), +ain er +Ġwedd ing +orm on +Ġrequ iring +Ġqual ified +Ġse gment +EN D +Ġs izes +e als +Ġcor rupt +ass ador +Ġcele b +Ġdream s +ĠM ess +Ġcheck ing +ĠV ersion +Ġprep aring +Ġact ively +ĠD iff +Ġl ux +ĠW inter +act eria +ĠN E +Ġdep uty +Ġtrans gender +Ġsum mary +Ġin her +er ies +ch ar +ĠY an +Ġkn ock +ĠP ath +Ġl ip +roll er +Ġimp ression +Ġcelebr ate +Ġsl ide +Ġgu ests +Ġcl ip +F S +Ġsav ings +Ġcapt ain +Ġleg acy +ĠDen ver +Ġw ounded +tab oola +AC T +Ġpurs ue +Ġo xy +Ġ q +Ġsem i +ĠN eed +ĠAff airs +Ġob sc +Ġcheck ed +Ġd ual +C ode +ĠM D +le m +ult y +Ġ © +ĠEl izabeth +Ġcent uries +ard ed +s rc +Ġev ident +enn is +at in +Ġunemploy ment +ĠMar io +Ġint im +Ch rist +Ġbi ological +Ġsold ier +ĠAdd ed +Ġm ath +ĠG il +Ġbi as +Ġd ating +ĠO cean +Ġm ice +M us +h ire +ĠT es +Ser ver +lim ited +S ize +Ġmet ers +Ġrock et +es see +Ġcertific ate +ĠIran ian +AS S +Ġgr id +D ec +Ġro lling +com mun +ĠSwed en +b ury +Ġtiss ue +Ġrac ism +ĠL ocal +Ġmyster y +Ġexam ine +Ġst em +Ġs its +Ġhop ed +ot ing +Ġdial ogue +Ġpers u +W atch +l ay +M AN +Ġch ronic +ĠPort land +mark et +ĠS EC +Ġparalle l +Ġsc andal +Ġcar ries +Ġphenomen on +h uman +ack er +ĠO x +Ġretire ment +tain ment +ov ie +ĠG ear +Ġd uties +Ġdo se +Ġsc roll +M B +in f +Ġsa uce +Ġland scape +red dit +ĠChampions hip +ĠRed dit +al id +Ġco in +Ġover s +Ġpost ing +ab out +Ġf el +and y +Ġb old +Ġfocus ing +e ffect +G R +Ġde emed +Ġrecommend ations +Ġste pped +Ġvot er +ĠDe ep +ĠInst agram +Ġmoder ate +ĠMary land +Ġrestrict ed +ĠM B +ĠCh all +Ġto b +Ġc ir +ĠO cc +ĠE ver +Ġcoll aps +IN FO += - +ĠP ict +ĠAcc ount +n c +Ġo ught +Ġex port +Ġdr unk +( ' +Ġw ise +ĠM ort +ne cess +Ġan cest +ĠInc re +Ġfrequ ent +m ir +Ġinterpret ation +Ġdepend ent +Ġco ins +ĠB ol +V ideo +ĠJust in +Ġfat al +Ġcook ing +Ġconf usion +ip her +Ġcust ody +ĠMor gan +om ach +ĠGovern or +Ġrestaur ants +el ing +Ġacknowled ged +Ġthe r +Ġgen es +ch ing +He y +Ġtact ics +ĠMex ican +Ġv end +Ġhe s +qu er +Ġnot ing +ĠCamer on +Ġtarget ing +ro ck +Ġcred its +Ġemot ions +Ġrepresent atives +new s +Ġlegisl ative +Ġrem oving +Ġtweet ed +ĠCar ter +ĠF ixed +Ġfor cing +Ġspeak er +Ġm ales +ĠViet nam +l ined +Ġconcept s +Ġvo ices +o ir +ĠT rib +W he +ĠJer usalem +ĠS ant +Ġc ul +Ġl ady +ĠHaw ai +Ġar ts +ĠIn n +ĠMach ine +ĠEm peror +Ġsl ot +g ly +ĠPro cess +II I +Ġathlet es +ĠTem ple +ĠRep resent +Ġpres c +Ġt ons +Ġgold en +Ġp unch +ĠG R +iver pool +Ġen act +Ġlob by +Ġm os +Ġpick ing +Ġlif etime +Ġcogn itive +E ach +z o +Ġd ub +Ġcons ists +ol n +Ġf estival +am ous +Ġint ellig +w ords +ĠSm art +Ġde le +Ġl apt +Ġmag ical +ĠS in +b us +ur ities +igh th +ĠRub y +ĠS ure +ol ving +Ġj un +O ST +Ġimp osed +Ġast ron +Ġcor rel +ĠN S +ĠK it +ĠF uture +b urn +Ġimm une +oc us +Ġcour ses +ĠSt ring +Ġle an +Ġg host +Ġout comes +Ġexp ense +Ġevery day +Ġaccept able +A h +Ġequ ipped +Ġor ange +F R +ĠD utch +Th ough +ĠR ank +Q U +ĠRober ts +wh at +re nd +Ġdisapp ear +Ġsp awn +ĠL am +o is +Ġdes erve +Ġmin imal +Ġnerv ous +ĠW ould +Ġro ok +ĠV ancouver +Ġres ign +sh ire +ĠW orks +ĠB uild +Ġafford able +ĠG ary +ĠAren a +Ġh anging +Ġimpl ications +ĠS ong +Ġmain taining +Ġgu ards +C ON +Ġder ived +Ġexecut ed +Ġthe ories +Ġqu oted +ĠAnd re +og a +sel ess +in fo +ĠBel g +Ġt ears +ĠSur v +Ġbirth day +ig ious +im mer +Ġspect rum +Ġarchitect ure +Ġrec ruit +arm a +T able +Ġmon sters +ĠG ov +Ġdest ination +Ġattract ive +Ġf oss +ĠMore over +Ġpres ents +TH E +Ġrep ly +pt on +Ġc um +Ġdel ight +Ġaffect s +Ġdon ations +ĠT oy +ĠH im +M ENT +Ġover come +it ched +ĠFant asy +ĠH at +ĠBe ast +b ott +Ġinvestig ations +R un +Ġhun ting +d i +f und +Ġs essions +est yle +Ġport ray +oid s +Y eah +Ġcommun icate +Ġcom edy +ĠY ang +Ġbel t +ĠMar ine +Ġpredict ed +Pl ay +Ġimportant ly +Ġremark able +Ġelim inate +D avid +Ġb ind +V ID +Ġadvoc ates +ĠG aza +im p +D B +ĠN a +ĠSim ilar +I ES +Ġchar ity +v as +m ath +Ġâ ĸ +ok er +nd um +Ġcap s +ĠH al +2 000 +e an +Ġfle et +Ġrec re +R ight +Ġsleep ing +ij ing +k ind +Ġdesign ated +à ¤ +Ġanim ation +ke e +ĠInt rodu +Ġ/ > +Ġdelay ed +Ġtrem end +Ġcur ious +U se +Ġle ct +d am +Ġinnov ation +ĠPoint s +Ġload ing +Ġdisp ute +ct ic +ird s +ĠB Y +Ġn urs +ĠVal ue +ION S +ĠH um +Ġtem plate +m ers +Ġappear ances +ĠEnter tainment +Ġtransl ation +Ġsa ke +Ġbene ath +Ġin hib +Ġe uro +abet es +Ġstud ying +ĠM as +Ġper ceived +Ġexam ined +Ġe ager +Ġco aches +Ġim per +ch i +Ġprodu ces +" ). +ĠEvery one +Ġm unicip +Ġg irlfriend +Ġh ire +ĠV ice +Ġsu itable +op y +Ġin equ +ĠD uke +f ish +f irst +ĠO bs +Ġinter ior +ĠBru ce +ĠR y +Ġanal ys +Ġconsider able +Ġfore cast +Ġf ert +ors hip +ĠD rug +ĠA LL +: " +th ur +ĠM ail +Ġball ot +Ġinst antly +ĠCh annel +Ġp icks +Ġ198 9 +Ġt ent +ol i +Ġcivil ian +b ling +ell o +b u +Ġin ch +Ġlog o +Ġcooper ation +Ġwal ks +Ġinvest ments +Ġimp rison +ĠF estival +ĠK y +Ġleg ally +Ġg ri +ch arg +S l +Ġthreat ening +du ction +fl ow +Ġdismiss ed +ibr aries +c ap +e le +ĠMc G +ĠHar vard +ĠConserv ative +ĠC BS +p ng +Ġro ots +ĠH aving +umb led +ĠF un +\ / +ĠS earch +ple x +Ġdiscuss ing +Ġcontin u +ĠT ai +ĠW ik +F ree +f it +Ġref use +Ġmanag ing +Ġsy nd +ip edia +w alk +Ġprofession als +Ġguid ance +Ġunivers ities +Ġas semb +unt u +F inally +AS E +ĠAut o +ĠH ad +Ġann iversary +L D +ĠD ur +ĠUlt imate +ih ad +pro duct +Ġtrans it +Ġrest ore +Ġexpl aining +Ġass et +Ġtransfer red +Ġbur st +ap olis +ĠMag azine +ĠC ra +ĠB R +gg ed +ĠH E +M ich +b et +ĠL ady +yl um +erv es +Ġme ets +wh ite +L og +Ġcorrespond ing +Ġins isted +G G +Ġsurround ed +Ġt ens +Ġl ane +Ġco inc +h ome +Ġexist ed +ect ed +ĠDou ble +lam m +Ġske pt +ex p +Ġper ception +ie v +ĠBe ing +o ft +Ġadop t +. : +] ; +Wind ows +Ġsatell ite +AS H +Ġinf ant +d escription +ĠMe anwhile +c m +oc a +ĠT reat +act or +Ġtob acco +ĠN orm +em ption +Ġfl esh +Ġj e +o op +ĠHe aven +Ġbe ating +an im +Ġgather ing +Ġcult iv +G O +ab e +ĠJon athan +ĠSaf ety +Ġbad ly +pro t +Ġcho osing +Ġcontact ed +Ġqu it +Ġdist ur +Ġst ir +Ġto ken +D et +ĠP a +Ġfunction ality +00 3 +s ome +Ġlimit ations +Ġmet h +b uild +con fig +N T +re ll +ble m +ĠM om +Ġveter ans +ĠH u +Ġtrend s +are r +ĠG iven +ĠCa ption +m ay +AS T +Ġwond ering +ĠCl ark +n ormal +Ġsepar ated +Ġdes p +st ic +b rew +Ġrel ating +ĠN ik +ĠF arm +Ġenthus i +g ood +d eb +Ġactiv ist +Ġm art +Ġexplos ion +ĠEconom ic +L ink +Ġins ight +Ġconven ient +Ġcounter part +su pport +ĠV irt +ag en +ĠTenn essee +ĠSim on +ĠA ward +OC K +ĠF igure +Ġoverse as +Ġpr ide +ĠC as +n ote +m g +C urrent +Ġdispl ays +cont ent +Ġtravel ing +Ġhosp itals +ĠFin ancial +ĠP ast +Ġdefend ant +Ġstream ing +m ble +ĠBer lin +uk i +Ġdist ribut +Ġant ib +Ġch ocolate +ĠCast le +Ġinter rupt +ĠR ow +Ġconvers ion +Ġbug s +ĠR ather +li est +L Y +ĠJe an +com mon +ak h +Ġ1 30 +ot ton +ĠDe an +Ġam endment +Ġgame play +ĠWar ren +od a +Ġhigh lights +Ġir re +ĠNAT O +Ġball s +Ġdemand ing +U RE +ĠL uke +F igure +st op +on ia +z one +iz ers +ĠW R +Ġaward ed +Ġregul atory +ĠH art +ĠS N +pl ing +Ġs our +ĠP ixel +us ive +Ġf et +ĠS ent +Ġautom atic +Ġf er +vern ment +ĠKh an +T ON +f ather +Ġextraord inary +th rop +ĠP ython +ĠG PU +Ġsex ually +Ġdesk top +it ivity +ĠAnton io +Ġo rient +Ġe ars +ob by +ous es +vertis ements +Ġmanufacture rs +ic ient +min ute +Ġconv iction +Ġg arden +p ublic +Ġsatisf ied +f old +O K +Ġin hab +ĠTh ink +Ġprogram me +Ġst omach +Ġcoord in +Ġh oly +Ġth reshold +Ġr het +Ġser ial +Ġemploy ers +ĠEvery thing +ra h +Ġb other +Ġbr ands +Val ue +ĠT ed +ĠPlan et +Ġp ink +ĠFurther more +s a +P E +re ck +ĠUS D +ot te +Ġ& & +Ġland ed +g ets +Ġprodu cers +Ġhealth care +Ġdomin ant +Ġdest ro +Ġam ended +ch ron +Ġf its +ĠSy d +ĠAuthor ity +AT CH +Ġfight s +ĠL LC +Ġ-- - +ĠCor p +Ġtox ic +spe cific +ĠC orn +ĠChe l +Ġtele phone +ĠP ant +Ġmyster ious +aun ch +od ox +med ia +Ġwitness es +ag u +Ġquestion ed +ĠBre xit +ĠRem ember +ene z +Ġend orse +iat ric +ĠId ent +Ġridic ulous +1 10 +Ġpr ayer +Ġscient ist +Ġ19 50 +ĠA qu +Ġunder ground +ĠU FC +m are +ĠL ater +w ich +Ġsubsc rib +Ġhost s +Ġer r +Ġgr ants +ant om +Ġsum mon +ear ly +ĠC lear +ĠPr im +Ġsusp ension +Ġguarant eed +app er +Ġr ice +ĠSe an +ĠSh in +Ġrefere ndum +Ġfl ed +r ust +Ġ3 60 +ter y +Ġsh ocked +B R +ĠO il +ĠAll ah +Ġpart ly +Ġign or +Ġtrans mission +Ġhom osexual +ivers al +Ġhop efully +ãĤ ¤ +Ġless on +L eg +Ġ .. +Y et +t able +app ropri +re tt +Ġbo ards +Ġincor rect +Ġb acteria +ar u +am ac +Ġsn ap +.' " +Ġpar ad +t em +he art +Ġav ailability +Ġw isdom +Ġ( + +Ġpri est +ĠÂł ĠÂł +O pen +Ġsp an +Ġparam eter +Ġconv ince +Ġ( %) +r ac +Ġf o +Ġsafe ly +Ġconver ted +ĠOlymp ic +Ġres erve +Ġhe aling +ĠM ine +M ax +Ġin herent +ĠGra ham +Ġinteg rated +D em +Ġpip eline +Ġapp lying +Ġem bed +ĠCharl ie +Ġc ave +200 8 +Ġcons ensus +Ġre wards +P al +ĠHT ML +Ġpopular ity +look ing +ĠSw ord +ĠAr ts +' ) +Ġelect ron +clus ions +Ġinteg rity +Ġexclus ively +Ġgr ace +Ġtort ure +Ġburn ed +tw o +Ġ18 0 +P rodu +Ġent reprene +raph ics +Ġg ym +ric ane +ĠT am +Ġadministr ative +Ġmanufacture r +Ġ vel +ĠN i +Ġisol ated +ĠMedic ine +Ġback up +Ġpromot ing +Ġcommand er +Ġfle e +ĠRus sell +Ġforg otten +ĠMiss ouri +Ġres idence +m ons +Ġrese mb +Ġw and +Ġmeaning ful +P T +Ġb ol +Ġhe lic +Ġwealth y +Ġr ifle +str ong +row ing +pl an +as ury +âĢ¦ . +Ġexpand ing +ĠHam ilton +Ġrece ives +S I +eat ures +ĠAn im +RE E +P ut +Ġbrief ly +ri ve +Ġstim ul +Ġ`` ( +Ġ __ +Ġch ip +Ġha z +Ġpri ze +ĠTh ings +AC E +ul in +d ict +ok u +Ġassoci ate +ock ets +y outube +St ory +ateg ory +Ġm ild +ail ing +ĠY e +O rig +ĠK a +or ig +Ġpropag anda +Ġan onymous +Ġstrugg led +Ġout rage +AT ED +ĠBe ijing +r ary +Ġle ather +Ġworld s +Ġbroad er +12 5 +id al +ĠBet ter +Ġt ear +E xt +Ġpropos als +Ġit er +ĠSqu ad +Ġvol unt +m i +D id +ĠP u +p in +Ġspeak ers +Ġb orders +Ġfig ured += ' +Ġsimultane ously +aed a +Ġcharg ing +Ġur ged +Ġcon j +25 6 +ĠG ordon +mer ce +Ġdocument ary +Sh are +it ol +ON E +ĠG arden +h att +ĠThom pson +ane ous +ap ore +Ġt anks +Ġless ons +tr ack +Ġout standing +Ġvolunte ers +Ġsp ray +Ġmanag ers +l arge +Ġcamp s +Ġart ificial +ĠR u +Ġb ags +th al +Ġcompat ible +ĠBl ade +Ġf ed +Ġarg ues +F I +Ġunf air +Ġcor n +Ġoff set +Ġdirect ions +Ġdisappoint ed +ĠCon vention +Ġview ing +M E +oc ity +Ġtown s +Ġlay ers +Ġro lled +Ġjump ed +Ġatt ribute +Ġun necess +inc oln +Ġsupp ose +ĠNet her +ch a +Ġbur ied +Ġsix th +B en +ress ing +OU R +Ġw ound +Ġcy cl +Ġmechan isms +Ġcongress ional +ĠE lement +Ġagre ements +Ġdec or +Ġclos est +ĠM it +Go ogle +} } +Ġm ixture +Ġflu id +S ign +ĠSch olar +Ġp ist +ask et +ab ling +Ġrac ing +he ro +ri el +ass y +Ġche aper +b en +Ġvert ical +amac are +ĠRead ing +g ments +Ġhelic op +Ġsacr ifice +ay a +p aren +V A +ĠL es +ĠStud io +Ġviol ations +ĠAn na +ac er +é ¾ +ĠR at +ĠBe ck +ĠD ick +ĠA CT +Ġcomp osition +Ġtext ure +ĠO wn +Ġsmart phone +ĠN A +Ġfor b +im port +Ġdef ending +il st +re r +Ġo h +ĠJere my +Ġbank ing +cept ions +Ġrespect ive +/ . +Ġdr inks +ĠW i +Ġb ands +ĠL iverpool +Ġg rip +ĠB uy +Ġopen ly +Ġreview ed +per t +Ġver ify +ĠCo le +ĠW ales +M O +Ġun pre +Ġshel ter +ĠIm perial +Ġgu i +ĠD ak +Ġsuggest ions +Ġexplicit ly +Ġsl ave +Ġblock chain +Ġcompet ing +Ġprom ising +S ON +Ġsoc cer +Ġconst itution +4 29 +Ġdist ract +ĠU ser +es ides +ĠMet hod +ĠTok yo +Ġaccompan ied +Cl ient +s ur +al og +Ġident ification +Ġinv asion +as ma +Ġindust ries +pp ers +Ġsub tle +ĠUn it +n atural +Ġsurv ived +Ġfl aw +ĺ ħ +ĠH oll +Ġdef icit +Ġtut orial +ĠCh ance +Ġarg uing +Ġcontem porary +Ġinteg ration +for ward +Ġt um +it is +Ġh iding +ĠD omin +ĠT an +ĠB uilding +ĠV in +Ġspokes person +ĠNot es +Ġemer ging +Ġprepar ation +Ġpro st +Ġsuspect s +Ġaut onom +D escription +Ġdeal t +ĠP ear +Ġstead y +Ġdecre ased +Ġso vere +ĠCl in +Ġgrad ually +ors es +ĠW AR +S erv +ãĤ ¢ +h r +Ġd irty +ĠB arn +ĠB C +Ġd il +Ġcal endar +Ġcompl iance +Ġch amber +b b +Ġpass enger +ate ful +ĠT itle +ĠSyd ney +ĠG ot +Ġdark ness +Ġdef ect +Ġpack ed +ass ion +Ġgod s +Ġh arsh +IC K +le ans +Ġalgorith m +Ġoxy gen +Ġvis its +Ġbl ade +Ġkil omet +ĠKent ucky +Ġkill er +P ack +enn y +Ġdiv ine +Ġnom ination +be ing +Ġeng ines +Ġc ats +Ġbuff er +ĠPh ill +Ġtra ff +AG E +Ġtong ue +Ġrad iation +ere r +m em +ĠExpl icit +é¾ į +Ġcou ples +Ġphys ics +ĠMc K +Ġpolit ically +aw ks +ĠBl oom +Ġwor ship +e ger +ut er +ĠF O +Ġmat hemat +Ġsent enced +Ġdis k +ĠM arg +Ġ/ * +P I +Ġoption al +Ġbab ies +Ġse eds +ĠScott ish +Ġth y +] ] +ĠHit ler +P H +ng th +Ġrec overed +ing e +Ġpow der +Ġl ips +Ġdesign er +Ġdis orders +Ġcour age +Ġch aos +" },{" +Ġcar rier +b ably +H igh +ĠR T +es ity +l en +Ġrout es +u ating +F il +N OT +w all +s burgh +Ġeng aging +ĠJava Script +ore r +li hood +Ġun ions +ĠF ederation +ĠTes la +Ġcomple tion +ĠT a +Ġprivile ge +ĠOr ange +Ġne ur +paren cy +Ġb ones +Ġtit led +Ġprosecut ors +ĠM E +Ġengine er +ĠUn iverse +ĠH ig +n ie +o ard +Ġheart s +ĠG re +uss ion +Ġmin istry +Ġpen et +ĠN ut +ĠO w +ĠX P +in stein +Ġbul k +S ystem +ic ism +ĠMarket able +Ġpre val +Ġpost er +Ġatt ending +ur able +Ġlicens ed +ĠG h +et ry +ĠTrad able +Ġbl ast +à ¤ +ĠTit an +ell ed +d ie +H ave +ĠFl ame +Ġprof ound +Ġparticip ating +Ġan ime +ĠE ss +Ġspec ify +Ġregard ed +ĠSpe ll +Ġs ons +own ed +Ġm erc +Ġexper imental +land o +h s +ĠDun geon +in os +Ġcomp ly +ĠSystem s +ar th +Ġse ized +l ocal +ĠGirl s +ud o +on ed +ĠF le +Ġconstruct ed +Ġhost ed +Ġsc ared +act ic +ĠIs lands +ĠM ORE +Ġbl ess +Ġblock ing +Ġch ips +Ġev ac +P s +Ġcorpor ation +Ġo x +Ġlight ing +Ġneighb ors +ĠU b +ar o +Ġbe ef +ĠU ber +F acebook +ar med +it ate +ĠR ating +ĠQu ick +Ġoccup ied +Ġaim s +ĠAdd itionally +ĠInt erest +Ġdram atically +Ġhe al +Ġpain ting +Ġengine ers +M M +ĠM ust +Ġquant ity +P aul +Ġearn ings +ĠPost s +st ra +ãĥ¼ ãĥ +Ġst ance +Ġdro pping +sc ript +Ġd ressed +M ake +Ġjust ify +ĠL td +Ġprompt ed +Ġscr ut +Ġspeed s +ĠGi ants +om er +ĠEd itor +Ġdescrib ing +ĠL ie +ment ed +Ġnow here +oc aly +Ġinst ruction +fort able +Ġent ities +Ġc m +ĠN atural +Ġinqu iry +Ġpress ed +iz ont +for ced +Ġra ises +ĠNet flix +ĠS ide +Ġout er +Ġamong st +im s +ows ki +Ġclim b +ne ver +Ġcomb ine +d ing +Ġcomp r +Ġsignific ance +Ġremem bered +ĠNev ada +ĠT el +ĠSc ar +ĠWar riors +ĠJ ane +Ġcou p +b as +Ġtermin al +, - +O H +Ġt ension +Ġw ings +ĠMy ster +�� �� +ĠUn like +val id +viron ments +ĠAl i +Ġn aked +book s +ĠM un +ĠG ulf +Ġd ensity +Ġdim in +Ġdesper ate +Ġpres idency +Ġ198 6 +h y +IN D +Ġun lock +im ens +Ġhand led +ĠE b +Ġdisapp eared +Ġgen re +Ġ198 8 +Ġdetermin ation +St ream +ik o +ap ters +Ġacknow ledge +J an +Ġcapital ism +P at +Ġ20 20 +Ġpain ful +Ġcur ve +Ġbom bs +st orm +ĠMet al +en cer +ĠF ig +ĠA aron +anc hes +Ġins piration +Ġexha ust +t ains +ash i +Ġdesc ript +Ġr itual +ĠChel sea +Ġpromot ion +ĠH ung +ĠW ard +iv a +ĠE T +Ġto ss +all ow +ĠFranc is +D ep +Ġhapp iness +ĠGl ass +Ġbet a +Ġstreng then +N E +o a +Ġbutt ons +ĠMur ray +Ġkick ed +Qu est +ĠT alk +ĠS everal +ĠZ ero +Ġdr one +ul k +Ġc am +ĠM obile +Ġprevent ing +Ġret ro +ĠA x +Ġcru el +Ġflo at +. ), +Ġfil ing +ĠGr ant +ĠB or +Ġr ib +Ġchampions hip +ĠM erc +Ġsty les +Ġc ake +Ġbuild s +ĠS elf +io x +Ġep ic +oy d +B el +ĠSt ew +. ( +ah u +ĠBe yond +Ġout s +Ġsol o +ĠT ree +Ġpres erve +Ġt ub +AR E +ro c +ĠIm pro +ĠW right +Ġbu nd +Ġtr aged +Ġoccas ional +b ian +Sec ond +r ons +Ġinter actions +form ed +s ing +Ġown s +Ġh ockey +Gener al +Ġlog ical +Ġexp end +Ġesc al +ĠGr iff +ĠC rown +ĠRes erve +Ġsto pping +Ġexc use +sec ond +Ġoper ated +Ġre aches +ĠMal ays +Ġpoll ution +ĠBrook lyn +Ġde lete +Ġhas h +Bl ock +ah a +âĢ ³ +Ġsh orter +p iece +> >> +ĠM ormon +t or +Ġpartic les +ĠB art +ry ption +Ġad min +Ġsqu ee +VID IA +Ġcreat or +iam eter +ic ular +N BC +Ġgrab bed +Ġn odd +Ġr ated +Ġrot ation +Ġgr asp +Ġexcess ive +ĠE C +ĠWh it +Ġinvent ory +ault s +ĠF B +Ġe cosystem +Ġbill ions +Ġvent ure +n amed +Ġdef ender +out e +Inst ead +ir able +W ar +Ġassum ption +Ġb ite +Ġearth qu +t ail +sp ace +Ġgif ts +boy s +Ġinev itable +Ġstruct ural +Ġbenef icial +Ġcompe lling +h ole +erv ation +Ġco at +o j +inc arn +ĠY ears +Ġdetermin ing +Ġrhet oric +Ġbound aries +Ġwh ites +A nt +add y +) - +ra ham +eter min +Ġhar vest +ĠCon c +Ġlapt op +ĠM atch +Ġenjoy ing +cc a +oll ar +Ġtri ps +Ġadd iction +ĠS ak +Ġpow ered +Ġc ous +ĠRuss ians +ie re +Ġret rie +qu ality +Ġdiff er +Ġking dom +ĠL aur +ĠCap itol +Ġcon clusions +ĠAl tern +ĠN av +Ġtrans parent +B ER +G roup +ĠCom plete +Ġinf er +Ġint rig +Ġins ane +R O +oph ob +is en +qu al +Mich ael +Ġm useum +ĠP ope +Ġres et +r ative +f ive +Ġagg reg +itte es +osit ory +Ġcar b +ĠRec ord +Ġdec ides +ĠF ix +Ġexcept ions +ĠCommission er +un s +ĠEnvironment al +Ġlegend ary +ist ence +Ġtun nel +k m +Ġins ult +Ġt roll +Ġsh ake +Ġdet ention +qu es +ĠCh rome +ĠF iles +Ġsub t +Ġprospect s +Ġpro l +re nder +pro of +Ġperform ances +St r +Ġh ref +ern ame +Ġachieve ment +Ġf ut +F ull +ĠLe ban +go ogle +ãĥ Ī +amp a +May be +Ġproject ed +ĠE mb +Ġcol leg +Ġa wards +Ġâ Ķ +G old +ĠBl ake +ĠR aj +if ting +Ġp ending +Ġinst inct +Ġdevelop ments +Con nect +ĠM and +ĠW ITH +ĠPhilipp ines +prof ile +Ġalt ogether +ĠB und +ĠT D +oo oo +amp ed +ip h +Ġste am +Ġold est +Ġdet ection +ul pt +Ġ ç +ĠWay ne +200 6 +f a +Ġcir cles +ĠF u +Ġdon ors +appropri ate +ĠDak ota +j amin +Ġmotiv ated +Ġpurch ases +ĠLouis iana +ĠS pl +Ġgl obe +Ġ10 5 +z ip +c all +Ġdepart ments +Ġsustain able +10 5 +ĠO P +if iers +Ġprevent ed +Ġinc omp +ĠComm ander +Ġdom inated +Ġ » +Ġinvest ed +Ġcomplex ity +Ġin cl +Ġens uring +Ġreal m +yn c +ĠInd ependent +r ained +ĠJ en +ĠFl ight +Ġat he +Ġspec ulation +ĠT E +oc ate +t ic +Ġpl aint +her ry +Ġto y +Ġ1 11 +Ġpl ates +st atus +ĠIs a +Ġdev oted +C op +ĠE S +25 5 +ur rency +M ain +Ġsl aves +Ġpe pper +Ġqu otes +Ġce iling +ĠF ish +Ġtrans formation +Ġfra ction +Ġadvant ages +Ġto ile +Ġstun ning +Ġmo ist +bre aking +s i +ĠL ocation +ĠMed ium +Ġtext s +Ġu gly +Ġb io +. âĢĶ +ĠB ased +Ġtr ains +ĠW ing +ĠAn cient +ĠRec ords +ĠH ope +Spe cial +ades h +ob i +[ / +Ġtempor arily +V er +h u +os er +Ġover night +Ġm amm +ĠTre asury +ĠV enezuel +ĠMeg a +Ġt ar +Ġexpect s +bl ack +or ph +\\ \\ +Ġaccept ance +Ġrad ar +s is +Ġjun ior +Ġfram es +Ġobserv ation +ac ies +P ower +ĠAdv anced +M ag +olog ically +ĠMe chan +Ġsent ences +Ġanaly sts +augh ters +force ment +Ġv ague +Ġcl ause +Ġdirect ors +Ġeval uate +Ġcabin et +M att +ĠClass ic +A ng +Ġcl er +ĠB uck +Ġresear cher +Ġ16 0 +Ġpoor ly +Ġexperien cing +ĠP ed +ĠMan hattan +Ġfre ed +Ġthem es +ad vant +Ġn in +Ġpra ise +10 4 +ĠLib ya +b est +Ġtrust ed +Ġce ase +Ġd ign +D irect +Ġbomb ing +Ġm igration +ĠSci ences +Ġmunicip al +ĠA verage +Ġgl ory +Ġreve aling +Ġare na +Ġuncertain ty +Ġbattle field +ia o +G od +Ġc inem +ra pe +el le +ap ons +Ġlist ing +Ġwa ited +Ġsp otted +ke ley +ĠAud io +e or +ard ing +idd ing +ig ma +ĠN eg +Ġl one +Ġ ---- +ex e +d eg +Ġtrans f +Ġwas h +Ġsl avery +Ġexpl oring +ĠW W +ats on +Ġen cl +l ies +ĠC reek +Ġwood en +Man ager +ĠBr and +um my +ĠAr thur +Ġbureau cr +Ġbl end +ar ians +F urther +Ġsupposed ly +Ġwind s +Ġ19 79 +Ġgrav ity +Ġanalys es +ĠTra vel +ĠV eter +Ġd umb +Ġaltern ate +g al +Ġconsum ed +Ġeffect iveness +.' ' +Ġpath s +ond a +L A +ĠStr ong +Ġen ables +Ġesc aped +Ġ" " +Ġ1 12 +Ġ198 3 +Ġsm iled +Ġtend ency +F ire +Ġp ars +ĠR oc +Ġl ake +Ġf itness +ĠA th +ĠH orn +Ġh ier +Ġimp ose +m other +Ġp ension +ic ut +bor ne +ic iary +. _ +ĠS U +Ġpol ar +is y +eng u +itial ized +AT A +w rite +Ġexerc ises +ĠD iamond +ot ypes +Ġharm ful +on z +Ġprint ing +st ory +Ġexpert ise +ĠG er +Ġtraged y +ĠF ly +Ġd ivid +amp ire +st ock +M em +Ġre ign +Ġun ve +Ġam end +ĠProp het +Ġmut ual +ĠF ac +Ġrepl acing +H ar +ĠCirc uit +Ġthro at +ĠSh ot +Ġbatter ies +Ġto ll +Ġaddress ing +ĠMedic aid +Ġp upp +ĠN ar +ol k +Ġequ ity +M R +ĠHis pan +ĠL arge +m id +D ev +Ġexp ed +Ġdem o +ĠMarsh all +erg us +Ġf iber +Ġdiv orce +ĠCre ate +Ġsl ower +ĠPark er +ĠStud ent +ĠTr aining +Ret urn +ĠT ru +Ġc ub +ĠRe ached +Ġpan ic +Ġqu arters +Ġre ct +Ġtreat ing +Ġr ats +ĠChristian ity +ol er +Ġsac red +Ġdecl are +ul ative +et ing +Ġdeliver ing +est one +Ġt el +ĠL arry +Ġmet a +ac cept +art z +ĠRog er +hand ed +Ġhead er +Ġtra pped +ĠCent ury +Ġkn ocked +ĠOx ford +Ġsurviv ors +b ot +Ġdemon stration +Ġd irt +Ġass ists +OM E +ĠD raft +ortun ate +fol io +pe red +ust ers +g t +ĠL ock +Ġjud icial +ver ted +Ġsec ured +out ing +ĠBook s +Ġhost ing +Ġlif ted +l ength +Ġj er +Ġwhe els +ĠR ange +umbn ails +Ġdiagn osis +te ch +ĠStew art +ĠP ract +Ġnation wide +Ġde ar +Ġoblig ations +Ġgrow s +Ġmand atory +Ġsusp icious +! ' +A pr +G reat +Ġmort gage +Ġprosecut or +Ġeditor ial +ĠK r +Ġprocess ed +ung le +Ġflex ibility +Ear lier +ĠC art +ĠS ug +Ġfoc uses +Ġstart up +Ġbre ach +ĠT ob +cy cle +ãĢ Į +ro se +Ġb izarre +ãĢ į +Ġveget ables +$ $ +Ġret reat +osh i +ĠSh op +ĠG round +ĠSt op +ĠHawai i +ĠA y +Per haps +ĠBe aut +uff er +enn a +Ġproduct ivity +F ixed +cont rol +Ġabs ent +ĠCamp aign +G reen +Ġident ifying +Ġreg ret +Ġpromot ed +ĠSe ven +Ġer u +ne ath +aug hed +ĠP in +ĠL iving +C ost +om atic +me ga +ĠN ig +oc y +Ġin box +Ġem pire +Ġhor izont +Ġbr anches +Ġmet aph +Act ive +ed i +ĠFil m +ĠS omething +Ġmod s +inc ial +ĠOrig inal +G en +Ġspir its +Ġear ning +H ist +Ġr iders +Ġsacr ific +M T +ĠV A +ĠS alt +Ġoccup ation +ĠM i +Ġdis g +lic t +Ġn it +Ġn odes +e em +ĠP ier +Ġhat red +ps y +ãĥ ī +Ġthe ater +Ġsophistic ated +Ġdef ended +Ġbes ides +Ġthorough ly +ĠMedic are +Ġbl amed +arent ly +Ġcry ing +F OR +pri v +Ġsing ing +ĠI l +Ġc ute +o ided +olit ical +ĠNe uro +å ¤ +Ġdon ation +ĠEag les +ĠG ive +T om +Ġsubstant ially +ĠLic ense +ĠJ a +Ġg rey +ĠAn imal +ĠE R +ĠU nd +Ġke en +Ġconclud e +ĠMississ ippi +Eng ine +ĠStud ios +P ress +o vers +ll ers +Ġ3 50 +ĠR angers +Ġr ou +ert o +E p +iss a +iv an +Ġse al +ĠReg ist +dis play +Ġwe aken +u um +ĠComm ons +ĠS ay +Ġcult ures +Ġl aughed +Ġsl ip +Ġtreat ments +iz able +m art +ĠR ice +Ġbe ast +Ġob esity +ĠLa ure +ig a +Wh ich +hold er +Ġelder ly +Ġp ays +Ġcompl ained +Ġc rop +Ġpro c +Ġexplos ive +ĠF an +ĠAr senal +A uthor +ef ul +Ġme als +Ġ( - +id ays +Ġimag ination +Ġann ually +Ġm s +as ures +H ead +ik h +m atic +Ġboy friend +ĠCom puter +Ġb ump +Ġsur ge +ĠCra ig +ĠKir k +D el +medi ate +Ġscen arios +ĠM ut +ĠSt ream +Ġcompet itors +Ù Ħ +ĠStan ford +ĠRes ources +az ed +b age +Ġorgan is +ĠRe lease +Ġsepar ately +Ġha bits +Ġmeasure ments +ĠCl ose +Ġaccomp any +Ġg ly +Ġt ang +ĠR ou +Ġplug in +Ġcon vey +ĠChall enge +oot s +j an +Ġcur s +ĠRel ations +ke eper +Ġapproach ing +p ing +Spe aking +Ġarrang ement +ĠV I +are ttes +Ġaffect ing +Ġperm its +b ecause +Ġu seless +ĠH us +!! !! +Ġdestro ying +Un fortunately +Ġfasc inating +S em +Ġelect oral +Ġtrans parency +ĠCh aos +Ġvolunte er +Ġstatist ical +Ġactiv ated +ro x +We b +H E +ĠHamp shire +is ive +M ap +Ġtr ash +ĠLaw rence +st ick +C r +Ġr ings +EX T +Ġoper ational +op es +D oes +ĠEv ans +Ġwitness ed +P ort +Ġlaunch ing +ec onom +w ear +ĠPart icip +um m +cul es +ĠR AM +ĠT un +Ġass ured +Ġb inary +Ġbet ray +Ġexpl oration +ĠF el +Ġad mission +it ated +S y +Ġav oided +ĠSim ulator +Ġcelebr ated +ĠElect ric +¥ ŀ +Ġcl uster +itzer land +he alth +L ine +ĠN ash +at on +Ġsp are +Ġenter prise +ĠD IS +clud es +Ġfl ights +Ġreg ards +ĠÃ Ĺ +h alf +Ġtr ucks +Ġcontact s +Ġunc ons +ĠCl imate +Ġimm ense +N EW +oc c +ect ive +Ġemb od +Ġpat rol +Ġbes ide +Ġv iable +Ġcre ep +Ġtrig gered +ver ning +Ġcompar able +q l +Ġg aining +ass es +Ġ( ); +ĠG rey +ĠM LS +s ized +Ġpros per +" ? +Ġpoll ing +Ġsh ar +ĠR C +Ġfire arm +or ient +Ġf ence +Ġvari ations +g iving +ĠP i +osp el +Ġpled ge +Ġc ure +Ġsp y +Ġviol ated +Ġr ushed +Ġstro ke +ĠBl og +sel s +ĠE c +,' ' +Ġp ale +ĠColl ins +ter ror +ĠCanad ians +Ġt une +Ġlabor atory +Ġn ons +t arian +Ġdis ability +ĠG am +Ġsing er +al g +ĠSen ior +Ġtrad ed +ĠWar rior +Ġinf ring +ĠFrank lin +Ġstr ain +ĠSwed ish +Ġsevent h +ĠB enn +ĠT ell +Ġsynd rome +Ġwond ered +id en +++ ++ +ig o +Ġpur ple +Ġjournal ism +Ġreb el +Ġf u +bl og +Ġinv ite +ren cies +ĠCont act +Is rael +ĠCont ent +Ġche er +Ġbed room +ĠEngine ering +ĠQue ens +Ġd well +ĠPlay Station +ĠD im +ĠCol on +l r +Ġoper ates +Ġmotiv ation +US A +ast ered +C ore +ĠTr uth +ol o +OS E +ĠMem ory +Ġpred ec +Ġan arch +Ġ19 20 +ĠY am +à ¨ +b id +Ġgr ateful +Ġexc itement +Ġtre asure +Ġlong est +ct ive +Ġdes erves +Ġreserv es +Ġcop s +ĠOtt awa +ĠEgypt ian +ank ed +Ġart if +Ġhypot hesis +: / +Ġpurch asing +Ġlove ly +H P +Ġdiv ide +Ġstrict ly +Ġquestion ing +Ġtaxp ayers +ĠJ oy +Ġroll s +ĠHe avy +Ġp orts +Ġmag netic +Ġinf lamm +Ġbr ush +t ics +â ĪĴ +Ġbott les +pp y +Ġp add +ãĤ ¯ +m illion +Ġdevast ating +Ġcomp iled +Ġmed ication +Ġtw elve +ĠPer ry +Sp ace +im b +y our +Ġle aked +ĠT ar +Ġun ity +Ġinfect ed +Ġtravel ed +ID E +ĠMc Donald +t xt +ĠPr inc +Ġinter ven +ĠTai wan +ĠP ow +Ġbe aring +ĠTh read +Ġz ones +iz ards +un ks +Ch apter +ll or +Ġ · +Ġw ounds +Ġdisc retion +Ġsucceed ed +ik ing +Ġicon ic +C all +Ġscreen ing +ĠM is +ict s +Ġmin isters +Ġsepar ation +Pl ayer +Ġb ip +Ġbel oved +Ġcount ing +ĠE ye +ar ound +ing ing +Ġtable t +Ġoff ence +in ance +h ave +ĠInf o +ĠNin ja +Ġprotect ive +ĠC ass +M ac +ĠQual ity +N orth +Ġ ic +ĠCub a +ĠChron icle +ĠPro perty +Ġfast est +ot os +ĠG erm +OW N +Ġbo om +ĠStan ley +ergus on +Ġcle ver +Ġent ers +m ode +ter ior +ĠS ens +Ġlin ear +AR K +Ġcomp aring +Ġpure ly +Ġsaf er +ĠPot ter +Ġc ups +R T +Ġgl uc +Ġatt ributed +Ġdu pl +ĠP ap +Ġprec ious +Ġp a +iction ary +ĠT ig +ĠTo o +ol utions +st an +Ġrob ots +Ġlob b +Ġstat ute +Ġprevent ion +w estern +16 0 +ĠAct ive +ĠMar ia +h al +N one +ell ar +ĠK B +ĠPart ners +ĠSing le +ĠFollow ing +ang o +ac ious +Ġth ou +Ġk g +Ġinflu ential +ĠFriend s +S ur +ain ted +Ġfor ums +Ġst arter +Ġcitizens hip +ĠE lection +on ge +ot ation +os ph +;; ;; +ut ical +p ur +ere n +Ġaccus ations +bit ious +ab bit +ĠOr d +Post ed +ir k +Ġsens itivity +ic he +ĠAm y +ĠF ab +Ġsum mit +Ġped est +Ġrub ber +Ġagric ultural +Ġcan cel +A E +Ġin aug +Ġcont am +Ġfirm ly +i w +st age +ĠK an +Ġt ier +Ġinv ention +Ġtransl ated +ĠR ules +B ox +Tw itter +ID S +Ġp izza +Ġdeb ug +ĠD rop +v s +Ġh orses +b ig +Ġb oring +Ġh ood +ĠMcC ain +at ched +ĠBro s +Ġsk ip +Ġess ay +st at +ĠLeg ends +Ġam munition +au c +Ġshoot er +Ġun h +Ġsuppl ied +Ġgener ic +ĠS K +ib an +yr ics +Ġ25 5 +Ġclim bing +Form er +Ġfl ip +Ġjump ing +Ġfrust ration +ĠTer ry +Ġneighborhood s +Ġmed ian +be an +Ġbr ains +Follow ing +Ġsh aped +Ġdraw s +Ġal tered +J ack +Ġrecip es +Ġsk illed +we alth +ach i +e lection +Ġbehavi ors +de als +ĠU ntil +F e +Ġdecl aration +mar ks +ĠBet ween +cel ona +Ġres on +Ġbub ble +Am ong +Ġim perial +G S +Ġfemin ist +200 5 +ĠK yle +Ġaccount ing +ĠTe le +ĠT yr +Ġconnect ing +Ġre hab +ĠP red +s im +Ġmeant ime +Ġphys ician +M W +ĠCamp bell +ĠBr andon +Ġcontribut ing +ĠR ule +ĠWe ight +ĠN ap +Ġinter active +Ġv ag +Ġhel met +ĠCom b +f our +Ġsh ipped +Ġcomple ting +ĠP D +PD ATE +Ġspread ing +Ġsc ary +erv ing +ĠG as +Ġfr ank +s chool +Ġrom antic +Ġstab il +R ob +Ġaccur ately +Ġac ute +ĠH ann +Ġsymbol s +Ġcivil ization +ĠA W +Ġlight ning +Ġcons iders +Ġven ue +Ġ × +Ġo ven +ĠS F +h is +Ġn u +ĠLear n +Ġpe oples +Ġst d +Ġsle e +Ġs lic +ĠStat istics +Ġcor ners +ĠB aker +Ġ: ) +ment ation +ol ver +Ġlaugh ing +ĠT odd +ond e +ĠH ills +Ġn uts +ĠW oman +pl ane +Ġl iver +ĠIn side +S orry +Ġagre es +Ġfund ament +ĠF isher +Ġa uction +Ġthread s +gl as +ĠBas ic +ĠN at +Ġlack ing +Ġceleb ration +j u +Ġs illy +E uro +Ġt att +ight y +cont rolled +T est +ĠSing h +Ġr age +Ġrh yth +o ffic +ĠPh antom +Ġhead lines +Ġrespond ing +ĠMor ning +Ġvit amin +Ġboot s +ĠS ite +al in +p i +Ġvir al +ĠU C +D ER +ĠSe x +Ġst ocks +c urrent +Ġch urches +ĠR are +ĠMur phy +Ġden ial +ĠG aming +Ġtou g +Ġn ick +Ġm akers +ĠRon ald +Ġgener ous +ĠD oc +ĠMor ris +Ġtransform ed +ĠN ormal +Ġ10 4 +ĠKick starter +ĠUp on +On line +ĠI RS +Ġw rap +Ġl oving +Ġarri ves +ĠD ue +Ġhe ter +ĠM ade +Ġrent al +Ġbelong s +Ġatt orneys +Ġcro ps +Ġmat ched +ul um +ol ine +10 9 +Ġdis par +Ġbuy ers +ĠCam bridge +Ġeth ics +rou ps +Ġjust ified +Ġmarg inal +Ġrespect ed +win ning +Ġnodd ed +ĠSer ge +ĠForm er +C raft +######## ######## +ĠWar ner +Ġd ash +et e +Ġent ert +ĠE scape +out heast +Ġkn ees +ĠB omb +Ġr ug +P ass +Ġatt itudes +go vernment +ĠPri or +Ġqual ities +Ġnot ification +ĠPh one +l ie +Ġanticip ated +ĠCom bat +ĠBar ry +Ġ198 2 +Us ers +on er +Ġcomput ing +ĠConnect icut +Ġless er +Ġpe ers +ĠC u +Ġtechn ically +Ġsub mission +ĠUn iversal +Ġman ually +our ge +Ġrespond ents +ĠB TC +ĠH ost +Ġf are +ĠB ird +Ġrece ipt +al so +Ġj ack +Ġagric ulture +Ġsk ull +Ġ! = +Ġpass ive +ĠC I +Ġsoc ieties +Ġremind ed +Ġinter ference +B uy +Ġâ ľ +g on +Ġscrut iny +ĠW itch +Ġconduct ing +Ġ ãĥ +Ġexch anges +ĠMit chell +Ġinhab it +Ġtw ist +B D +Ġwhere ver +group on +Ġj okes +ĠBen jamin +ĠR andom +fr ame +ĠL ions +Ġhighlight ed +ĠArk ansas +E nt +Ġp ile +Ġpre lim +g s +mind ed +Ġfel ony +ĠG A +ĠL uck +Ġpract ically +ĠB os +Ġact ress +D am +ĠB ou +Ġvis a +Ġembed ded +Ġhy brid +Ġear liest +Ġsoon er +s ocial +ĠH A +Ġste ep +Ġdis advant +Ġexplo it +ĠE gg +ĠUlt ra +Ġnecess ity +L ocal +ie ge +Ġd ated +Ġmass es +Ġsubsc ription +pl ess +Ġan onym +Ġpresum ably +Bl ue +The ir +asket ball +ĠPhil ip +Ġcom ed +load ed +r ane +Ġref lection +Ch ina +Ġext ends +Ġform ing +Ġund ers +200 1 +Ġgr at +Ġconcent rations +Ġins ulin +Ġsec ular +Ġwh ilst +Ġwin ners +Ad vertisements +Ġdeliber ately +ĠWork ing +Ġs ink +et ics +d ale +Ġmand ate +Ġg ram +Ġvac ation +Ġwarn ings +ri pp +ĠTH AT +Ġcomment ary +Ġint u +Ġa est +Ġreason ing +Ġbreak down +ĠZ ombie +Ġ-- > +ĠPolit ical +c ott +Ġthr ust +Ġtechn ological +Ġdec iding +Ġtraff icking +L ong +W elcome +pr ising +ĠCommun ications +Ġend ors +Ġsw ift +Ġmetab ol +co ins +res a +ĠHT TP +Ġen roll +ĠH appy +us r +int age +Ġ[ " +u ably +ĠM aterial +Ġrepe al +Se pt +k h +ĠMod i +Ġunder neath +ĠI L +sh ore +Ġdiagn osed +ace utical +Ġsh ower +au x +ĠSw itch +ĠStre ngth +Ġj ihad +n ational +Ġtra uma +uss y +on i +Ġcons olid +Ġcal ories +ĠF lynn +ag ged +16 8 +ĠP ink +Ġfulf ill +Ġch ains +Ġnot ably +ĠA V +L ife +ĠCh uck +m us +ĠUr ban +ĠH end +Ġdep osit +ĠS ad +Ġaff air +OR K +ie val +ĠF DA +Ġt rop +ĠOver all +Ġvirt ue +Ġsatisf action +au nd +Ġl un +ĠSw itzerland +ĠOper ation +pro cess +Ġsh ook +Ġcount ies +le ased +ĠCharl otte +1 12 +Ġtrans cript +Ġre dd +p ush +ĠHe y +ĠAn alysis +[ " +Ġaltern atives +ard less +Ġele ph +Ġpre jud +ĠLe af +H aving +ĠH ub +Ġexpress ions +ĠVol ume +Ġshock ing +ĠRed s +Ġread ily +Ġplan ets +ad ata +Ġcollaps ed +ĠMad rid +Ġir rit +i pper +ĠEn c +ĠW ire +Ġbu zz +ĠG P +ash a +Ġaccident ally +ur u +Ġfrust rated +ĠS A +Ġhung ry +ĠH uff +Ġlab els +ant o +ĠE P +Ġbar riers +) | +ĠBer keley +ĠJ ets +Ġp airs +ĠL an +J ames +ĠB ear +Ġhum or +ĠLiber ty +Ġmagn itude +Ġag ing +ĠM ason +Ġfriends hip +umb ling +Ġemer ge +Ġnewsp apers +Ġam bitious +ĠRich ards +atern al +Ġ198 1 +Ġcook ies +Ġsc ulpt +Ġpur suit +L ocation +Ġscript s +p c +Ġarrang ements +Ġd iameter +Ġl oses +am ation +Ġl iqu +ĠJ ake +aret te +Ġunderstand s +ĠZ en +v m +Ġappro ve +Ġw ip +Ġult ra +Ġint end +ĠD I +asc ular +Ġst ays +ĠK or +ĠK l +Ġinvest ing +L a +Ġbelie ving +b ad +m outh +Ġtaxp ayer +ãĥ ĥ +ĠQue bec +Ġl ap +ĠSw iss +d rop +Ġdr ain +ir i +et c +ft en +ĠN ex +Ġst raw +Ġscream ing +Ġcount ed +Ġdam aging +Ġamb assador +cent ury +Ġpro x +Ġarrest s +u v +il ateral +ĠCh arg +Ġpresc ribed +Ġindepend ently +Ġf ierce +ĠB aby +Ġb rave +Ġsu its += > +Ġbas eline +ĠR ate +Ġis lands +Ġ( ( +g reen +ix els +Ġname ly +ĠVill age +th an +am y +V ersion +g mail +ential s +ĠS ud +ĠMel bourne +Ġarri ving +Ġquant um +e ff +rop olitan +T ri +Ġfun eral +ĠI R +ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ +ĠC ob +it ably +Ġt urb +Ġcomb o +Re view +Ġdeploy ment +u ity +ĠB ott +Ġinv isible +Ġrender ing +Ġunl ocked +Ġa qu +ĠVlad imir +Ġp ad +ĠBr ain +ĠLeg acy +dr agon +ĠKurd ish +Ġsound ed +Ġdet ained +ĠD M +g ary +Ġd aughters +Ġdistur bing +uk a +ĠPar ad +Ġt ast +Ġunf ortunate +Ġu l +em in +Ġattend ance +tr l +Ġpar ks +ĠMem orial +ĠAl ice +oth y +gu ard +ĠD ise +ĠSh an +ĠFor um +R ich +Ġshif ted +ue z +Ġl ighter +ĠMag n +Ġc od +S ch +ham mad +P ub +3 50 +ĠP okemon +Ġprot otype +Ġun re +B ase +ĠStud ents +ĠRep ly +ĠCommun ist +Ġg au +ĠTy ler +I Z +Ġparticip ated +Ġsup rem +ĠDet ails +Ġvessel s +ro d +Ġt ribe +ke ep +Ġassum ptions +Ġp ound +Ġcr ude +ĠAv ailable +Ġswim ming +Ġin clusion +Ġadv ances +c ulation +Ġconserv ation +Ġover d +ĠBuff alo +Art icle +ed ge +Ġaw a +ĠMad ison +Ġsid ew +Ġcat ast +ĠK rist +uc le +ĠHigh way +ĠTer ror +Ġactiv ation +Ġuncons cious +ĠSat an +ĠSus an +ill ery +Ġarr anged +i op +Ġrum ors +ur ring +th ink +ĠKe ith +ĠK ind +Ġavoid ing +by n +n ut +ĠSpe aker +r us +n ames +Ġgu ilt +ĠOlymp ics +Ġsa il +ĠM es +lev ant +ĠColumb us +a ft +C ity +S outh +ĠHar vey +ĠP un +S everal +Ġment ally +Ġimp ress +m ount +ĠUb untu +âĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +ĠSuper man +ĠMP s +Ġintent ions +ĠR acing +Ġlike lihood +Ġ2 40 +T otal +Ġto ys +ĠW atson +Ġur ge +L ear +ĠP aper +Ġoccur ring +ĠB eng +ĠC ert +Ġst ones +T im +ĠTw in +z b +ĠD ynam +Ġpolit ician +k ens +ĠEnter prise +UT ERS +Ġab ol +Ġref resh +Ġarbit rary +pe ction +Ġtrou bles +Ġ} ); +t v +Ġpil ots +Ġdist ribute +Ġaud it +Ġp ause +orig inal +Ġr ivals + £ +F ig +T L +ab il +ry ing +L in +ion ed +l on +Ġf ancy +Ġcr ashed +Ġt ract +Ġshe d +Ġcons ume +B ased +down load +in it +Ġvolt age +Int rodu +Ġcondem ned +ĠFin ance +res pect +Ġex cluded +Ġestablish ing +her ic +Ġher itage +Ġspect acular +Ġun st +ĠSnow den +ĠL ane +S an +Ġprotect ions +st ruction +inc inn +Ġmac ro +C ustom +ios ity +Ġes p +Ġfunction ing +Ġm ush +Ġp uzzle +Ġeth ical +M al +Ġgo verning +ĠF erguson +Ġrest ored +Ġst ressed +ĠCoun ter +ĠK as +cl ip +AN S +Ġse iz +U K +by ss +old own +ap i +Ġperman ently +oun ters +W est +Th rough +L ight +at oes +Ġne at +Ġc ord +ure r +Ġsevere ly +ĠA ven +Ġinter rog +Ġtri ple +G iven +N umber +Ġar ise +Ġs her +pl ant +Ġfl ower +ĠC ou +Ġat e +Ġnew er +b ul +Ġmean while +ĠL air +Ġadjust ment +ĠCop yright +Ġd ivers +i ological +Ġgam ers +o at +Ġhistor ically +Ġanal og +Ġlong time +Ġpres cription +ĠM ist +ĠHy per +ĠM aine +ĠDe ity +Ġmulti pl +ĠRe incarn +ĠH yd +ĠP ic +S il +r ants +ĠC ris +. ; +( { +epend ence +Ġrec y +ate ur +Ġqu ad +Ġgl ob +Ġcon ced +te am +Ġcapital ist +ĠL ot +Ġroy al +ĠCy ber +Ġblack s +met ic +ri v +ĠD anny +Ġsp o +ĠR O +Ġanim ated +rypt ed +ĠDep uty +Ġrend ered +F E +Ġstre ak +Ġcloud s +ĠDou g +~~~~ ~~~~ +Ġdisc our +ĠVe h +Ġpsych ology +ĠJ ourney +Ġcry stal +ĠFro st +Ġsuspic ion +Ġrel ate +or us +ĠC rypt +ĠN VIDIA +com ed +ut ing +incinn ati +Ġvulner ability +ost ic +Ġisol ation +Ġcool ing +ĠCoal ition +Ġ1 19 +F our +ĠDe al +Ġâ ī +se mble +ram ent +ĠBar celona +Ġ10 2 +Ġcoc aine +ocaly pse +F eb +ogen ic +Ġmut ation +Ġcrypt oc +ĠK el +ĠG it +a is +Ġs isters +AN K +Ġactiv ate +T er +Ġd read +yl on +Ġprop ri +A ust +ĠDef ault +Ġout door +Ġshe er +ce ive +Ġg ently +Ð ¾ +Pro gram +Ġâ ĨĴ +Ġve gan +ĠCr us +Ġrespons ibilities +ĠH R +OL D +Ġprev ents +Ġst iff +ĠW ere +Ġathlet ic +ĠSc ore +Ġ) : +Ġcolumn s +ĠL oc +av ailable +ĠF ram +ĠS essions +Ġcompan ion +Ġpack s +14 0 +ĠKn ights +Ġf art +Ġstream s +Ġsh ore +Ġapp eals +ĠPer formance +h aul +ĠSt ra +ĠN ag +10 3 +ĠTrans portation +B B +E v +z an +P ublic +Ġtw in +uls ion +M ult +Ġelect ro +Ġstat ue +ation ally +ĠN ort +Ġins pection +/ * +ig ue +Ġcomp assion +ĠT ales +ĠSte in +ĠSc reen +ĠB ug +ĠL ion +g irl +Ġwithdraw al +Ġobject ives +Ġblood y +Ġprelim inary +Ġj acket +Ġdim ensions +ĠC ool +ĠOcc up +Ġw reck +Ġdoub led +ank ing +Ġ19 75 +Ġglass es +ĠW ang +pro v +P ath +connect ed +ĠMult i +ĠNor way +agon ist +Ġfe ared +Ġtouch ing +Ġarg uably +¯¯¯¯ ¯¯¯¯ +ĠNC AA +che m +Ġsp at +ĠW WE +ĠC el +ig ger +Ġattack er +ĠJo in +ob ject +ett a +Ġelim inated +d et +Ġdest ruct +ĠLuc as +ct uary +18 0 +ĠBr ady +ĠBl ues +B ay +au kee +Ġtim eline +Ġdeleg ates +w ritten +uff icient +Ġsh apes +Cop yright +ou ble +serv ice +Ġp ione +Ġcolleg es +Ġrow s +Ġsp ite +Ġassess ed +3 60 +Ġle ase +Ġconfident ial +ck er +ĠMan ning +ĠV oice +Ġse aled +Ġcalcul ate +N O +ĠAss istant +Ġteen ager +ul ent +ather ine +Ġm ock +Ġd iamond +Ġf est +Ġsw itched +Ġres ume +ĠPu erto +Ġl anes +ir ation +ĠSimilar ly +Ġro d +ĠS el +ĠPal ace +ĠLim ited +e ous +Ġvar iant +Ġw ard +Ġ) ) +Sh ow +OO K +A lex +ĠN ep +br is +ĠWik ipedia +Ġexcept ional +Ġman ages +ĠD raw +Ag ain +Ġco pper +ut t +Ġex ports +Ġport folio +Ġelev ated +R ated +ĠOther wise +ĠT act +ĠShe l +ĠT X +" âĢĶ +Ġres ur +ĠW a +ven ant +Ġmon etary +pe ople +E mail +Ġfif ty +ĠS weet +ĠMalays ia +Ġconf using +ĠR io +ud a +uten ant +" ); +Ġpra ised +Ġvol umes +t urn +Ġm ature +Ġnon profit +Ġpassion ate +ĠPriv ate +Ġ10 3 +Ġdesc end +ç ¥ŀ +uff y +head ed +Whe ther +ri en +ze ch +be it +Ġch rom +ĠMc M +Ġd ancing +Ġe leg +ĠNot iced +11 5 +Ġadvoc acy +ENT S +amb ling +ĠMin or +ĠF inn +Ġprior ities +Ġthere of +ĠSt age +ĠRog ers +Ġsubst itute +ĠJ ar +ĠJeff erson +Ġlight ly +10 2 +ĠL isa +u its +ys ical +Ġshif ts +Ġd rones +Ġwork place +Ġres id +ens ed +ah n +Ġpref erences +ser ver +Ġdeb ates +d oc +ĠGod s +Ġhelicop ter +Ġhon our +Ġconsider ably +ed ed +ĠF emale +ĠAn ne +Ġre un +ĠF ace +ĠHall ow +ĠBud get +Ġcondem n +Ġt ender +Pro f +ocr atic +ĠTurn er +ĠAg ric +Ġ19 76 +Ġa pt +d isc +ĠF ighter +ĠA ur +Ġgar bage +in put +ĠK arl +ĠOl iver +ĠL anguage +k n +N on +ĠCl ar +Ġtrad itions +Ġad vertisement +ĠS or +Ġarch ive +Ġvill ages +7 50 +Ġimplement ing +w aukee +Ġdiet ary +Ġswitch ing +Rep ublic +Ġvel ocity +Ġc it +ĠA wards +Ġfin ancing +Ġlast ed +) ] +Ġrem inder +P erson +Ġprec ision +Ġdesign ers +ĠF ried +ĠB order +Ġtr agic +Ġw ield +Ġiniti atives +ĠT ank +w er +Ġjo ins +R o +in ery +Ġar row +Ġgener ating +found er +Ġsear ches +Ġrandom ly +A ccess +Ġb atch +Ġp osed +l at +Ġpursu ing +as a +Ġtest ified +form ing +ĠSh ar +w iki +ĠE ither +S ometimes +Ġsen ators +ĠJohn ny +ĠTal iban +ĠG PS +":" / +ãģ® å +Ġanaly zed +ĠRub io +ĠMove ment +op ard +ii i +St and +f ight +Ġign oring +i ang +ĠG N +so ever +ĠST AT +Ġref using +Ġswe at +Ġb ay +P ORT +ir med +ak y +Ġdis pro +Ġlabel ed +Ġ10 8 +H ello +Ġple asant +ab a +Ġtri umph +Ġab oard +Ġinc om +ĠC row +le tt +Ġfol k +Ġch ase +` ` +ĠBr us +Ġte ens +c ue +Ġter rain +h yd +il ight +OR Y +Su pport +ew s +ll i +rain ts +ĠC and +Ġab used +ach ment +l arg +B as +ĠC ancer +Ġ19 78 +Ġsupp orter +ac cess +ĠTer min +ĠT ampa +ĠAN Y +Ġnew est +ĠCrim inal +ed u +Ġ19 30 +Ġadm its +Ġend e +Ġfail ures +ur ate +ful ness +cy cl +ĠSub ject +Ġinf inite +th ree +W A +p it +ĠInst all +R ad +ili ation +G M +Ġcontin ent +Ġaccommod ate +ĠCl ay +Ġp up +ĠF unction +Ġham mer +ĠAlbert a +Ġrev ised +Ġminor ities +Ġmeasure ment +Con nell +Ġdis able +ĠM ix +In cre +Ġfor k +ĠR osen +Ġimpl ies +umb lr +AN G +Ġprote ins +Ġagg ression +Ġfacilit ate +S N +Ġilleg ally +u er +Ġacad em +Ġp uzz +ĠSh ift +p ay +oll o +Ġaud iences +B uild +Ġno ble +Ġsynt ax +â ĺħ +Ġbe am +ĠB ed +ĠA ld +Ġorig ins +v ideo +Ġ19 77 +ĠAss ault +Ġgar age +Te am +Ġver dict +Ġd war +ĠVirt ual +e vent +Ke ep +Ġsent iment +Ġwild life +sh irt +Ġb urg +Ġrecommend ation +rep resent +Ġgall ery +own ers +Ġsch olar +Ġconven ience +ĠSw ift +Ġconv inc +C ap +Ġwar fare +ĠVis ual +Ġconst itute +Ġab ort +ĠWe ather +ĠLook ing +ĠH em +Ġmart ial +Ġinc oming +et ition +Ġtoler ance +ĠCre ated +Ġfl ows +ĠE lder +Ġsoul s +Ġf oul +ĠP ain +ĠC AN +Ġ2 20 +b c +he nd +Ġgen ius +R eal +ĠW r +omet er +p ad +Ġlim iting +ĠS i +ĠL ore +ĠAd ventures +Ġvar ied +D isc +f in +ĠPerson al +Ch ris +Ġinv ented +Ġd ive +ĠR ise +Ġo z +ĠCom ics +Ġexp ose +ĠRe b +let ters +s ite +im ated +Ġh acking +Ġeduc ated +ĠNob ody +Ġdep ri +Ġincent ive +ãĤ · +Ġovers ight +Ġtrib es +ĠBelg ium +Ġlicens ing +our t +Produ ct +ah l +ĠG em +Ġspecial ist +Ġc ra +ann ers +ĠCor byn +Ġ19 73 +RE AD +Ġsum mar +Ġover look +ĠApp lication +Ġin appropriate +Ġdownload ed +Q ue +ĠB ears +Ġth umb +ĠChar acter +ĠReincarn ated +ĠS id +Ġdemonstr ates +s ky +ĠBloom berg +ĠAr ray +ĠRes ults +ĠFour th +ĠED T +ĠO scar +c end +Ġ10 6 +ĠN ULL +ĠH ERE +m atch +ĠBr un +Ġgluc ose +ie g +eg u +Ġcert ified +Ġrel ie +Ġhuman itarian +Ġpr ayers +K ing +Ġn an +h ou +10 8 +ul u +Ġrenew able +Ġdistingu ish +Ġd ense +ĠV ent +ĠPack age +ĠB oss +Ġedit ors +Ġm igr +T ra +ĠPet ers +ĠAr ctic +200 4 +ĠC ape +Ġloc ally +Ġlast ing +Ġhand y +. ). +P an +ĠR ES +Ind ex +Ġt ensions +Ġformer ly +Ġide ological +Ġsens ors +Ġdeal ers +Ġdef ines +S k +Ġproceed s +Ġpro xy +az ines +ĠB ash +ĠP ad +ĠC raft +eal ous +Ġshe ets +omet ry +J une +cl ock +T T +ĠThe atre +ĠB uzz +Ġch apters +Ġmill enn +Ġd ough +ĠCongress ional +Ġimag ined +av ior +Ġclin ic +Ġ19 45 +Ġhold er +ro ot +oles ter +Ġrest art +B N +ĠHam as +ĠJ ob +Ġor b +Ġr am +Ġdiscl ose +Ġtransl ate +Ġimm igrant +Ġannoy ing +Ġtreat y +an ium +ĠTe a +ĠLeg ion +Ġcrowd s +ĠB ec +ĠA er +oh yd +B ro +Look ing +Ġl bs +Ġagg ress +Ġse am +Ġinter cept +ĠM I +mer cial +act iv +ĠC it +Ġdim ension +Ġconsist ency +Ġr ushing +ĠDou glas +Ġtr im +Inst all +ick er +Ġsh y +10 6 +Ġment ions +pe lled +ĠT ak +c ost +Ġclass room +Ġfort une +dri ven +Ġun le +ĠWhe el +Ġinvest or +ĠM asters +k it +Ġassoci ations +ĠEv olution +op ing +us cript +Ġprov incial +ĠWal ter +av i +S O +Ġun limited +Eng lish +ĠC ards +ĠEb ola +ne red +Ġreven ge +Ġout right +um per +Ġf itting +ĠSol id +Ġform ally +Ġproblem atic +Ġhaz ard +Ġenc ryption +Ġstraight forward +ĠA K +Ġp se +ĠOr b +ĠCh amber +ĠM ak +Cont ents +Ġloyal ty +Ġl yrics +ĠSy m +Ġwel comed +Ġcook ed +Ġmon op +Ġn urse +Ġmis leading +Ġe ternal +Ġshif ting +Ġ+ = +V is +Ġinst itutional +ill ary +Ġp ant +VER T +ĠA CC +ĠEn h +Ġinc on +ĠRE UTERS +Ġdon ated +âĢ¦âĢ¦ âĢ¦âĢ¦ +In tern +Ġexhib it +Ġt ire +ĠR ic +ĠCh ampion +ĠMu hammad +N ING +ĠSoc cer +Ġmob ility +Ġvary ing +ĠM ovie +Ġl ord +o ak +F ield +Ġve ctor +us ions +Ġsc rap +Ġen abling +m ake +T or +. * +| | +ĠWe bsite +ĠN PC +Ġsocial ist +ĠBill y +ĠAdd itional +Ġc argo +Ġfar ms +ĠSo on +ĠPri ze +Ġmid night +Ġ9 00 +se en +ĠSp ot +Ġshe ep +Ġspons ored +ĠH i +ĠJ ump +Ġ19 67 +Micro soft +ĠAg ent +Ġch arts +d ir +Ġadj acent +Ġtr icks +Ġman ga +Ġex agger +/ > +foot ball +ĠF CC +G C +ĠT ier +and ra +OU ND +% ), +Ġfru its +V C +ĠA A +R ober +Ġmid st +â Ĺ +ank a +Ġlegisl ature +ĠNe il +Ġtour ists +" " +ĠWar ning +ĠNever theless +ĠOffic ial +ĠWh atever +Ġm old +Ġdraft ed +Ġsubst ances +Ġbre ed +Ġt ags +ĠT ask +Ġver b +Ġmanufact ured +com ments +ĠPol ish +Pro v +Ġdetermin es +Ob ama +k ers +Ġutter ly +Ġse ct +sc he +ĠG ates +ĠCh ap +Ġal uminum +Ġz ombie +ĠT ouch +ĠU P +Ġsatisf y +Ġpred omin +asc ript +Ġelabor ate +Ġ19 68 +Ġmeas uring +ĠV ari +any ahu +Ġs ir +ul ates +id ges +ick ets +ĠSp encer +T M +oub ted +Ġpre y +Ġinstall ing +ĠC ab +re ed +re ated +Su pp +Ġwr ist +ĠK erry +10 7 +ĠK le +ĠR achel +Ġc otton +ĠA RE +ĠE le +Cont rol +Ġload s +ĠD od +an as +b one +Ġclass ical +ĠReg ional +ĠInt eg +V M +Ġdes ires +Ġaut ism +support ed +ĠM essage +Ġcomp act +writ er +Ġ10 9 +ĠHur ricane +c ision +Ġcy cles +Ġdr ill +Ġcolle ague +Ġm aker +G erman +Ġmist aken +S un +ĠG ay +Ġwhat soever +Ġsell s +ĠA irl +l iv +ĠO ption +Ġsol ved +Ġse ctors +Ġhorizont al +Ġequ ation +ĠSk ill +ĠB io +g ement +ĠSn ap +ĠLeg al +Ġtradem ark +Ġmake up +Ġassemb led +Ġsa ves +ĠHallow een +ĠVer mont +ĠFR OM +Ġfar ming +ĠP odcast +accept able +ĠHig her +Ġas leep +ull ivan +Ġrefere n +ĠLe v +Ġbul lets +ok o +H C +Ġst airs +Ġmain tains +ĠL ower +ĠV i +Ġmar ine +Ġac res +Ġcoordin ator +ĠJ oh +Ġcounterpart s +ĠBrother s +Ġind ict +b ra +Ġch unk +Ġc ents +H ome +ĠMon th +Ġaccording ly +if les +ĠGerm ans +ĠSy n +H ub +Ġey eb +âĶĢâĶĢ âĶĢâĶĢ +Ġr anges +ĠHoll and +ĠRob ot +f c +M ike +Ġpl asma +Ġsw ap +Ġath lete +ĠR ams +,' " +Ġinfect ions +Ġcor rid +Ġv ib +Ġpat ches +Ġtradition ally +Ġrevel ation +Ġswe ep +Ġgl ance +Ġin ex +200 3 +ĠR aw +work ing +os ures +ĠD at +ĠLyn ch +Ġle verage +ĠRe id +Ġcorrel ation +ian ces +av ascript +Ġrep ository +ret ty +Ġ19 72 +24 0 +Ġo un +p ol +ĠRe ed +Ġtact ical +is ite +App le +ĠQu inn +Ġrap ed +ill o +Euro pe +Ġalgorith ms +ĠRod rig +i u +Ġill um +Ġf ame +Ġintrodu cing +Ġdel ays +ĠRaid ers +Ġwh istle +Ġnovel s +ĠRe ally +Ġder iv +Ġpublic ations +ĠNe ither +ĠCom merce +Ġa ston +l anguage +Not es +ĠR oth +ĠF ear +Ġm ate +Ġpar ade +ĠQ B +Ġman eu +ĠC incinnati +m itting +Ġwa ist +ĠR ew +Ġdisc ont +Ð ° +Ġst aring +Ġal ias +Ġsec urities +Ġtoile t +ĠJ edi +Ġun law +v ised +//// //// +] ( +ĠWe iss +Ġpre st +ĠComp an +Ġmem o +ĠGr ace +J uly +ĠEl ite +cent er +ĠSt ay +Ġgal axy +Ġto oth +ĠS ettings +Ġsubject ed +ãĤ ¦ +Ġline back +Ġretail ers +ĠW ant +Ġd angers +A ir +Ġvolunt ary +ew ay +Ġinterpret ed +ot ine +à § +Ġp el +Serv ice +ĠEvent ually +Ġcare ers +Ġthreat en +Ġmem or +ĠBrad ley +anc ies +s n +ĠUn known +N ational +Ġsh adows +ail and +ĠD ash +Every one +izz ard +M arch += ( +Ġpull s +Ġstr anger +Ġback wards +ĠBern ard +imens ional +Ġch ron +Ġtheoret ical +k top +Ġw are +ĠInvest ig +ĠIn iti +ĠOper ations +o ven +oc ide +* / +Ġfl ames +ĠC ash +sh it +Ġc ab +ĠAn aly +ĠSe ah +Ġdefin ing +Ġorder ing +Ġimm un +Ġpers istent +AC H +Russ ian +m ans +Ġh ind +Ġphot ography + © +Ġh ug +Ġ10 7 +ĠH ence +i ots +ude au +Ġsubsid ies +Ġroutine ly +ĠDev ice +it ic +Ġdisg ust +land er +Ġ19 40 +Ġassign ment +ĠB esides +w ick +ĠD ust +us c +struct ed +11 1 +de velop +Ġf ond +Ġinter section +Ġdign ity +Ġcommission er +With out +re ach +Ġcart oon +Ġsc ales +ãĥ Ń +F IG +Ġsurve ys +ĠIndones ia +Ġart work +Ġun ch +Ġcy cling +un ct +au er +or ate +ĠOb viously +Ġcharacter ized +fe ld +Ġaff irm +Ġinn ings +Ġ é +Ġal iens +Ġcl oth +et ooth +ĠC ertain + § +Ġdig est +k now +ĠX L +Ġpredict ions +Ġd in +W AR +Ġafter math +Ex ample +ĠSu ccess +ĠTh r +IG N +Ġmin er +B us +Ġcl arity +heim er +ĠO UT +ĠS end +ĠCirc le +ĠD iet +Ġpron ounced +Ġcreat ors +Ġearthqu ake +atter y +ge ons +Ġo d +Ġlay ing +or p +U lt +pro ject +Ġunder min +Ġsequ el +S am +ĠDark ness +Ġre ception +b ull +Y S +ĠV ir +Ġsequ ences +ĠCo in +Ġout fit +ĠW ait +1 19 +Ġdel ivers +.... .. +Ġbl own +ĠE sc +ĠM ath +per m +ĠU l +Ġgl im +Ġfac ial +Ġgreen house +Ġto kens +/ - +ĠAnn ual +ĠON E +Ġteen age +ĠPhys ical +ĠL ang +ĠC elt +Ġsu ed +ivid ually +Ġpat ience +ch air +reg ular +Ġa ug +in v +ex cept +ĠL il +Ġn est +f d +s um +ĠCh ase +Russ ia +ĠJenn ifer +Ġoff season +Over all +F ore +Ġr iot +A ud +form er +Ġdefend ers +ĠC T +iot ic +rib ly +Ġautom ated +Ġpen is +Ġins ist +Ġdi agram +ĠS QL +ĠG arc +Ġw itch +cl ient +ier ra +am bers +Ġrec ount +f ar +V ery +oster one +Ġappreci ated +ĠPer fect +S ection +Ġd oses +oca ust +Ġcost ly +Ġg rams +ĠSh i +Ġwrest ling +Ġ19 71 +Ġtro phy +Ġn erve +ĠK az +ĠExper ience +Ġpled ged +Ġplay back +Ġcreat ivity +by e +Ġattack ers +Ġhold ers +ĠCo ach +ĠPh D +Ġtransf ers +Ġcol ored +ĠH indu +Ġd rown +Ġlist ened +ĠW A +ias m +P O +Ġappeal ing +Ġdiscl osed +ĠCh icken +ag ging +Ġple aded +Ġnav igation +ĠReturn s +Ġ[ [ +R OR +E A +Ġphotograp her +ĠR ider +ipp ers +Ġsl ice +Ġe rect +Ġhe d +iss ance +ĠVik ings +ur ious +Ġapp et +oubted ly +Ch ild +Ġauthent ic +o os +ĠM aking +Ġannoun cing +Ġb od +Ġmet er +ĠN ine +ĠR ogue +Ġwork force +Ġrenew ed +Ġorganis ations +ac s +P LE +Sh ort +Ġcomp ounds +ĠVis it +Ġen velop +ear th +Ġsupport ive +gg le +ĠBrus sels +ĠGu ild +Cre ate +RE L +Ġaver aged +Ġ19 69 +ri ages +Ġlength y +Ġforg ot +O kay +ĠE rd +Ġdeal er +Ġrec ession +D D +Ġdesper ately +Ġhun ger +Ġst icks +Ġm ph +ĠF aith +Ġintention ally +Ġdem ol +ue ller +ĠS ale +Ġde bris +s pring +Ġle ap +>> >> +Ġcontain ers +se lling +rane an +atter ing +Ġcomment ed +ĠC M +on ut +Ġwood s +es pecially +Ġorgan ize +iv ic +ĠWood s +ang a +s qu +Ġm aj +am on +Ġax is +Ġ19 74 +ĠDen mark +Ġwar rior +ĠP and +Ġout lined +ĠB O +ins ula +z illa +eb ook +Ġd are +Ġsear ched +Ġnav igate +S n +writ ing +Ġun ited +J apan +ĠHe brew +Ġfl ame +Ġrel ies +Ġcatch ing +ĠSh o +Ġimprison ment +Ġp ockets +Ġclos ure +ĠF am +t im +ade qu +Act ivity +Ġrecru iting +ĠW ATCH +ĠArgent ina +d est +Ġapolog ize +or o +Ġlack s +Ġtun ed +ĠGriff in +Ġinf amous +Ġcelebr ity +ss on +Ġ ---------------------------------------------------------------- +ĠIs is +ĠDis play +Ġcred ibility +Ġeconom ies +Ġhead line +ĠCow boys +Ġind ef +Ġl ately +Ġincent ives +but ton +ĠM ob +A ut +Ġres igned +ĠO m +c amp +Ġprof iles +Ġsche mes +olph ins +ay ed +Cl inton +en h +ĠY ahoo +Ġab st +Ġan k +su its +Ġw ished +ĠMar co +udd en +Ġsp here +ĠB ishop +Ġincorpor ated +ĠPl ant +11 4 +Ġh ated +p ic +Ġdon ate +Ġl ined +Ġbe ans +Ġsteal ing +Ġcost ume +Ġsher iff +Ġfor ty +Ġint act +Ġadapt ed +Ġtrave lling +b art +Ġnice ly +Ġdri ed +Ġsc al +os ity +NOT E +ĠB h +ĠBron cos +ĠI gn +Ġint imate +Ġchem istry +Ġopt imal +D eb +ĠGener ation +Ġ] , +ich i +ĠW ii +ĠYOU R +vent ions +W rite +Ġpop ul +un ning +ĠW or +V ol +Ġqu een +head s +K K +Ġanaly ze +op ic +ear chers +Ġd ot +leg raph +ast ically +Ġupgr ades +Ġca res +Ġext ending +Ġfree ze +Ġin ability +Ġorg ans +Ġpret end +Ġout let +11 3 +ol an +ĠM all +ul ing +t alk +Ġexpress ing +ĠAl ways +ĠBe gin +f iles +Ġlic enses +% % +ĠM itt +Ġfil ters +ĠMil waukee +G N +Ġunf old +M o +Ġnut rition +pp o +B o +Ġfound ing +Ġunder mine +Ġeas iest +ĠC zech +ĠM ack +Ġsexual ity +ĠN ixon +W in +ĠAr n +ĠK in +ãĤ £ +ic er +Ġfort un +Ġsurf aces +agh d +Ġcar riers +ĠP ART +ĠT ib +Ġinter val +Ġfrust rating +ĠSh ip +ĠAr med +ff e +Ġbo ats +ĠAb raham +in is +Ġsu ited +th read +i ov +ab ul +ĠVenezuel a +Ġto m +su per +Ġcast le +alth ough +iox ide +ec hes +Ġevolution ary +Ġnegoti ate +Ġconfront ed +Rem ember +Ġ17 0 +S uch +Ġ9 11 +m ult +ĠA byss +ur ry +ke es +spe c +ĠBarb ara +Ġbelong ing +Ġvill ain +ist ani +Ġaccount able +Ġport ions +ĠDe cl +U r +ĠK ate +g re +Ġmag azines +UC K +Ġregul ate +om on +ĠAl most +Ġover view +Ġsc ram +Ġl oot +ĠF itz +Ġcharacter istic +ĠSn ake +s ay +ĠR ico +Ġtra it +ĠJo ined +au cus +Ġadapt ation +ĠAirl ines +Ġarch ae +ĠI de +Ġb ikes +Ġliter ary +Ġinflu ences +ĠUs ed +C reat +Ġple a +ĠDef ence +ĠAss ass +Ġp ond +UL T +) " +Ġeval uated +Ġob taining +Ġdem ographic +Ġvig il +ale y +Ġsp ouse +ĠSeah awks +resp ons +ĠB elt +um atic +Ġr ises +run ner +ĠMichel le +Ġpot ent +r ace +ĠP AC +F ind +olester ol +IS S +ĠIntrodu ced +ress es +ign ment +O s +ĠT u +ĠDe x +ic ides +Ġspark ed +ĠLaur a +ĠBry ant +Ġsm iling +ĠNex us +Ġdefend ants +ĠCat al +Ġdis hes +sh aped +Ġpro long +m t +( $ +ãĢ Ĥ +Ġcalcul ations +ĠS ame +Ġp iv +H H +Ġcance lled +Ġgr in +Ġterrit ories +ist ically +C ome +ĠP arent +Pro ject +Ġneg lig +ĠPriv acy +Ġam mo +LE CT +olute ly +ĠEp ic +Ġmis under +w al +Apr il +m os +path y +ĠC arson +Ġalbum s +ĠE asy +Ġpist ol +< < +Ġ\ ( +t arget +hel p +Ġinter pre +cons cious +ĠH ousing +ĠJ oint +12 7 +Ġbe ers +s cience +ĠFire fox +effect ive +ĠC abin +ĠO kay +ĠApp lic +Ġspace craft +ĠS R +ve t +ĠStr ange +S B +Ġcor ps +iber al +e fficient +Ġpreval ence +Ġeconom ists +11 8 +Th read +ord able +OD E +ĠC ant +=- =- +if iable +ĠA round +Ġpo le +Ġwilling ness +CL A +ĠK id +Ġcomple ment +Ġsc attered +Ġin mates +Ġble eding +e very +Ġque ue +ĠTr ain +Ġh ij +Ġme lee +ple ted +Ġdig it +Ġg em +offic ial +Ġlif ting +Ð µ +Re qu +it utes +Ġpack aging +ĠWork ers +h ran +ĠLeban on +ol esc +Ġpun ished +ĠJ uan +Ġj am +ĠD ocument +Ġm apping +ic ates +Ġinev itably +Ġvan illa +ĠT on +Ġwat ches +Ġle agues +Ġiniti ated +deg ree +port ion +Ġrec alls +Ġru in +Ġm elt +I AN +Ġhe m +Ex p +Ġb aking +ĠCol omb +at ible +Ġrad ius +pl ug +ĠI F +et ically +Ġf ict +H ER +ĠT ap +atin um +Ġin k +Ġco h +ĠW izard +b oth +te x +Ġsp ends +ĠCurrent ly +ĠP it +Ġneur ons +ig nt +Ġr all +Ġbus es +b uilding +Ġadjust ments +Ġc ried +ibl ical +att ed +ĠZ ion +ĠM atter +Ġmed itation +ĠD ennis +Ġour s +ĠT ab +Ġrank ings +ort al +Ġad vers +Ġsur render +ĠG ob +ci um +om as +im eter +Ġmulti player +Ġhero in +Ġoptim istic +Ġindic ator +ĠBr ig +Ġgro cery +Ġapplic ant +ĠRock et +v id +Ex ception +p ent +Ġorgan izing +Ġenc ounters +ĠT OD +Ġjew el +S ave +ĠChrist ie +Ġhe ating +Ġl azy +ĠC P +Ġcous in +Con fig +Ġreg ener +Ġne arest +Ġachie ving +EN S +th row +ĠRich mond +ant le +200 2 +Ġan ten +b ird +13 3 +Ġn arc +r aint +un ny +ĠHispan ic +ourn aments +Ġprop he +ĠTh ailand +ĠT i +Ġinject ion +Ġinher it +rav is +Ġmed i +Ġwho ever +ĠDE BUG +G P +ĠH ud +C ard +p rom +Ġp or +Ġover head +L aw +Ġviol ate +Ġhe ated +Ġdescript ions +Ġachieve ments +ĠBe er +ĠQu ant +W as +Ġe ighth +ĠI v +Ġspecial ized +U PDATE +ĠD elta +P op +J ul +ĠAs k +oph y +Ġnews letters +ĠT ool +Ġg ard +ĠConf eder +ĠGM T +ĠAb bott +Ġimm unity +ĠV M +Is lam +Ġimpl icit +w d +Ġ19 44 +rav ity +omet ric +Ġsurv iving +ur ai +ĠPr ison +Ġr ust +ĠSk etch +Ġbe es +ĠThe ory +Ġmer it +T ex +ch at +Ġm im +Ġpast e +ĠK och +Ġignor ance +ĠSh oot +Ġbas ement +Un ited +ĠAd vis +he ight +Ġf oster +Ġdet ain +in formation +Ġne ural +' ; +Ġprov es +all ery +Ġinv itation +um bers +Ġc attle +Ġbicy cle +z i +Ġconsult ant +Ġap ology +ĠT iger +Ġ12 3 +99 9 +Ġind ividually +r t +ig ion +ĠBrazil ian +Ġdist urb +Ġentreprene urs +Ġfore sts +cer pt +pl ates +p her +clip se +Ġtw itter +Ġac ids +ograph ical +h um +ĠB ald +if ully +Ġcomp iler +ĠD A +Ġdon or +as i +Ġtrib al +l ash +ĠCon fig +Ġapplic ants +Ġsal aries +13 5 +Put in +ĠF ocus +ir s +Ġmisc onduct +ĠH az +Ġeat en +M obile +Mus lim +ĠMar cus +v iol +Ġfavor able +Ġst ub +ad in +ĠH ob +Ġfaith ful +Ġelectron ics +Ġvac uum +w ait +back ed +econom ic +d ist +Ġten ure +Ġsince re +ĠT ogether +ĠW ave +Ġprog ression +Ġden ying +Ġdist ress +br aska +th ird +Ġmix ing +Ġcolon ial +Ġpriv ately +Ġun rest +atern ity +Ġprem ises +ant i +greg ation +Ġlic ence +ĠH ind +ĠSam uel +Ġconvinc ing +ĠA ce +ĠR ust +ĠNet anyahu +Ġhand les +ĠP atch +orient ed +ah o +ĠG onz +Ġhack ers +claim er +Ġcustom s +ĠGr an +f ighters +Ġl uc +Ġman uscript +aren thood +Ġdev il +Ġwar riors +Ġoff enders +Will iam +Ġhol idays +Ġnight mare +Ġle ver +iff erent +St at +Ġexhib ition +put ed +ĠP ure +Ġal pha +Ġenthus iasm +ĠRepresent atives +E AR +ĠT yp +Ġwhe at +ĠAl f +Ġcor rection +Ġev angel +AT T +M iss +Ġs oup +Ġimpl ied +par am +Ġsex y +ĠL ux +Ġrep ublic +p atch +ab lish +Ġic ons +Ġfather s +ĠG ET +ĠCar ib +Ġregul ated +ĠCo hen +ĠBob by +Ġn er +Ġb ent +vent ory +ĠAl ong +ĠE ST +ĠWall ace +Ġmurd ers +r ise +ke ll +ĠCommon wealth +Ġn asty +et a +ĠM IT +Ġadminist ered +Ġgenuine ly +Ed itor +n ick +Ġhyd ro +**************** **************** +ĠB le +Ġfin es +Ġg orge +aus ible +r h +Ġapp le +ment ioned +Ġro pe +ot yp +H R +Ġdisappoint ing +Ġc age +n ik +Ġdoub ts +ĠF REE +print s +ĠM UST +Ġvend ors +ĠIn qu +Ġliber als +Ġcontract or +Ġup side +child ren +Ġtrick y +Ġregul ators +charg ed +l iter +Ġ *** +Ġreb ell +l ang +Ġloc als +Ġphys icians +Ġhe y +ar se +t m +ĠLe x +Ġbehavior al +success ful +F X +Ġbr ick +ov ic +Ġcon form +Ġreview ing +Ġins ights +Ġbi ology +ĠRem ove +ĠExt ra +Ġcomm itting +indu ced +ignt y +ig m +Ġat omic +Comm on +ĠE M +ĠP ere +ĠIt ems +e h +Ġpres erved +ĠH ood +Ġprison er +Ġbankrupt cy +Ġg ren +us hes +Ġexplo itation +Ġsign atures +Ġfin an +] ," +ĠM R +Ġme g +rem lin +Ġmusic ians +Ġselect ing +Ġexam ining +IN K +l ated +H i +Ġart ic +Ġp ets +Ġimp air +ĠM AN +Ġtable ts +in clude +R ange +Ġca ut +Ġlog s +Ġmount ing +Ġun aware +Ġdynam ics +ĠPalest ine +ĠQu arter +ĠPur ple +Ġm a +ĠIm port +Ġcollect ions +ci ation +Ġsuccess or +Ġcl one +Ġaim ing +Ġposs essed +Ġstick ing +Ġsh aking +Ġloc ate +ĠH ockey +T urn +17 0 +Ġfif teen +ĠHar rison +Ġcontinu ously +ĠT C +ĠVal ent +ĠRes cue +Ġby pass +am ount +Ġm ast +Ġprotect s +Ġart istic +Ġsomet ime +Ġsh oe +Ġshout ed +ific ant +et itive +ĠReg ister +ĠJ in +Ġconcent rated +ling ton +on ies +Ġgener ator +yr im +ĠAr men +Ġclear ing +id o +ĠT W +al ph +Ġlad ies +H ard +Ġdial og +Ġinput s +æ ľ +Ġpos es +Ġsl ots +ĠPrem ium +Ġle aks +Ġboss es +Ġ11 3 +c ourse +A cc +ĠNew ton +ĠAust ria +ĠM age +Ġte aches +ab ad +Ġwe ars +Ġc yl +Ġcur se +ĠS ales +ĠW ings +Ġp sy +Ġg aps +ĠIce land +ĠP interest +Ġland lord +Ġdefin itions +ĠK er +Ġsufficient ly +ĠP ence +ĠArch itect +Ġsur pass +Ġ11 4 +Ġsuper hero +ĠDise ase +Ġpri ests +ĠC ulture +Ġdefin itive +Ġsecret ly +ĠD ance +inst all +ch ief +ĠJess ica +W ould +Up dated +Ġlock er +ĠK ay +Ġmem orial +è ¦ +f at +Ġdis gu +Ġflav ors +ĠBase ball +ĠRes istance +Ġk icks +Ġen v +Ġteen agers +D ark +ĠC AR +Ġh alt +ĠL G +ĠGab riel +Ġfe ver +Ġs atur +Ġm all +Ġaffili ate +ĠS leep +ĠSpe cific +ĠV el +Ġj ar +ĠSac red +ĠEd wards +ĠA CL +Ġret ained +ĠG iant +Ġlim itation +in ces +Ġref usal +ĠT ale +ĠBut ler +Ġacc idents +ĠC SS +Ġimport ed +ĠCop y +Î ± +ER T +z el +Ġdiv isions +h ots +ĠAl b +ĠD S +Load er +W ashington +at isf +ĠCreat ive +\ . +ĠAut om +red ict +Ġrecept or +ĠCarl os +Met hod +ok a +Ġmal icious +Ġste pping +, [ +ĠD ad +Ġatt raction +ĠEffect s +ĠPir ate +ĠC er +ĠIndust ry +ĠR ud +Ġchar ter +Ġd ining +Ġins ists +Ġconfig ure +Ġ( # +ĠSim ple +ĠSc roll +UT C +17 5 +ĠK on +Ġmarket place +Ġ ãĤ +Ġref res +Ġg ates +er red +ĠP od +Ġbeh ave +Fr ank +n ode +Ġendors ed +he tt +as ive +ĠHom eland +Ġr ides +ĠLe ave +er ness +Ġflood ing +A FP +Ġris en +Ġcontin ually +Ġun anim +ĠCont ract +ĠP as +Ġgu ided +ĠCh ile +b d +Ġsu cc +pt ic +Ġcomm ittees +ĠL uther +ĠAny one +Ġs ab +12 4 +Ġp ixel +ĠB ak +ĠT ag +ĠBenn ett +En ter +sm all +ĠPresident ial +Ġp ul +Ġcontr ace +arch ive +Ġcoast al +ĠK ids +19 2 +âĢ ² +ick y +ING TON +Ġw olf +ĠSt alin +T ur +id get +am as +ĠUn less +Ġspons or +Ġmor ph +ĠCho ose +Ġrun ner +Ġun bel +Ġm ud +ĠMan a +Ġdub bed +Ġg odd +ure rs +wind ow +Ġrel ied +Ġcelebr ating +os c +Ġ13 5 +Ġlobb ying +Ġincom plete +Ġrestrict ion +Ġinc ap +it us +Ġexpect ation +ĠAp ollo +Ġint ens +Ġsyn c +G H +Ġmanip ulation +B Y +Ġspe ar +Ġbre asts +Ġvol can +il ia +M aterial +Ġform ats +ĠB ast +Ġparliament ary +Ġsn ake +Ġserv ants +ĠTr udeau +ĠGr im +ĠArab ic +ĠSC P +ĠBoy s +st ation +Ġprospect ive +ord e +in itialized +Ġb ored +AB LE +Ġaccess ed +Ġtax i +ĠShe ll +aid en +urs ed +in ates +ĠIns urance +ĠPet e +Sept ember +6 50 +Ġad ventures +ĠCo ver +Ġt ribute +Ġsk etch +Ġem power +Ġ Ø +ĠGl enn +ĠD aw += \" +ĠPolit ics +Ġgu ides +Ġd ioxide +ĠG ore +ĠBr ight +ĠS ierra +Ġval ued +c ond +Ġpo inter +Se lect +Ġrisk y +Ġabsor b +im ages +Ġref uses +Ġbon uses +__ _ +Ġh ilar +ĠF eatures +2 20 +ĠCollect or +F oot +Ġ19 64 +cul us +Ġd awn +Ġwork out +ĠL O +Ġphilosoph ical +ĠSand y +ĠYou th +Ġl iable +A f +bl ue +Ġovert urn +less ness +ĠTrib une +ĠIn g +Ġfact ories +Ġcat ches +Ġpr one +Ġmat rix +Ġlog in +Ġin acc +Ġex ert +s ys +Ġneed le +ĠQ ur +Ġnot ified +ould er +t x +Ġremind s +Ġpublisher s +Ġn ort +Ġg it +Ġfl ies +ĠEm ily +Ġflow ing +ĠAl ien +ĠStr ateg +Ġhard est +Ġmod ification +AP I +ĠM Y +Ġcr ashes +st airs +n umber +Ġur ging +ch annel +ĠFal con +Ġinhabit ants +Ġterr ifying +Ġutil ize +Ġban ner +Ġcig arettes +Ġsens es +ĠHol mes +Ġpract ition +ĠPhill ips +ott o +Ġcomp ile +Mod el +ĠK o +Ġ[ ] +Americ ans +ĠTer ms +Ġmed ications +ĠAn a +Ġfundament ally +ĠNot ice +Ġwe aker +Ġ 0000 +Ġgar lic +Ġout break +Ġeconom ist +ĠB irth +Ġobst acles +ar cer +ĠOr thodox +Ġplace bo +ĠC rew +asp berry +ĠAng els +Ġdis charge +Ġdestruct ive +11 7 +ĠR ising +Ġd airy +l ate +Ġcoll ision +ĠTig ers +ean or +ocument ed +ĠIn valid +Ġd ont +ĠL iter +ĠV a +Ġhyd rogen +Ġvari ants +ĠBrown s +Ġ19 65 +Ġind igenous +Ġtrad es +Ġremain der +Ġswe pt +ĠImp act +Ġred ist +Ġun int +grad uate +ãĥ ķ +ĠW ILL +ãģ® ç +ĠCrit ical +Ġf isher +Ġv icious +Ġrevers ed +Y ear +ĠS ox +Ġshoot ings +Ġfil ming +Ġtouchdown s +ai res +m el +Ġgrand father +Ġaffect ion +ing le +Ġover ly +Add itional +Ġsup reme +ĠGr ad +Ġsport ing +Ġmer cy +ĠBrook s +ount y +Ġperform s +Ġtight ly +Ġdem ons +Ġkill ings +Ġfact ion +ĠNov a +aut s +Ġund oubtedly +ar in +Ġunder way +ra k +Ġl iv +ĠReg ion +Ġbrief ing +s ers +cl oud +ĠM ik +us p +Ġpred iction +az or +Ġport able +ĠG and +Ġpresent ing +Ġ10 80 + » +ush i +ĠSp ark +there um +Ġjust ification +ĠN y +Ġcontract ors +ming ham +ĠSt yle +å ħ +ĠChron icles +ĠPict ure +Ġprov ing +Ġw ives +set t +Ġmole cules +ĠFair y +Ġconsist ing +Ġp ier +al one +in ition +Ġn ucle +j son +Ġg otta +Ġmob il +Ġver bal +ar ium +Ġmon ument +uck ed +Ġ25 6 +T ech +mine craft +ĠTr ack +Ġt ile +Ġcompat ibility +as is +Ġs add +Ġinstruct ed +ĠM ueller +Ġle thal +Ġhorm one +Ġor che +el se +Ġske let +Ġentert aining +Ġminim ize +ag ain +Ġunder go +Ġconst raints +Ġcig arette +ĠIslam ist +Ġtravel s +ĠPant hers +l ings +C are +Ġlaw suits +ur as +Ġcry st +Ġlow ered +Ġaer ial +Ġcomb inations +Ġha un +Ġch a +Ġv ine +Ġquant ities +Ġlink ing +b ank +Ġso y +B ill +ĠAngel a +Ġrecip ient +ĠProt est +Ġs ocket +Ġsolid arity +Ġâ Ĩ +m ill +Ġvar ies +ĠPak istani +Dr agon +Ġun e +Ġhor izon +³³³³ ³³³³ +Ġprov inces +Ġfrank ly +Ġenact ed +not es +[ ' +Ġ19 2 +ocr acy +Ġendorse ment +Ġover time +Tr ue +L ab +lic ted +ĠD NC +Ġbe ats +ĠJam ie +15 2 +ĠIN T +Cont act +Ġaccount ed +h ash +ĠPack ers +p ires +Ġles bian +Ġamend ments +Ġhop eful +ĠFin land +Ġspot light +Ġconfig ured +Ġtrou bled +Ġg aze +ĠCal gary +Ġrel iability +Ġins urg +sw er +b uy +ĠSk in +Ġp ixels +Ġhand gun +Ġpar as +Ġcateg or +ĠE L +ĠRe x +Ind eed +Ġkind a +Ġconj unction +ĠBry an +ĠMan ufact +y ang +Pl us +S QL +ish ment +Ġdom inate +Ġn ail +Ġo ath +Ġeru pt +ĠF ine +it bart +ĠCh ip +ĠAb d +ĠN am +Ġbuy er +Ġdiss ent +Le aks +Cont in +Ġr ider +ĠSome one +Ġill usion +c in +ĠBoe ing +Ġin adequ +ov ation +i ants +Ġreb uild +4 50 +ĠDest iny +S W +ĠT ill +H it +ia z +ĠBang l +acher s +ĠRe form +Ġse gments +Ġsystem atic +d c +ĠConserv atives +Ġport al +h or +ĠDragon bound +Ġdrag ged +om o +Ġthe e +ad vert +ĠRep orts +ĠE t +Ġbarrel s +Aug ust +Ġcompar isons +Ġhe x +Ġan throp +" [ +bor ough +ab i +Ġpict ured +play ing +ĠAdd ress +ĠMir ror +Sm ith +Ġt ires +ĠN PR +AA AA +Ġclass ification +ĠTh an +ĠH arm +ĠR A +Ġreject ion +min ation +Ġr anged +ĠF alls +D I +H ost +ãĤ ´ +ĠEx ample +list ed +th irds +Ġsaf egu +br and +Ġprob able +Can ada +IT ION +ĠQ aeda +Ġch ick +Ġimport s +h it +l oc +W W +Ġble w +Ġany time +Ġwh oles +ik ed +Ġcal culation +cre ate +ĠO ri +Ġupgr aded +Ġapp ar +ut ory +ĠM ol +B rit +ĠJ ong +IN AL +ĠStart ing +Ġd ice +urt le +Ġre lying +cl osure +Ġprof itable +Ġsl aughter +ĠMan ual +c aster +Ġ" $ +Ġfe ather +ĠSim ply +ie ves +Ġdeter ior +ĠPC I +Ġst amp +Ġfl aws +Ġsh ade +ham mer +Ġpass port +Ġcont ing +am el +Ġobser vers +Ġneg lect +ĠR B +ĠBrother hood +Ġskept ical +f amily +us k +Ġemotion ally +â Ļ +ĠBet a +ason able +id ity +ĠM ul +Ġkick ing +ĠC arm +oll ah +VERT IS +ĠAt hen +Ġlad der +ĠBul let +å £ +00 01 +ĠWild life +ĠM ask +ĠN an +R ev +Ġun acceptable +leg al +Ġcrowd ed +ag i +ĠC ox +j e +Ġmor ality +Ġfu els +Ġc ables +Ġman kind +ĠCarib bean +Ġanch or +Ġby te +ĠO ften +ĠO z +Ġcraft ed +Ġhistor ian +ĠW u +Ġtow ers +ĠCitiz ens +Ġhel m +Ġcred entials +Ġsing ular +ĠJes se +Ġtack les +Ġcont empt +Ġa fore +ĠSh adows +Ġn il +Ġur gent +app le +bl ood +Ġv on +Ġoff line +Ġbreat he +Ġj umps +Ġirre levant +ox ic +om al +import ant +J im +Ġgl oves +arm ing +dep th +Ġtal ents +ook ie +ĠS B +Ġpal m +uff s +est a +IG H +Ġcan on +ĠVer izon +ĠP le +Ġcou pled +vel t +Ġfundra ising +ĠGet ting +ĠD LC +Ġmathemat ical +ĠH S +ĠCard inals +te lling +Ġspons ors +Ġ Ï +ĠBull s +op tion +Ġprop ose +Ġmem orable +Ġembr aced +Ġdecl ining +He alth +ed a +Ġ} ; +Ġsp am +m ile +Ġpit cher +ĠE ight +Ġcar ing +ut ic +ro le +Ġair line +ernand ez +ĠAth let +Ġcert ification +ux e +rig er +Ġem pir +Ġsens ation +Ġdis m +Ġb olt +Ġev olve +H ouse +Ġconsult ation +ĠD uty +Ġtou ches +ĠN athan +Ġf aint +h ad +" ( +ĠCons umer +ĠExt reme +Ġ12 7 +ĠHer m +ĠSac rament +iz oph +Ġanx ious +ul ously +Ġsoc ially +ĠU TC +Ġsol ving +ĠLet ter +Hist ory +ed uc +Pr ice +) ); +Ġrel oad +am ic +Ġp ork +Ġdisc ourse +Ġt ournaments +ai ro +ĠK ur +ĠCost a +Ġviol ating +Ġinterf ere +Ġrecre ational +uff le +Ġspe eches +Ġneed ing +Ġremem bers +Ġcred ited +n ia +f ocused +amer a +Ġb ru +um bs +ĠCub an +Ġpreced ing +Ġnons ense +ac ial +Ġsmart phones +ĠSt ories +S ports +ĠEmer gency +oun cing +ef ined +Ġb er +Ġconsult ing +Ġm asters +he astern +." [ +ĠRun ning +Ġsus cept +ĠF eng +Americ a +pr ises +st itial +ĠWeek ly +ĠGreat er +mod ules +if ter +G raphics +ul er +Ġwho lly +Ġsupp ress +Ġconce aled +Ġhapp ily +Ġaccept s +ĠEn joy +Ġr ivers +ĠEx cept +2 25 +ĠN HS +ĠMc Connell +Ġp ussy +fer red +ut able +Ġatt ain +Ġ> = +Ġdepos its +roph ic +Ġnot orious +ĠSh aw +il itation +Ġepid emic +all ic +Ġsmall est +ov ich +Ġaccess ories +per ties +Ġsur plus +ĠMe ch +Ġamb ig +ĠImm igration +Ġch im +ev al +Ġpract icing +ĠMyster y +Ġdom ains +ĠSil icon +app s +Ġkilomet ers +e a +ĠSm ash +Ġwarrant y +Ġn ost +s il +re v +J on +ĠDub lin +Ġtast es +Ġb out +g reat +er ror +Ġsw itches +ĠB apt +D O +ok i +Ġsour ced +pro du +Ġattach ment +ĠIss ue +ĠQuest ion +Jo in +Ġf itted +Ġunlaw ful +^ ^ +ere k +Ġauthent ication +Ġst ole +Ġaccount ability +l abel +S earch +Ġal beit +atic an +fund ed +ĠAdd ing +ĠI Q +Ġsub mar +l it +a que +ĠLear ning +Ġint eger +M aster +ĠCh rom +Ġprem ier +O p +ĠLi u +Ġbl essed +ĠGl obe +ĠResp onse +Ġlegit im +ĠMer kel +Ġdispos al + ´ +Ġgau ge +pe at +Ġindu ced +Ġquestion able +arth y +ĠV it +ĠF eed +U ntil +U t +worth y +R Y +ĠH erald +ĠHam mer +Ġmed al +ĠR ivers +ĠH ack +Ġclar ify +Ġtrack ed +Ġautonom ous +Ġten ant +ĠQ atar +er ie +Ġgr im +ĠMon itor +Ġresist ant +ĠSpe c +ĠWell s +N AS +14 8 +Ġmin ers +iot ics +Ġmiss es +11 6 +g ian +g it +ĠE yes +p res +Ġgrad uated +Ġang el +Ġsyn chron +Ġefficient ly +Ġtrans mitted +H arry +Ġglob ally +EN CE +ĠMont ana +r aged +ĠPre vention +Ġp iss +ĠL l +Ġshe lf +ĠB JP +ĠTest ament +ĠL ate +ik er +ĠH app +ĠJul ian +h all +Ġsp ont +Ġshut down +Ġincons istent +Ġsubscrib ers +Ġske leton +ĠNe braska +Ġins pire +ĠV oid +F eed +Ġang les +ĠSpr ings +Ġbench mark +Ġvacc ines +izoph ren +se xual +uff ed +Ġsh ine +ĠK ath +Ġgest ure +ine a +Ġr ip +Ġopp ression +Ġcons cience +b t +ĠL um +Ġinc idence +ĠF a +w r +Ġmin eral +ĠSp urs +alk y +Ġth under +Ġop io +Be ing +ĠPal m +Ġwas ted +Ġl b +i aries +ĠIniti ative +Ġcur ric +Ġmark er +ĠMc L +Ġext ensions +ĠP v +ĠAr ms +Ġoffer ings +Ġdef enses +Ġvend or +Ġcontrad ict +ĠCol in +Ġredd it +Ġper ipher +12 2 +Ġs ins +E dit +IC T +So ft +ĠSh ah +Ġadministr ator +ĠT rip +Ġporn ography +Ġtu ition +in ence +ĠPro gress +Ġcat alog +Ġsu ite +Ġh ike +Ġreprodu ctive +eng ine +Ġd rought +ĠNo ah +Ġ2 30 +Ġd ude +Ġrelax ed +Ġpart ition +Ġparticip ant +Ġtel esc +Ġfe as +ĠF F +own er +Ġswe eping +Ġl enses +Ġmatch up +ĠRe pl +ourn als +Ġcred ible +Ġgrand mother +Ġther mal +Ġsubscrib ing +Ġident ities +col m +U CT +Ġreluct ant +us ers +ĠC ort +Ġassist ed +OS S +ATION S +IS H +Ġpharm aceutical +ic able +ad ian +ĠSon ic +ĠF ury +ĠM ong +A H +ĠPsych ology +Ġph osph +Ġtreat s +Ń Ķ +Ġstead ily +ĠHell o +Ġrel ates +Ġcl ue +Ex pl +a uth +Ġrev ision +Ġe ld +os ion +Ġbr on +14 4 +ri kes +Ġmin es +Ġblank et +ĠF ail +el ed +ĠIm agine +ĠPl anned +a ic +Re quest +M ad +ĠHor se +ĠEag le +Ġcap ac +15 7 +Ġl ing +ĠN ice +ĠP arenthood +min ster +og s +ens itive +Not hing +Ġcar n +F in +ĠP E +Ġr ifles +ĠL P +S and +Ġgui Active +Ġtour ist +C NN +Ġunve iled +Ġpredec essor +} { +u ber +Ġoff shore +Ġopt ical +ĠR ot +ĠPear l +et on +Ġst ared +Ġfart her +at ility +cont in +ĠG y +ĠF oster +ĠC oc +ri ents +Ġdesign ing +ĠEconom y +ON G +W omen +ĠN ancy +er ver +Ġmas cul +Ġcasual ties +Ġ2 25 +ĠS ullivan +ĠCh oice +Ġa ster +w s +Ġhot els +Ġconsider ations +Ġcou ch +ĠSt rip +ĠG n +Ġmanip ulate +l ied +Ġsynt hetic +Ġassault ed +Ġoff enses +ĠDra ke +Ġim pe +Oct ober +ĠHer itage +h l +ĠBl air +Un like +Ġg rief +Ġ4 50 +Ġopt ed +Ġresign ation +il o +Ġver se +ĠT omb +Ġu pt +Ġa ired +ĠH ook +ĠML B +Ġassum es +out ed +ĠV ers +Ġinfer ior +Ġbund le +ĠD NS +ograp her +Ġmult ip +ĠSoul s +Ġillust rated +Ġtact ic +Ġdress ing +Ġdu o +Con f +Ġrel ent +Ġc ant +Ġscar ce +Ġcand y +ĠC F +Ġaffili ated +Ġspr int +yl an +ĠGarc ia +Ġj unk +Pr int +ex ec +C rit +Ġport rait +ir ies +ĠOF F +Ġdisp utes +W R +L ove +ãģ Ħ +ĠRe yn +Ġh ipp +op ath +Ġflo ors +ĠFe el +Ġwor ries +Ġsett lements +ĠP os +Ġmos que +Ġfin als +Ġcr ushed +ĠPro bably +ĠB ot +ĠM ans +ĠPer iod +Ġsovere ignty +Ġsell er +Ġap ost +Ġam ateur +Ġd orm +Ġconsum ing +Ġarm our +ĠRo ose +Ġint ensive +Ġelim inating +ĠSun ni +ĠAle ppo +j in +Ġadv ise +p al +ĠH alo +Ġdes cent +Ġsimpl er +Ġbo oth +ST R +L ater +ĠC ave +== = +Ġm ol +Ġf ist +Ġshot gun +su pp +Ġrob bery +E ffect +Ġobsc ure +ĠProf essional +Ġemb assy +Ġmilit ant +Ġinc arcer +Ġgener ates +Ġlaun ches +Ġadministr ators +Ġsh aft +Ġcirc ular +Ġfresh man +ĠW es +ĠJo el +ĠD rew +ĠDun can +ĠApp arently +s ight +ĠIntern al +ĠInd ividual +ĠF E +Ġb ore +ĠM t +Ġbroad ly +ĠO ptions +ount ain +ip es +ĠV ideos +20 4 +Ġh ills +Ġsim ulation +Ġdisappoint ment +it an +ĠLabor atory +Ġup ward +Ġbound ary +Ġdark er +h art +Ġdomin ance +C ong +ĠOr acle +ĠL ords +Ġscholars hip +ĠVin cent +ed e +ĠR ah +Ġencour ages +ro v +Ġqu o +Ġprem ise +ĠCris is +ĠHol ocaust +Ġrhyth m +Ġmet ric +cl ub +Ġtransport ed +Ġn od +ĠP ist +Ġancest ors +ĠFred er +th umbnails +ĠC E +ON D +Ph il +ven ge +ĠProduct s +cast le +Ġqual ifying +ĠK aren +VERTIS EMENT +Ġmight y +Ġexplan ations +Ġfix ing +D i +Ġdecl aring +Ġanonym ity +Ġju ven +ĠN ord +ĠDo om +ĠAct ually +O k +ph is +ĠDes ert +Ġ11 6 +I K +ĠF M +Ġinc omes +V EL +ok ers +Ġpe cul +Ġlight weight +g ue +Ġacc ent +Ġincre ment +ĠCh an +Ġcompl aining +ĠB aghd +Ġmidfield er +Ġover haul +Pro cess +ĠH ollow +ĠTit ans +Sm all +man uel +ĠUn ity +ĠEv ents +S ty +Ġdispro portion +n esty +en es +ĠC od +Ġdemonstr ations +ĠCrim son +ĠO H +Ġen rolled +Ġc el +ĠBre tt +Ġa ide +Ġhe els +Ġbroad band +Ġmark ing +Ġw izard +ĠN J +ĠChief s +Ġingred ient +Ġd ug +ĠSh ut +urch ase +end or +Ġfar mer +ĠGold man +12 9 +15 5 +Or der +Ġl ion +i ably +Ġst ain +ar ray +ilit ary +ĠFA Q +Ġexpl oded +ĠMcC arthy +ĠT weet +ĠG reens +ek ing +l n +ens en +Ġmotor cycle +Ġpartic le +Ġch olesterol +B ron +Ġst air +Ġox id +Ġdes irable +ib les +Ġthe or +for cing +Ġpromot ional +ov o +b oot +ĠBon us +raw ling +Ġshort age +ĠP sy +Ġrecru ited +Ġinf ants +Ġtest osterone +Ġded uct +Ġdistinct ive +Ġfirm ware +bu ilt +14 5 +Ġexpl ored +Ġfact ions +Ġv ide +Ġtatt oo +Ġfinan cially +Ġfat igue +Ġproceed ing +const itutional +Ġmis er +Ġch airs +gg ing +ipp le +Ġd ent +Ġdis reg +ç Ķ +st ant +ll o +b ps +aken ing +Ġab normal +ĠE RA +å£ « +ĠH BO +ĠM AR +Ġcon cess +Ġserv ant +Ġas pir +l av +ĠPan el +am o +Ġprec ip +Ġrecord ings +Ġproceed ed +Ġcol ony +ĠT ang +ab lo +Ġstri pped +Le ft +to o +Ġpot atoes +Ġfin est +% ). +Ġc rap +ĠZ ach +ab ases +ĠG oth +Ġbillion aire +w olf +Ġsan ction +S K +Ġlog ged +P o +ey ed +un al +Ġcr icket +Ġarm ies +Ġunc overed +Cl oud +ó n +Ġreb ounds +Ġm es +O per +P ac +Ġnation ally +Ġinsert ed +p ict +Ġgovern ance +Ð ¸ +Ġprivile ges +G ET +Ġfavor ites +im ity +Ġlo ver +the m +em pl +Ġgorge ous +An n +Ġsl ipped +Ġve to +B ob +Ġsl im +u cc +ĠF ame +udden ly +Ġden ies +ĠM aur +Ġdist ances +Ġw anna +t ar +ĠS ER +Ġâ Ī +Ġle mon +at hetic +Ġlit eral +Ġdistingu ished +Ġansw ering +G I +Ġrelig ions +ĠPhil os +ĠL ay +Ġcomp os +ire ments +ĠK os +ine z +roll ing +Ġyoung est +and ise +ĠB orn +Ġalt ar +am ina +ĠB oot +v oc +Ġdig ging +Ġpress ures +Ġl en +26 4 +Ġassass ination +ĠBir mingham +ĠMy th +Ġsovere ign +ĠArt ist +ĠPhot ograph +Ġdep icted +Ġdisp ens +orth y +Ġamb ul +int eg +ĠC ele +ĠTib et +Ġhier archy +Ġc u +Ġpre season +ĠPet erson +Ġcol ours +Ġworry ing +Ġback ers +ĠPal mer +ĠÎ ¼ +Ġcontribut or +Ġhear ings +Ġur ine +Ġ Ù +ourge ois +Sim ilar +ĠZ immer +s omething +ĠUS C +Ġstrength s +ĠF I +Ġlog ging +As ked +ĠTh ai +in qu +ĠW alt +Ġcrew s +it ism +3 01 +Ġshar ply +um ed +Ġred irect +r ators +In f +ĠWe apons +Ġte asp +19 99 +L ive +ĠEs pecially +ĠS ter +ĠVeter ans +Ġint ro +other apy +Ġmal ware +Ġbre eding +Ġmole cular +ĠR oute +ĠCom ment +oc hem +Ġa in +Se ason +Ġlineback er +Ä « +ĠEconom ics +es ar +ĠL ives +ĠEm ma +Ġk in +ĠTer rit +Ġpl anted +ot on +ĠBut ter +ĠSp ons +P ER +Ġdun geon +Ġsymb olic +Ġfil med +Ġdi ets +Ġconclud es +Ġcertain ty +ĠForm at +Ġstr angers +form at +ĠPh ase +Ġcop ied +Ġmet res +ld a +ĠUs ers +Ġdeliber ate +Ġwas hed +ĠL ance +im ation +Ġimpro per +ĠGen esis +ick r +ĠK ush +Ġreal ise +Ġembarrass ing +alk ing +b ucks +Ġver ified +Ġout line +year s +ĠIn come +20 2 +Ġz ombies +F inal +ĠMill enn +Ġmod ifications +ĠV ision +ĠM oses +ver b +iter ranean +ĠJ et +Ġnav al +ĠA gg +Ġur l +Ġvict ories +Ġnon etheless +Ġinj ust +ĠF act +ç ļ +Ġins ufficient +re view +face book +Ġnegoti ating +Ġguarant ees +im en +uten berg +Ġg ambling +Ġcon gr +Load ing +Ġnever theless +Ġpres idents +ĠIndust rial +Ġ11 8 +Ġp oured +ĠT ory +Ġ17 5 +Ġ: = +Sc ott +ange red +T ok +Ġorgan izers +M at +ĠG rowth +Ġad ul +Ġens ures +Ġ11 7 +é¾į å +Ġmass acre +Ġgr ades +be fore +AD VERTISEMENT +ĠSl ow +ĠM MA +âĢĶ " +ĠV atican +Q aeda +Ġo we +66 66 +ĠS orry +ĠGr ass +Ġbackground s +Ġexha usted +Ġcl an +Ġcomprom ised +ĠE lf +ĠIsa ac +ens on +In vest +IF A +Ġinterrupt ed +ãĥī ãĥ© +Ġtw isted +ĠDrag ons +M ode +ĠK remlin +Ġfert il +he res +ph an +ĠN ode +f ed +ĠOr c +Ġunw illing +C ent +Ġprior it +Ġgrad uates +Ġsubject ive +Ġiss uing +ĠL t +Ġview er +Ġw oke +Th us +bro ok +Ġdep ressed +Ġbr acket +ĠG or +ĠFight ing +Ġstri ker +Rep ort +ĠPortug al +Ġne o +w ed +19 9 +Ġflee ing +sh adow +ident ified +US E +Ste am +Ġstret ched +Ġrevel ations +art ed +ĠD w +Ġalign ment +est on +ĠJ ared +S ep +Ġblog s +up date +g om +r isk +Ġcl ash +ĠH our +Ġrun time +Ġunw anted +Ġsc am +Ġr ack +Ġen light +on est +ĠF err +Ġconv ictions +Ġp iano +Ġcirc ulation +ĠW elcome +Ġback lash +ĠW ade +Ġrece ivers +ot ive +J eff +Ġnetwork ing +ĠPre p +ĠExpl orer +Ġlect ure +Ġupload ed +ĠMe at +B LE +ĠNaz is +ĠSy nd +st ud +ro ots +ri ans +Ġportray ed +Ġ ?? +ĠBudd ha +s un +Rober t +ĠCom plex +Ġover see +Ġste alth +T itle +ĠJ obs +ĠK um +Ġappreci ation +ĠM OD +Ġbas ics +Ġcl ips +Ġnurs ing +Ġpropos ition +Ġreal ised +ĠNY C +Ġall ocated +ri um +ar an +ĠPro duction +ĠV ote +Ġsm ugg +Ġhun ter +az er +ĠCh anges +Ġfl uct +y on +Ar ray +Ġk its +W ater +Ġuncom mon +Ġrest ing +ell s +w ould +Ġpurs ued +Ġassert ion +omet own +ĠMos ul +ĠPl atform +io let +Ġshare holders +Ġtra ils +P ay +ĠEn forcement +ty pes +ĠAn onymous +Ġsatisf ying +il ogy +Ġ( ' +w ave +c ity +Ste ve +Ġconfront ation +ĠE ld +C apt +ah an +ht m +ĠC trl +ON S +2 30 +if a +hold ing +Ġdelic ate +Ġj aw +ĠGo ing +or um +S al +Ġd ull +ĠB eth +Ġpr isons +Ġe go +ĠEl sa +avor ite +ĠG ang +ĠN uclear +Ġsp ider +ats u +Ġsam pling +Ġabsor bed +ĠPh arm +iet h +Ġbuck et +ĠRec omm +O F +ĠF actory +AN CE +Ġb acter +H as +ĠObs erv +12 1 +Ġprem iere +De velop +Ġcur rencies +C ast +Ġaccompany ing +ĠNash ville +Ġfat ty +ĠBre nd +Ġloc ks +Ġcent ered +ĠU T +augh s +or ie +ĠAff ordable +v ance +D L +em et +Ġthr one +ĠBlu etooth +Ġn aming +if ts +AD E +Ġcorrect ed +Ġprompt ly +ĠST R +Ġgen ome +Ġcop e +Ġval ley +Ġround ed +ĠK end +al ion +p ers +Ġtour ism +Ġst ark +v l +Ġblow ing +ĠSche dule +st d +Ġunh appy +Ġlit igation +ced es +Ġand roid +Ġinteg ral +ere rs +ud ed +t ax +Ġre iter +ĠMot ors +oci ated +Ġwond ers +ĠAp ost +uck ing +ĠRoose velt +f ram +Ġyield s +Ġconstit utes +aw k +Int erest +Ġinter im +Ġbreak through +ĠC her +Ġpro sec +ĠD j +ĠM T +Res p +ĠP T +Ġs perm +ed it +B T +Lin ux +count ry +le ague +Ġd ick +Ġo ct +Ġinsert ing +Ġsc ra +ĠBrew ing +Ġ19 66 +Ġrun ners +Ġpl un +id y +ĠD ian +Ġdys function +Ġex clusion +Ġdis gr +Ġincorpor ate +Ġrecon c +Ġnom inated +ĠAr cher +d raw +achel or +Ġwrit ings +Ġshall ow +Ġh ast +ĠB MW +ĠR S +Ġth igh +Ġ19 63 +Ġl amb +Ġfav ored +ag le +Ġcool er +ĠH ours +ĠG U +ĠOrig in +Ġglim pse +---------------- ---- +L im +Ġche ek +Ġj ealous +- ' +Ġhar ness +ĠPo ison +Ġdis abilities +ne apolis +Ġout look +Ġnot ify +ĠIndian apolis +Ġab rupt +ns ic +Ġenc rypted +Ġfor fe +reat h +Ġr abb +Ġfound ations +Ġcompl iment +ĠInter view +ĠS we +Ġad olesc +Ġmon itors +ĠSacrament o +Ġtime ly +Ġcontem pl +Ġposition ed +Ġpost ers +ph ies +iov ascular +v oid +ĠFif th +Ġinvestig ative +OU N +Ġinteg rate +ĠIN C +ish a +ibl ings +ĠRe quest +ĠRodrig uez +Ġsl ides +ĠD X +Ġfemin ism +Ġdat as +Ġb end +ir us +ĠNig eria +F ox +Ch ange +Ġair plane +ĠLad en +Ġpublic ity +ixt y +Ġcommit ments +Ġaggreg ate +Ġdisplay ing +ĠAr row +Ġ12 2 +Ġrespect s +and roid +s ix +ĠSh a +Ġrest oration +) \ +W S +oy s +Ġillust rate +with out +12 6 +ĠâĶ Ĥ +Ġpick up +n els +Ġ .... +f ood +ĠF en +) ? +Ġphenomen a +Ġcompan ions +ĠW rite +Ġsp ill +Ġbr idges +ĠUp dated +ĠF o +Ġinsect s +ASH INGTON +Ġsc are +il tr +ĠZh ang +Ġsever ity +Ġind ul +14 9 +ĠCo ffee +Ġnorm s +Ġp ulse +ĠF T +Ġhorr ific +ĠDest roy +ĠJ SON +Ġo live +Ġdiscuss es +R est +E lect +ĠW inn +ĠSurv iv +ĠH ait +S ure +op ed +Ġro oted +ĠS ke +ĠBron ze +Ġl ol +Def ault +Ġcommod ity +red ited +Ġliber tarian +Ġforb idden +Ġgr an +à ¨ +Ġl ag +en z +dri ve +Ġmathemat ics +Ġw ires +Ġcrit ically +Ġcarb ohyd +ĠChance llor +ĠEd die +Ġban ning +ĠF ri +Ġcompl ications +et ric +ĠBangl adesh +Ġband width +St op +ĠOrig inally +Ġhalf way +yn asty +sh ine +Ġt ales +rit ies +av ier +Ġspin ning +ĠWH O +Ġneighbour hood +b ach +Ġcommer ce +ĠS le +B U +Ġentreprene ur +Ġpecul iar +ĠCom ments +f re +3 20 +IC S +Ġimag ery +ĠCan on +ĠElect ronic +sh ort +( ( +D ig +Ġcomm em +u ced +Ġincl ined +ĠSum mon +Ġcl iff +ĠMed iterranean +Ġpo etry +Ġprosper ity +ĠRe ce +Ġp ills +m ember +Ġfin ale +un c +ĠG ig +ä ½ +Ġl od +Ġback ward +- + +ĠFor ward +Ġth ri +s ure +Ġso ap +ĠF X +R ES +ĠSe xual +oul os +Ġfool ish +Ġright eous +Ġco ff +terror ism +ust ain +ot er +Ġab uses +ne xt +Ġab usive +Ġthere after +Ġprohib ition +ĠS UP +Ġd ip +Ġr ipped +Ġinher ited +Ġb ats +st ru +G T +Ġflaw ed +ph abet +Ġf og +do ors +Ġim aging +Ġdig its +ĠHung ary +Ġar rog +Ġteach ings +Ġprotocol s +ĠB anks +à ¸ +p ound +ĠC urt +." ) +. / +Ġex emption +end ix +ĠM ull +Ġimpro ves +ĠG amer +d imensional +I con +ĠMarg aret +St atus +d ates +Ġint ends +Ġdep ict +Ġpark ed +J oe +ĠMar ines +chn ology +! ). +Ġjud ged +Ġwe ights +R ay +Ġapart ments +he ster +Ġrein force +Ġoff ender +occ up +Ġs ore +e pt +ĠPH P +ĠB row +Ġauthor ization +ĠR isk +ĠDel aware +ĠQ U +Ġnot ifications +Ġsun light +Ġex clude +d at +Ġm esh +ĠSud an +Ġbelong ed +Ġsub way +Ġno on +ĠInter ior +ol ics +ĠL akers +Ġc oding +Dis claimer +Cal if +O ld +Ġdis l +???? ? +Ġconfir ms +Ġrecruit ment +Ġhom icide +Cons ider +ĠJeff rey +ft y +} ; +Ġobject ion +do ing +ĠLe o +W ant +Ġgl ow +ĠClar ke +ĠNorm an +Ġver ification +Ġpack et +ĠForm ula +Ġpl ag +es ville +Ġshout ing +Ġo v +ĠR EC +ĠB ub +Ġn inth +Ġener g +Ġvalid ity +Ġup s +j ack +Ġneighbor ing +ĠN ec +ew orks +ĠH ab +are z +Ġsp ine +Ġevent ual +ĠLe aders +ĠC arn +Ġprob ation +Ġrom ance +ms g +ĠMechan ical +ER Y +R ock +Ġpart isan +N ode +ass ets +min ent +Ġforeign ers +Ġtest ify +ĠUs ually +l ords +ĠG ren +ĠPow ell +BI L +Ġs r +Ġadd ict +Ġshell s +Ġs igh +ĠY ale +tern ity +Ġ7 50 +E U +ĠR ifle +Ġpat ron +em a +ĠB annon +an ity +Ġtrop ical +ĠV II +c ross +Every thing +ĠIS O +Ġhum ble +ass ing +ĠF IG +Ġupd ating +ys on +Ġcal cium +Ġcompet ent +Ġste ering +Pro t +ĠS Y +ĠFin als +ĠR ug +15 9 +13 7 +ĠG olf +Ġ12 6 +Ġaccommod ation +ĠHug hes +Ġaest hetic +art isan +ĠTw ilight +Ġpr ince +ĠAgric ulture +ĠDis co +Ġpreced ent +Ġtyp ing +author ized +O ption +ĠA ub +l ishes +ach t +m ag +P eter +ĠU FO +mont on +ĠL ith +Ġa rom +Ġsec uring +Ġconf ined +priv ate +Ġsw ords +Ġmark ers +Ġmetab olic +se lect +ĠCur se +ĠO t +g ressive +Ġinc umb +ĠS aga +Ġpr iced +Ġclear ance +Cont ent +Ġdr illing +Ġnot ices +Ġb ourgeois +Ġv est +Ġcook ie +ĠGuard ians +ry s +in yl +Ġ12 4 +Ġpl ausible +on gh +ĠOd in +Ġconcept ion +ĠY uk +ĠBaghd ad +ĠFl ag +Aust ral +ĠI BM +Ġintern ationally +ĠWiki Leaks +I ED +Ġc yn +Ġcho oses +ĠP ill +Ġcomb ining +Ġrad i +ĠMoh ammed +def ense +atch ing +Sub ject +ic iency +Fr ame +Ġ{ " +Ġche ss +Ġtim er +19 0 +Ġt in +Ġord inance +emet ery +Ġacc using +Ġnotice able +Ġcent res +Ġl id +ĠM ills +img ur +Ġz oom +erg ic +Ġcomp ression +pr im +f ind +Ġsur g +Ġp and +ĠK ee +ĠCh ad +cell ence +oy le +Ġsocial ism +ĠT ravis +ĠM Hz +Ġgu ild +ALL Y +ĠSub scribe +ĠRel ated +Ġoccur rence +itch ing +Ġfict ional +Ġcr ush +ĠE A +c od +m ix +ĠTri ple +Ġretrie ve +Ġstimul us +Ġpsych iat +ĠDo or +Ġhomosexual ity +Ġelement ary +Ġcell ular +id ian +ĠL aun +Ġintrig uing +Ġfo am +ĠB ass +id i +its u +Ġass ure +Ġcongr at +Ġbusiness man +ĠBo ost +cl ose +Ġl ied +Ġsc iences +ĠO mega +ĠG raphics +Ġ< = +sp oken +Ġconnect ivity +S aturday +ĠAven gers +Ġto ggle +Ġank le +Ġnational ist +mod el +ĠP ool +ophob ia +V ar +ĠM ons +ator ies +Ġaggress ively +C lear +For ge +act ers +Ġhed ge +Ġpip es +Ġbl unt +Ġs q +Ġremote ly +W ed +as ers +Ġref riger +Ġt iles +Ġresc ued +Ġcompr ised +ins ky +Ġman if +avan augh +Ġprol ifer +Ġal igned +x ml +Ġtri v +Ġcoord ination +ĠP ER +ĠQu ote +13 4 +b f +ĠS aw +Ġtermin ation +Ġ19 0 +Ġadd itions +Ġtri o +Ġproject ions +Ġpositive ly +Ġin clusive +Ġmem br +19 90 +old er +Ġpract iced +ink le +Ar ch +Ġstar ters +ari us +Ġinter mediate +ĠBen ef +ĠK iller +Ġinter ventions +ĠK il +ĠF lying +In v +Ġprem ature +Ġpsych iatric +Ġind ie +Ġcoll ar +ĠRain bow +af i +Ġdis ruption +ĠFO X +cast ing +Ġmis dem +c ro +Ġw ipe +ard on +Ġb ast +ĠTom my +ĠRepresent ative +Ġbell y +ĠP O +ĠBre itbart +13 2 +Ġmess aging +Sh ould +Ref erences +ĠG RE +ist ical +L P +ĠC av +ĠC razy +Ġintu itive +ke eping +ĠM oss +Ġdiscont in +ĠMod ule +Ġun related +ĠPract ice +ĠTrans port +Ġstatist ically +orn s +Ġs ized +p u +Ġca f +ĠWorld s +ĠRod gers +ĠL un +ĠCom ic +l iving +Ġc ared +Ġclim bed +) { +Ġconsist ed +Ġmed ieval +fol k +Ġh acked +Ġd ire +ĠHerm ione +Ġt ended +ce ans +D aniel +w ent +Ġlegisl ators +Ġred es +g ames +Ġg n +am iliar +Ġ+ + +gg y +th reat +Ġmag net +Ġper ceive +Ġz ip +Ġindict ment +Ġcrit ique +g ard +ĠSaf e +ĠC ream +Ġad vent +ob a +Ġv owed +ous ands +Ġsk i +Ġabort ions +u art +Ġstun ned +Ġadv ancing +Ġlack ed +Ġ\ " +Ġsch izophren +Ġeleg ant +Ġconf erences +Ġcance led +ĠHud son +ĠHop efully +Ġtr ump +Ġfrequ encies +Ġmet eor +ĠJun ior +ĠFle et +ĠMal colm +ĠT ools +Ġ ........ +Ġh obby +ĠEurope ans +Ġ15 00 +ĠInt o +Ġs way +ĠApp ro +ĠCom pl +Comm unity +Ġt ide +ĠSum mit +ä » +Ġinter vals +ĠE ther +Ġhabit at +ĠSteven s +lish ing +ĠDom ain +Ġtrig gers +Ġch asing +Ġchar m +ĠFl ower +it ored +Ġbless ing +Ġtext ures +F ive +Ġliqu or +R P +F IN +Ġ19 62 +C AR +Un known +Ġres il +ĠL ily +Ġabund ance +Ġpredict able +r ar +Ġbull shit +le en +che t +M or +M uch +ä ¹ +Ġemphas ized +Ġcr ust +Ġprim itive +Ġenjoy able +ĠPict ures +Ġteam mate +pl er +ĠT ol +ĠK ane +Ġsummon ed +th y +ram a +ĠH onda +Ġreal izing +Ġquick er +Ġconcent rate +cle ar +Ġ2 10 +ĠErd ogan +ar is +Ġrespond s +ĠB I +Ġelig ibility +Ġpus hes +ĠId aho +Ġagg rav +Ġru ins +ur ations +Ġb ans +Ġan at +sh are +Ġgr ind +h in +um en +Ġut ilities +ĠYan kees +Ġdat abases +ĠD D +Ġdispl aced +Ġdepend encies +Ġstim ulation +h un +h ouses +ĠP retty +ĠRaven s +ĠTOD AY +Ġassoci ates +Ġthe rape +cl ed +Ġde er +Ġrep airs +rent ice +Ġrecept ors +Ġrem ed +ĠC e +Ġmar riages +Ġball ots +ĠSold ier +Ġhilar ious +op l +13 8 +Ġinherent ly +Ġignor ant +Ġb ounce +ĠE aster +REL ATED +ĠCur rency +E V +ãĥ ŀ +ĠLe ad +Ġdece ased +B rien +ĠMus k +J S +Ġmer ge +heart ed +c reat +m itt +m und +ĠâĢ ĭ +ĠB ag +Ġproject ion +Ġj ava +ĠStand ards +ĠLeon ard +Ġcoc onut +ĠPop ulation +Ġtra ject +Ġimp ly +Ġcur iosity +ĠD B +ĠF resh +ĠP or +Ġheav ier +ne ys +gom ery +Ġdes erved +Ġphr ases +ĠG C +Ġye ast +d esc +De ath +Ġreb oot +Ġmet adata +IC AL +Ġrep ay +ĠInd ependence +Ġsubur ban +ical s +Ġat op +Ġall ocation +gener ation +ĠG ram +Ġmoist ure +Ġp ine +ĠLiber als +Ġa ides +Ġund erest +ĠBer ry +Ġcere mon +3 70 +ast rous +ĠPir ates +Ġt ense +ĠIndust ries +ĠApp eals +ĠN ear +Ġè£ı ç +Ġlo vers +ĠC AP +ĠC raw +Ġg iants +Ġeffic acy +E lement +ĠBeh avior +ĠToy ota +Ġint est +P riv +A I +Ġmaneu ver +Ġperfect ion +Ġb ang +p aper +r ill +Ge orge +b order +in ters +ĠS eth +Ġcl ues +ĠLe vi +ĠRe venue +14 7 +Ġv apor +Ġfortun ate +Ġthreat ens +Ġve t +Ġdepend ency +ers ed +art icle +ĠBl izzard +Ġch lor +Ġmin us +ĠB ills +Ġcryptoc urrency +Ġmetabol ism +ter ing +Ġp estic +step s +ĠTre asure +ract ed +ĠConst ant +Ġtem p +13 9 +ĠDet ective +ur ally +Ġrecover ing +Ġcort ex +Ġ14 4 +cl osed +Ġprejud ice +aun ted +Ġstorm s +ĠN OW +Ġmach inery +Add ress +Ġcompe lled +27 0 +Ġdesp air +b ane +Ġveget able +Ġbed s +Lear n +Ġcolor ful +Ġsp ike +Ġmarg ins +Ġsymp athy +Ġworks hop +ĠC BC +S at +Ġburn s +ĠG ender +Ġ12 9 +ĠC able +Ġdeb ts +ĠThe resa +Ġreflect ing +Ġa irst +Ġr im +ram id +Ġweakness es +W rit +ogg le +t i +ĠCh arge +Ġwe ighed +Ġ( . +Ġl aughter +Ġrou ter +ĠDemocr acy +D ear +Ġhas ht +Ġd y +Ġhint s +run ning +Ġfin ishes +ar us +M ass +res ult +asc us +Ġv intage +Ġcon qu +Ġwild ly +ac ist +Ġl ingu +Ġprot agonist +st rom +te enth +ĠSol o +m ac +f illed +Ġre nown +it ives +Ġmot ive +ĠAnt ar +ĠM ann +ĠAd just +Ġrock ets +Ġtrou bling +e i +Ġorgan isms +ass is +Christ ian +Ġ14 5 +ĠH ass +Ġsw all +Ġw ax +ĠSurv ival +V S +ĠM urd +v d +stand ard +Ġdrag ons +Ġacceler ation +r ational +f inal +Ġp aired +ĠE thereum +Ġinterf aces +Ġres ent +Ġartif acts +Å « +are l +Ġcompet itor +ĠNich olas +ĠSur face +c pp +ĠT ot +Ġeconom ically +Ġorgan ised +Ġen forced +in ho +Ġvar ieties +Ġab dom +ĠBa iley +id av +ĠSal v +p aid +Ġalt itude +ess ert +ĠG utenberg +are a +op oulos +Ġprofess ors +igg s +ĠF ate +he y +Ġ3 000 +D ist +Ġtw ins +c ill +ĠM aps +Ġtra ps +Ġwe ed +ĠK iss +Ġy oga +Ġrecip ients +ĠWest minster +Ġpool s +ĠWal mart +18 8 +ĠSchool s +att ack +ĠAR M +par agraph +W arning +j l +Ġself ish +anche z +ĠHe ights +F re +ĠS oph +Ġ -------------------------------- +t ml +33 3 +Ġraid s +Ġsatell ites +KE Y +Ġlast s +Ñ Ĥ +In s +ĠD ame +Ġunp redict +// / +gh ai +Ġart illery +Ġcru ise +Ġg el +ĠCabin et +Ġbl ows +ĠE sp +Ġprox imity +ot he +ĠSk ills +ĠU pper +ob o +ĠN DP +Ġenjoy s +Ġrepe ating +ĠConst ruction +ĠQuest ions +H illary +Ġu int +Ġprocess ors +ĠGib son +ĠMult iple +q a +ĠB om +ĠM iles +vent ional +Ġhur ts +s kin +ĠA IDS +Ġadvis ers +ĠR oot +Ġmethod ology +ĠD ale +Ġdet on +ĠKnow ledge +sequ ently +Ġ12 1 +Ġconnect s +C y +ĠD anger +Ġcontribut ors +ĠB ent +Ġbr ass +ĠGun s +int o +ĠFort une +Ġbro ker +bal ance +Ġlength s +Ġv ic +Ġaver aging +Ġappropri ately +ĠCamer a +Ġsand wich +ĠCD C +Ġcoord inate +Ġnav ig +Ġgood ness +l aim +Ġbra ke +Ġextrem ist +ĠW ake +ĠM end +ĠT iny +ĠC OL +ĠR F +ĠD ual +ĠW ine +C ase +Ġref ined +Ġl amp +L ead +Ġb apt +ĠCar b +ĠS add +ĠMin neapolis +PD F +Ear ly +ĠH idden +I ts +ĠT IME +Ġp ap +Ġcommission ed +ĠF ew +ĠCol ts +ĠB ren +Ġbot hered +Ġlike wise +Ex per +ĠSch w +c ry +n n +ĠM itch +im on +M G +b m +UM P +r ays +Ġregist ry +Ġ2 70 +ach ine +re lla +ant ing +00 000 +Ġru ined +sp ot +Ġt a +Ġmaxim ize +Ġincon ven +D ead +H uman +En abled +ĠMar ie +Ġch ill +ĠParad ise +Ġstar ring +ĠLat ino +ĠProt ocol +ĠE VER +Ġsuppl iers +m essage +ĠBro ck +Ġser um +âĸĪâĸĪ âĸĪâĸĪ +Ġen comp +Ġamb ition +ues e +Ġar rows +And rew +Ġanten na +Ġ19 61 +ĠB ark +Ġb ool +ãĤ ª +ĠSt orage +Ġrail way +Ġtoug her +ĠC ad +Ġwas hing +P y +' ] +em bed +ĠMem phis +ack le +Ġfam ously +ĠF ortunately +ov ies +Ġmind set +Ġsne ak +ĠD h +RA W +ĠSim pson +Ġliv est +Ġland mark +Ġc ement +L ow +Ġthr illed +ĠCour se +in el +Ġch uck +id ate +gl obal +Ġwh it +Ġ � +ad ays +s ki +ĠS V +Ġvir uses +30 6 +ĠResp ons +Ġthe aters +ĠBr anch +ĠGene va +ĠM K +Ġunbel iev +Ġcommun ist +Orig inal +ĠRe ceived +ĠTrans fer +ĠAr g +In put +ĠStr ategy +Ġpal ace +the ning +D ri +Ġsent encing +umbn ail +Ġp ins +re cy +Ġs iblings +Get ting +ĠB U +ĠNorth west +Ġprolong ed +ĠSak ura +C omb +ĠB our +Ġinadequ ate +ĠK ash +Ġus ername +ĠImpro ve +Ġbatt ling +ĠM AC +Ġcurric ulum +Ġs oda +ĠC annon +Ġsens ible +sp ons +De cember +Ġw icked +ĠP engu +Ġdict ators +ĠHe arts +og yn +Ġsimilar ities +ĠSt ats +Ġh ollow +it ations +": [ +Ġh over +ĠList en +s ch +S und +Ġc ad +ĠPar ks +Ġl ur +Ġhy pe +ĠL em +N AME +is ure +Fr iday +Ġshoot s +Ġclos es +Ġd b +ĠR idge +ĠDiff erent +Ġrepl ies +ĠBroad way +op ers +Ġint oler +ĠZe us +akes pe +Ġpropri etary +Ġrequest ing +Ġcontro llers +ĠM IN +im edia +be cca +Ġexp ans +Ġoil s +B ot +ĠCh and +Ġpr inter +Ġto pped +ĠP OL +ĠEar lier +S ocial +av in +Ġdecre ases +ĠSe b +Ġspecific ations +ĠBl ast +ĠK urt +Ġfre el +B rown +Ġdil ig +ro e +ĠPro blem +ĠQu ad +Ġdecent ral +ĠV ector +an ut +Ġplug ins +ĠGreg ory +Ġfuck ed +el ines +ĠAmb assador +t ake +Ġcle ans +ong yang +An onymous +st ro +" } +al ine +ĠO dd +ĠE ug +2 16 +Ġbo il +ĠP owers +Ġnurs es +Ob viously +ĠTechn ical +Ġexceed ed +OR S +Ġextrem ists +Ġtr aces +ex pl +Ġcom r +ĠS ach +) / +Ġm asks +Ġsc i +B on +Ġreg ression +we gian +Ġadvis or +it ures +ĠV o +ex ample +ĠInst ruct +Ġs iege +Ġredu ctions +pt r +Ġstat utory +Ġrem oves +Ġp uck +red its +Ġbe e +Ġsal ad +Ġpromot ions +ĠJosh ua +with standing +ET H +ĠCh a +im us +Ġexpend iture +aun ting +Ġdelight ed +Ġ15 5 +be h +Ġcar pet +ĠSp art +Ġj ungle +l ists +Ġbull ying +ĠNob el +ĠGl en +Ġreferen ced +Ġintrodu ces +se in +Ġcho pped +gl ass +ĠW rest +Ġneutral ity +Ġâ Ļ +Ġinvestig ator +Ġshel ves +Ġun constitutional +Ġreprodu ction +Ġmer chant +m ia +Ġmet rics +Ġexplos ives +ĠSon ia +Ġbod ily +Ġthick ness +Ġpredomin antly +ĠAb ility +Ġmon itored +IC H +Ġ] . +ĠMart inez +Ġvis ibility +Ġqu eries +Ġgen ocide +ĠWar fare +Qu ery +Ġstud ios +Ġemb ry +Ġcorrid or +Ġclean ed +com plete +ĠM H +Ġenroll ment +ING S +Ġimpact ed +Ġdis astrous +ĠY un +ĠCl aire +ĠBas ically +y t +uster ity +Ġindirect ly +w ik +Ġd od +ĠCar r +Ġam p +Ġprohib it +ĠIn itial +ĠR d +ij i +Ġeduc ate +c orn +i ott +ĠBeaut y +Ġdetect ive +ĠCon n +s ince +Ġst agger +Ġob ese +Ġb ree +olog ic +is se +walk er +Ġbl ades +Ġlaw ful +fun c +ĠBeh ind +Ġappet ite +Ġ( * +Ġt ennis +Ġoff spring +Ġj ets +Ġstruct ured +Ġafore mentioned +N ov +Ġsc aling +f ill +Ġst ew +Ġcur b +ĠStep han +ed In +S F +ob ic +é ŃĶ +ou g +ĠM M +Ġgen etically +ope z +13 6 +Ġu mb +anc ers +Ġcoh ort +Ġmerch andise +Ġimp osing +ĠLegisl ature +ĠArch ive +iv ia +ĠN aval +Ġoff ences +Ġmir acle +Ġsn apped +Ġf oes +Ġextensive ly +ĠR af +Ġc ater +ed ience +K it +ĠB in +Ġrecomm ends +ĠC ities +Ġrig id +ĠRE AD +ĠNob le +ĠT ian +Ġcertific ates +ant is +o iler +ĠBudd hist +d id +Ġsurvey ed +Ġdown ward +Ġprint s +ĠMot ion +ron ics +ĠS ans +oss ibly +u ctions +Ġcolon ies +ĠDan ish +un it +Ġsp oil +Ġadvis ory +ber ries +Pl an +Ġspecific ation +op hers +ĠRes ource +Ġsh irts +prising ly +commun ications +Ġtriv ial +Ġmention ing +ise xual +Ġsupp lements +Ġsuper vision +B P +v or +Ġw it +Ġco oldown +Ġplaint iff +ĠReview s +ĠS ri +ĠM int +ĠSug ar +Ġafter ward +ĠPri est +ĠInvest ment +og ene +ĠT aking +Ġstretch ing +Ġinflamm ation +ĠTe hran +Ġl ining +Ġfree zing +ĠEnt ity +Ġins piring +spe cial +pr ice +Ġsu e +ĠP orter +oun ge +ET A +ĠD erek +ĠLu is +u o +ym ph +Ġex terior +ih il +ĠAsh ley +in ator +Ġnut rients +ĠTh rones +Ġfin ances +ĠIn spect +Ġspe cially +ĠRequ ired +ĠP TS +ĠViol ence +oint ed +sh ots +Ġex cerpt +co on +IN S +ĠG ri +Ġrecogn ised +We ek +You ng +Ġv om +is le +ĠCur ry +ĠBudd h +Ġnot ebook +Ġd urable +/ ? +ĠG ad +ĠP upp +Ġforg ive +p ark +Ġpersonal ities +an alysis +cl amation +Ġelev ator +Ġware house +ĠR ole +un n +Ġillust ration +ĠSc an +Ġatmosp heric +Im port +AN C +rict ed +f u +01 0 +Ġar che +Ġreward ed +akespe are +Ġintern ally +ĠR BI +alk er +Ġeleph ant +ow itz +ĠP izza +Ġbip artisan +é s +Ġslow ed +ĠSt ark +Ġover ride +OU S +Ġ3 20 +undred s +ĠDe ck +ĠC ensus +be e +14 6 +ot or +Ġ ip +Ġu b +oc ations +ĠBut ton +r ice +Ġc ripp +ff f +Ġorig inated +Ġoverwhel med +app a +Ġfore most +âĢ ij +ĠL EG +re lease +eat ured +at ches +Ġre ps +Ġl ending +ĠRe ference +ĠCl ient +16 5 +vent h +Com plete +ĠPat rol +Ġsw orn +c am +Ġshut tle +ĠR alph +Ġh ometown +- , +on al +ĠB P +å ı +Ġpersu ade +ĠAlex and +Ġcomb ines +Ġv ivid +ĠL ag +Ġenc oding +Ġsal vation +w en +ĠRec overy +i ya +Un iversity +ĠB iden +Ġbud gets +ĠTex ans +f its +Ġhon ored +Ġp ython +T D +## # +cl one +Ġbl ink +ĠL iquid +Ġunemploy ed +Ġcl ashes +ĠCoun sel +Ġdirect ing +Ġpun ct +ĠFal cons +Ġsh ark +ĠDam ascus +Ġje ans +Ġemb ark +Ġse ize +Ġup wards +2 80 +ĠE z +ĠAny thing +Ġex otic +l ower +ĠCreat or +ĠU m +Ġsubur bs +ber ger +ĠW end +Ġm int +ĠX X +ĠD ro +Ġsuff ers +Ġher b +t ree +Ġfrag ile +Ġflood ed +ĠAl cohol +ole an +ny der +ĠK O +F ram +Ġ13 6 +Ġow ed +ĠMe lee +ĠH ash +Ġwh isk +Ġsu do +r r +Qu ick +app ro +Ġi i +ĠEx amples +he e +Ġpromot es +per ature +k ar +ĠHon or +Ġs odium +ĠL if +ros so +intend ent +Ġcorrespond ent +F ound +sec ret +Ġident ifies +ag ne +Ġl ou +ĠP P +Ġcoinc idence +m ove +Ġmilit ia +Ġinf iltr +ĠPrim ary +Ġpitch ing +ĠI b +ĠGO OD +ãĤ ¸ +ĠW izards +ir al +ĠVen us +R R +ĠâĢ ķ +ĠCase y +Ġsad ly +Ġadm ire +Ġembarrass ed +c b +M el +Ġtub es +Ġbeaut ifully +ĠQueens land +Bel ow +re z +qu et +ple asant +Ġ « +C amp +Ġdec isive +19 98 +ĠL amb +ut ton +h n +ĠJ agu +au nder +ĠC ord +Ġcl erk +Ġca ffe +Ġwip ed +Ġre im +ĠMount ains +Ġimprison ed +Ġdevelop s +ĠP ra +Ġmodel ing +Any one +ance l +ĠS it +Ġshield s +Ġl awn +Ġcard iovascular +Ġdemonstr ating +Ġpar se +ĠIsrael is +Ġeuro s +14 3 +Ġgl orious +ins ki +ec d +Ġcondition ing +Ġhel pless +Ġmicro sc +ĠHar bor +Ġst akes +Ġ2 60 +Ġun equ +ĠFl oyd +Ġd amp +Ġappar atus +ĠLaw s +Ġcoun ters +Ġindu ce +at able +ĠAh med +Ġsl am +N ovember +Ġpers ist +Ġim minent +á n +Ġsh red +Ġph ases +ĠEd monton +ĠArm strong +ĠMe et +ĠK itty +Ñ Ģ +c irc +ĠAd ult +Ġa rose +ĠX en +D an +g ow +Ġsuper f +ĠAd mir +Ġend ure +Ġkey word +yr us +Ġy arn +Ġpath way +ĠHop kins +mid t +Ġcens orship +d ependent +Ġinstruct or +S ources +Ġto e +Ġball oon +N ob +Ġsw ear +ĠCast ro +Ġgl oss +ĠK avanaugh +Ġremark ably +Ph otos +ĠN om +ĠS outheast +y ers +Ġvalid ation +Ġcann on +ĠVict ory +ĠPier re +Ġcaut ious +Aud io +Ġf etch +ĠG ift +ĠH yp +Ġrem edy +Z E +Ġsc ent +Ġbe ard +ĠR ut +- " +Ġpat ents +H y +Ġun just +Ġpot ato +Ġforth coming +Ġche f +ĠR ift +aff e +ĠR OM +ĠL aunch +Ġp ads +ĠNe o +Ġon set +Ġsquee ze +s afe +Ġpref ix +ĠT M +ĠN early +ĠClin ical +ĠM ental +ot iation +ĠUn ic +ant ry +ĠC ir +Ġep it +à ¦ +Ġextract ed +verse ly +ri ad +Ġstr ains +Ġto ps +Ġpo em +ĠRand y +ĠMap le +TH ER +up iter +ĠSS D +ļ é +Ġun con +per ing +Ġsle pt +in ers +Ġunder water +ĠEv idence +g one +20 5 +Ġhistor ians +Ġsynt hesis +Ġf rog +b asketball +Ġvibr ant +Ġsub ord +Ġ3 65 +ĠD ial +Ġcooper ate +HA HA +Ġgreet ed +15 8 +Ġj azz +Ġinto x +ĠWalk ing +Ġsuper visor +ĠF usion +ĠMer cedes +s end +H am +s d +n l +Ġtour s +ĠF IFA +Ġcul p +g d +30 4 +Ġple as +Ġillust rates +ĠColomb ia +Ġhighlight ing +ĠSum mary +Ġexp osing +ĠD ru +Ġir ony +r itional +ĠCar roll +ĠEll is +P ict +ĠR apt +Ġad apter +Ġun m +Ġcor pse +Ġceleb rities +D en +at um +ĠAp ocalypse +ĠW ag +lin ing +Ġhorm ones +R ub +ĠX i +ĠV aults +20 8 +alky rie +inos aur +Ġfeed s +v ity +Ġdefe ating +W ait +Ġemphas ize +ĠSteel ers +yr inth +le ys +ĠWhe never +Current ly +ĠCl ock +Ġcollect ively +any on +ĠJ P +Ġment ality +Ġdownload s +Ġsurround ings +ĠBarn es +Ġflags hip +Ġindic ators +Ġgra pp +Jan uary +ĠElement al +ĠAthen a +ib al +Ġs ights +Ġcap ita +ĠTreat y +Ġvo iced +ĠG az +let te +Ġy a +Ġexp ired +Leg end +H ot +n ature +Ġunst able +Ġ2 80 +à º +Com ment +AL E +Ġquest s +Ġhand ler +n is +Ġvers atile +Ġconce al +enge ance +ĠInter active +Ġobs essed +ĠDog s +Ġcr acked +S ound +s v +ĠD ylan +ro ads +f x +ĠCath olics +ĠH ag +Ġsl ammed +Ġgl owing +s ale +Ġtiss ues +ĠCh i +ne e +Ġc her +s ic +ur rection +Ġb acon +ul atory +) ." +Ġir regular +FOR M +ass ed +Ġintention al +Ġcompens ate +ĠSpe aking +ĠS ets +15 3 +Ġconvent ions +b ands +em ade +Ġe cc +ĠWin ston +ĠAssass in +ĠBelg ian +Ġdepend ence +Ġnic he +Ġb ark +ĠJ azz +Ġdisadvant age +Ġgas oline +Ġ16 5 +çļ Ħ +ess a +mod ule +ang ular +O Y +ĠTreat ment +it as +ol ation +ĠArn old +Ġfe ud +ĠN est +Ġthe atre +ew ater +Ġmin ors +olic y +ĠH aven +div ision +Ġtr unk +F ar +ĠP ull +Ġcapt uring +Ġ18 00 +ĠTe en +Ġex empl +Ġclin ics +ĠB urg +Ġsubst it +Ġpay load +ĠL av +ĠT roy +ĠW itness +Ġfrag ments +Ġpass words +Ġg ospel +ĠG in +Ġten ants +ol ith +S ix +Pre vious +ĠAg es +ĠDar win +Ġbl at +Ġem pathy +sm ith +b ag +ĠE cho +ĠC amb +ĠM add +ĠB oo +Ġred e +ĠBurn ing +Ġsmooth ly +ĠAd rian +ĠV ampire +ĠMon sters +ste am +Sty le +M a +re a +ĠD war +aly st +urs or +Ġelim ination +Ġcrypt o +ch t +ĠE ternal +âĢ¦ ] +ĠS orce +I ll +N ER +Ġu h +Con clusion +w age +Ġresp ir +Ġrem inis +het ical +Ġg y +Ġutil ized +ic idal +Ġ19 00 +Ġhun ters +ĠSw an +ĠRe act +Ġvis itor +ĠThanks giving +30 8 +Post s +Ġh ips +19 97 +om ers +Ġkn ocking +ĠVeh icle +Ġt il +Ġ13 8 +Ġm i +ĠInvest igation +ĠKen ya +Ġcas ino +Ġmot ives +Ġreg ain +re x +Ġweek ends +Ġstab bed +bor o +Ġexplo ited +ĠHA VE +ĠTe levision +c ock +Ġprepar ations +Ġende av +ĠRem ote +ĠM aker +ĠPro du +ĠEv an +Ġinform ational +ĠLouis ville +15 4 +ĠDream s +Ġpl ots +ĠRun ner +Ġhur ting +Ġacad emy +ĠMont gomery +n m +ĠL anc +ĠAl z +2 10 +el ong +Ġretail er +Ġar ising +Ġrebell ion +Ġbl onde +play ed +Ġinstrument al +C ross +Ġret ention +Ġtherape utic +Ġse as +Ġinfant ry +ĠCl int +Ġprompt ing +Ġbit ch +Ġst ems +ĠK ra +Ġthe sis +ĠB og +ru ed +Ġk ings +Ġcl ay +ific ent +ĠY ES +ĠTh ing +ĠCub s +vey ard +els h +in arily +ĠE y +ĠRoll ing +Ġev olving +Ind ia +Ġrecogn izes +Ġgrad uation +is ers +Ġfert ility +ĠMil an +Comm and +Ġbox ing +Ġ19 43 +Ġgl uten +ĠEm ir +Ġid ol +Ġcon ceived +ĠCre ation +Mer it +udd y +uss ions +ĠLie utenant +iet al +Ġunch anged +ĠSc ale +ĠCrime a +ball s +ator ial +Ġdepth s +Ġempir ical +Ġtrans m +Ġuns afe +miss ible +com fort +15 6 +Ġmechan ic +00 2 +l ins +Ġsm oked +P os +Ġslow ing +Ġl av +Tex as +Ġche ating +ĠMet ropolitan +eth yl +Ġdiscover ing +as se +Ġpen cil +ĠPy ongyang +Ġclos et +ĠShe et +ĠEnt ry +ou stic +Ġmy st +er ate +ari at +Ġminer als +Ġmusic ian +ĠP ul +ĠM az +24 9 +Ġper missions +Ġ iv +en ary +ick ers +ĠB ing +he a +en able +Ġgri ev +Ġassert ed +ĠColon el +Ġaff idav +w o +Ġse ated +ĠR ide +Ġpaint ings +ĠP ix +Ġ13 7 +ish i +umb ai +g otten +ĠEar l +Ġin ning +Ġc ensus +Ġtrave lled +ĠCons ult +18 5 +b ind +Ġsimpl icity +Ġoverlook ed +ĠHelp ful +Ġmon key +Ġoverwhelming ly +Bl ood +ĠFl int +ĠJ ama +ĠPres ent +ĠR age +ĠT A +pt ive +Ġturn out +w ald +ĠD olphins +ĠV PN +Ġon ion +Ġcraft ing +m ma +ĠMerc ury +Ġarr ange +Ġalert s +ĠO T +zb ollah +Ġg ases +ĠRichards on +s al +l ar +Ġfro st +Ġlower ing +Ġacc laim +Ġstart ups +ĠG ain +ess ment +Ġguard ian +äº º +ĠP ie +ĠL inks +Ġmer its +Ġaw ake +Ġparent al +Ġexceed s +Ġid le +ĠPil ot +Ġe Bay +ĠAc cept +ipe g +C am +ĠK ot +Ġtrad ers +olit ics +unk er +ĠP ale +os i +an mar +Ġ19 47 +ĠF ell +est ial +it ating +G F +ĠS r +if ted +Ġconnect or +ĠB one +ill es +2 60 +h ma +Ġoverl ap +ĠGit Hub +Ġclean er +ĠBapt ist +ĠW AS +Ġlung s +Ñ ģ +ĠB UT +Ġc ite +Ġpit ched +reat ment +Ġtro phies +ĠN u +38 6 +ĠPr ide +Ġattend ees +[ ] +17 9 +Ġspat ial +Ġpri zes +ĠRel igion +Ġshow case +ĠC ategory +vid ia +T arget +Pro perty +? , +Ġf usion +p ie +ĠU CLA +Ġsound track +Ġprin cess +ĠC aval +sh ould +Ġlim bs +Back ground +Ġlone ly +Ġc ores +ĠT ail +she et +Ġ13 2 +R a +ãĤ « +ĠB olt +Ġbook ed +Ġadmin ister +Ġequ als +w y +Ġobserv ing +ĠBar on +ĠAd obe +Ġv irgin +ĠSocial ist +M ove +gh azi +ĠLind a +2 12 +Ġbre wing +Ġmerch ants +bur se +Ġdiv or +Ġmet als +ĠN er +Ġsum s +ĠEn emy +Ġen vision +Ġgrant ing +ĠH oney +ĠSk yrim +Ġsoc io +gr aded +Ġselect ive +W ASHINGTON +Ġ19 48 +ĠSir ius +ĠG ross +act ivity +ĠI van +Ġfur ious +BS D +ĠPre vious +Ġrespons ive +Ġchar itable +Ġle aning +ĠP ew +Ġviol ates +\\\\ \\\\ +ĠCom ing +w ire +Ġpo et +Ġres olutions +comm and +ĠPortug uese +Ġnick name +Ġde af +Feb ruary +Ġrecogn ise +Ġentire ty +Ġseason al +pl aced +ĠTe legraph +Ġmicro phone +our ing +Ġgr ains +Ġgovern ed +Ġpost p +ĠW aters +in ement +Ġund ocumented +ĠCom cast +Ġf ox +Ġassault s +re on +man y +ĠJen kins +ĠAny way +Ġassess ments +Ġdown s +ĠM ouse +Ġsuper b +k t +ĠD ow +Ġtax ation +4 01 +Ġsm iles +Ġundert aken +Ġex h +Ġenthusi astic +Ġtw ent +Ġgovernment al +Ġautonom y +ĠTechn ologies +ĠCh ain +Ġpreval ent +f b +Ġnic otine +og ram +j ob +Ġawa iting +ĠMen u +Ġdep uties +k ov +ish ops +But ton +ĠShan ghai +Ġdies el +ĠD uck +R yan +ĠPC s +N F +j ury +ent e +Ġinacc urate +edd y +Wh atever +Ġshow c +ĠN ad +od us +et r +Ġplaint iffs +ĠW OR +ĠAss ange +Ġpriv at +Ġpremium s +Ġt am +UR L +Ġel ites +ĠR anger +otten ham +ĠH off +ĠAt hens +Ġdefin ite +Ġs ighed +Ġeven ly +2 11 +ĠAm ber +ak ia +Ġmail ing +Ġcr ashing +ĠConfeder ate +ru gged +W al +ĠDep ths +Ġjuven ile +Ġreact or +Introdu ction +ĠDel uxe +19 95 +ĠS anchez +ĠM ead +iv able +: - +ĠPlan ning +ĠT rap +qu in +ĠProt ect +ve red +In formation +Ġkid ney +inn amon +l as +Ġpolic ing +Ġtoler ate +ĠQ i +Ġbi ased +F ort +ĠK i +s ave +Ġprivile ged +Ġbe asts +ĠGl as +ĠC inem +Ġcome back +Sund ay +Ġext inction +h ops +Ġtrans mit +Ġdoub les +ĠFl at +16 7 +Ġdis puted +Ġinjust ice +f oo +V ict +role um +ĠJul ie +Con text +ĠR arity +iss ue +Comp onent +Ġcounsel ing +an ne +d ark +Ġobject ions +u ilt +Ġg ast +Ġpl ac +Ġun used +ãĥ ĩ +ĠT rial +ĠJ as +hed ral +ob b +Ġtempor al +ĠPR O +ĠN W +ĠAnn iversary +L arge +Ġther m +Ġd avid +Ġsystem ic +ĠSh ir +m ut +ĠNe pt +add ress +Ġscan ning +Ġunderstand able +Ġcan vas +C at +ĠZ oo +Ġang els +L O +ĠStat ement +ĠS ig +ov able +ĠA way +sh aring +ocr ats +st ated +Ġweigh ing +N or +w ild +B ey +Ġaston ishing +ĠReyn olds +Ġop ener +Ġtrain er +Ġsurg ical +p n +Ġadjust ing +whe el +Ġf rown +erv ative +Ġsusp end +With in +te in +Ġobst acle +Ġliber ties +ym es +Ġur anium +ans om +an ol +ub a +ĠL oss +Ġa rous +ĠHend erson +W ow +s pl +c ur +ĠÂ Ń +Ġtheir s +Dam age +Ġdownload ing +Ġdisc ern +ĠSt o +ĠFl a +Ġh ath +ĠA j +Ġun pleasant +Europe an +exp ensive +Ġscreens hot +ĠU V +Ġall ied +ĠPers ian +Ġmonop oly +Ġat om +ĠReds kins +"> < +Ġcan cell +Ġcinem a +13 1 +f air +ĠAlf red +Ġd uck +arg s +22 3 +ĠIS I +Ġsign aling +in ar +Ġlaugh s +Ġfor wards +Ġreck less +Ġlisten ers +at ivity +Ġvast ly +n ant +L ess +ĠHun ting +ĠScient ific +IT ED +Ġkn ight +ĠH TC +us a +t mp +Ġr ude +ĠLegend ary +Ġar ises +B ad +ĠCl aim +pe g +Ġreal ities +Th ink +Ġ ° +Ġro de +Ġstri ve +Ġan ecd +Ġshort s +Ġhypot hes +Ġcoord inated +ĠGand hi +ĠF PS +R ED +Ġsuscept ible +Ġshr ink +ĠCh art +Hel p +Ġ ion +de ep +rib es +ĠK ai +ĠCustom er +Sum mary +Ġc ough +w ife +Ġl end +Ġposition ing +Ġlot tery +ĠC anyon +Ġf ade +Ġbron ze +ĠKenn y +Ġbo asts +ĠEnh anced +rec ord +Ġemer gence +Ġa kin +ĠB ert +it ous +âĸ ij +Ġst ip +Ġexch anged +om ore +als h +Ġreserv oir +Ġstand point +W M +Ġiniti ate +Ġdec ay +Ġbrew ery +Ġter ribly +Ġmort al +lev ard +Ġrev is +N I +el o +Ġconf ess +ĠMS NBC +Ġsub missions +Cont roller +Ġ20 2 +ĠR uth +} ); +ĠAz ure +Ġ ." +20 6 +ĠMarket ing +Ġl aund +ien cies +Ġrenown ed +ĠT rou +ĠN GO +ble ms +Ġterr ified +Ġwar ns +Ġper t +Ġuns ure +4 80 +ale z +ult z +ĠOut side +Ġst yl +ĠUnder ground +Ġp anc +Ġd ictionary +Ġf oe +rim inal +ĠNor wegian +Ġj ailed +Ġm aternal +é e +ĠLu cy +c op +Ch o +Ġuns igned +ĠZe lda +ĠIns ider +ĠContin ued +Ġ13 3 +ĠNar uto +ĠMajor ity +16 9 +ĠW o +ãĤ ĵ +Ġpast or +Ġinform al +Ð ½ +an throp +jo in +ãģ Ĺ +it ational +N P +ĠWrit ing +f n +ĠB ever +19 5 +Ġy elling +Ġdr astically +Ġe ject +Ġne ut +Ġth rive +ĠFre qu +ou x +Ġpossess es +ĠSen ators +ĠD ES +ĠSh akespeare +ĠFran co +ĠL B +uch i +Ġinc arn +Ġfound ers +F unction +Ġbright ness +ĠB T +Ġwh ale +ĠThe ater +m ass +ĠD oll +S omething +Ġecho ed +ĠHe x +c rit +af ia +Ġgodd ess +Ġele ven +ĠPre view +ĠAur ora +Ġ4 01 +uls ive +ĠLog an +in burgh +ĠCent ers +ĠON LY +ĠA id +Ġparad ox +Ġh urd +ĠL C +D ue +c ourt +Ġoff ended +Ġeval uating +ĠMatthew s +Ġto mb +Ġpay roll +Ġextra ction +ĠH ands +if i +Ġsuper natural +ĠCOM M +] = +dog s +Ġ5 12 +ĠMe eting +Rich ard +ĠMax imum +Ġide als +Th ings +m and +ĠReg ardless +Ġhum ili +b uffer +L ittle +ĠD ani +ĠN ak +Ġliber ation +ĠA be +ĠO L +Ġstuff ed +ac a +ind a +raph ic +Ġmos qu +Ġcampaign ing +Ġoccup y +S qu +r ina +ĠW el +ĠV S +Ġphys ic +Ġp uls +r int +oad ed +ET F +ĠArch ives +Ġven ues +h ner +ĠTur bo +Ġl ust +Ġappeal ed +que z +il ib +ĠTim othy +Ġo mn +d ro +Ġobs ession +ĠSav age +19 96 +Gl obal +J es +2 14 +Ġsl iding +Ġdisapp ro +ĠMag ical +Ġvolunt arily +g b +ane y +Ġprop het +ĠRe in +ĠJul ia +ĠW orth +aur us +Ġb ounds +ie u +)) ) +Ġcro re +ĠCitiz en +S ky +Ġcolumn ist +Ġseek ers +ond o +IS A +ĠL ength +Ġnost alg +Ġnew com +Ġdet rim +ent ric +3 75 +ĠG E +Ġaut op +Ġacadem ics +App Data +ĠS hen +Ġid iot +ĠTrans it +Ġteasp oon +W il +K O +ĠCom edy +> , +Ġpop ulated +W D +Ġp igs +ĠO culus +Ġsymp athetic +Ġmar athon +19 8 +Ġseiz ure +s ided +Ġd op +irt ual +L and +ĠFl oor +osa urs +... ] +Ġl os +Ġsubsid iary +E Y +ĠPart s +ĠSt ef +ĠJud iciary +Ġ13 4 +Ġmir rors +Ġk et +t imes +Ġneuro log +Ġc av +ĠGu est +Ġtum or +sc ill +ĠLl oyd +E st +Ġcle arer +Ġstere otypes +Ġd ur +not hing +Red dit +Ġnegoti ated +---------------- -------- +23 5 +Ġfl own +ĠSe oul +ĠRes ident +ĠS CH +Ġdisappear ance +ĠV ince +g rown +Ġgrab s +r il +ĠInf inite +ĠTw enty +Ġpedest rian +Ġjer sey +ĠF ur +ĠInf inity +ĠEll iott +Ġment or +Ġmor ally +Ġob ey +sec ure +iff e +Ġantib iotics +ang led +ĠFre eman +ĠIntrodu ction +J un +Ġm arsh +ic ans +ĠEV ENTS +och ond +W all +icult y +Ġmisdem eanor +Ġl y +Th omas +ĠRes olution +Ġanim ations +ĠD ry +Ġinter course +ĠNew castle +ĠH og +ĠEqu ipment +17 7 +Ġterrit orial +Ġarch ives +20 3 +Fil ter +ĠMun ich +Ġcommand ed +ĠW and +Ġpit ches +ĠCro at +Ġrat ios +ĠM its +Ġaccum ulated +ĠSpecific ally +Ġgentle man +acer b +Ġp enn +Ġa ka +ĠF uk +Ġinterven e +ĠRef uge +ĠAlz heimer +Ġsuccess ion +oh an +d oes +L ord +Ġsepar at +Ġcorrespond ence +Ġsh iny +P rior +Ġs ulf +Ġmiser able +Ġded ication +( ). +Ġspecial ists +Ġdefect s +ĠC ult +ĠX ia +Ġje opard +ĠO re +Ab ility +Ġle ar +Ġamb itions +ĠB MI +ĠArab s +Ġ19 42 +Ġpres ervation +ific ate +Ġash amed +l oss +ĠRest aur +Ġrese mble +Ġen rich +ĠK N +ĠCl an +fl oat +Ġplay able +IT T +Ġharm ony +arr ison +ĠWe instein +w ere +Ġpoison ing +ĠCom put +ĠWord Press +m ajor +ĠVal ve +F an +ĠTh row +ĠRom ans +ĠDep ression +ad os +Ġtort ured +Ġbal ancing +bott om +Ġacqu iring +ĠMon te +ard i +Ġa ura +Ġ# # +ĠStand ing +ĠAtl as +C F +Ġintr ins +ĠBen ghazi +Ġcamp ing +Ġt apped +bl ade +st rous +ĠR abb +ĠW ritten +t ip +ĠNe igh +ster dam +ĠAll ow +ĠHe aling +ĠR hod +n um +Ġcaffe ine +ĠPer cent +Ġbo o +Ġapp les +30 5 +Ġwel coming +Ġappl aud +Ġa usterity + ± +ĠRe ality +ef e +å ® +Ġsu cks +Ġtab s +ĠPay Pal +Ġback pack +Ġgif ted +abul ary +ĠSc out +ir teen +Ġch in +Ġo mitted +Ġnegative ly +Ġaccess ing +ĠE arn +Ġambul ance +Ġhead phones +Ġ20 5 +ĠRef resh +p resident +ĠKit chen +ĠEnt ered +ĠS nyder +00 5 +om ical +Ġborrow ed +ĠN em +Ġav iation +Ġst all +rim ination +Ġuniform s +it ime +ĠSim mons +ener gy +ab lished +y y +qual ified +Ġrall ies +ĠSt uart +fl ight +Ġgang s +r ag +Ġv ault +lu x +ĠCom par +Ġdesign ation +20 9 +ĠJ os +d ollar +z ero +Ġwell s +30 3 +Ġconstitu ents +Ġhe ck +Ġc ows +Ġcommand ers +Ġdifferent ial +ĠC atherine +29 9 +Ġval ve +Ġbr ace +Ġperspect ives +c ert +f act +icular ly +ĠMc N +pl anes +Ġint ric +Ġpe as +ov an +Ġtoss ed +ret ch +ĠL opez +Ġunf amiliar +de ath +ĠA part +ĠCh ang +Ġrelie ved +rop he +Ġair ports +Ġfre ak +ut il +M ill +ĠCh in +ĠOw en +m ale +ĠBro ken +ĠWind s +ro b +r ising +Ġfire fighters +Ġauthor itarian +Ġ14 8 +Bit coin +ex ternal +Ġbrow sers +iche ver +or ian +Ġun b +Ġpo ke +ĠZ ot +M id +ĠPop ular +Ġco vert +Ġcont ributes +Ġ6 50 +Ġcont ention +G ate +Ġcons oles +Ġchrom os +ĠI X +Ġvis ually +ĠE isen +Ġjewel ry +Ġdeleg ation +Ġacceler ate +ĠR iley +Ġsl ope +Ġind oor +it ially +Ġhuge ly +Ġtun nels +Ġfin ed +Ġdirect ive +Ġfore head +ustom ed +Ġsk ate +Mus ic +g as +Ġrecogn izing +am bo +Ġover weight +ĠGr ade +Ù Ĭ +Ġsound ing +Ġlock ing +ĠR EM +St ore +Ġexc av +ĠLike wise +ĠL ights +Ġel bow +ĠSupp ly +w ic +Ġhands ome +19 94 +C oll +Ġadequ ately +ĠAssoci ate +Ġstri ps +Ġcrack down +Ġmar vel +ĠK un +Ġpass ages +@@ @@ +ĠT all +Ġthought ful +names e +Ġprost itution +bus iness +Ġball istic +person al +c ig +iz ational +R ound +ĠÂłĠÂł ĠÂłĠÂł +ĠCole man +Ġadm itting +ĠPl ug +Ġbit coins +ĠSu z +Ġfair ness +Ġsupp lier +Ġcatast rophic +ĠHel en +o qu +M arc +ĠArt icles +g ie +Ġend angered +Ġdest iny +ĠVol t +ol ia +ax is +Ġche at +Ġun ified +IC O +qu ote +30 2 +ĠS ed +Ġsupp ression +Ġanaly zing +Ġsqu at +Ġfig uring +Ġcoordin ates +Ġch unks +Ġ19 46 +Ġsub p +Ġw iki +ĠFor bes +ĠJ upiter +ĠE rik +im er +ĠCom mercial +\ ) +Ġlegitim acy +Ġd ental +ĠMe an +Ġdefic its +5 50 +Orig inally +ĠHor ror +Ġcontam ination +ll ah +Ġconf isc +ĠCl are +T B +ĠF ailed +an ed +Ġrul er +ĠCont roller +Ġfemin ists +F ix +g ay +20 7 +Ġr abbit +Th ird +ownt own +Ġgl ue +Ġvol atile +Ġsh ining +Ġf oll +Ġimp aired +Ġsup ers +æ Ī +Ġcl utch +ļé ĨĴ +Ġpro let +Ġ( ! +Ġy elled +ĠK iev +ĠEr n +ĠSh ock +K B +Ġsit uated +qu ery +ĠN as +Ġan nex +char acter +ĠHol iday +Ġautom ation +ĠJ ill +ĠRem astered +Ġl inem +Ġwild erness +ĠHor izon +ĠGu inea +A Z +Ġmain land +Ġsec recy +LE ASE +Ġp unk +ĠProv ince +( ), +Spe ed +Ġhand ing +ĠSeb ast +S ir +r ase +Ġj ournals +Ġcon gest +ĠT ut +ir rel +Ġschizophren ia +Ġmis ogyn +health y +I ron +Ġreact ed +- $ +25 2 +Ġpl ural +Ġpl um +Ġbarg ain +Ġground ed +f inder +Ġdis se +ĠL az +O OD +Ġat roc +F actory +Ġmin ions +Ġo ri +ĠB rave +ĠP RE +ĠMy anmar +ĠH od +Ġexped ition +Ġexpl ode +ĠCo ord +Ġext r +ĠB rief +ĠAD HD +Ġhard core +feed ing +Ġd ile +ĠF ruit +Ġvacc ination +ĠM ao +osp here +Ġcont ests +- | +Ġf ren +isp here +R om +ĠSh arp +ĠTre nd +Ġdis connect +âĢ¢ âĢ¢ +Ġper secution +Ear th +Ġhealth ier +38 4 +Ġc ob +ĠTr inity +OW S +AN N +Ġspecial ty +Ġg ru +Ġcooper ative +wh y +Start ing +ĠIss ues +st re +ens or +Ġ18 5 +Ad v +! ? +ĠRe vel +em ia +ĠH ulk +Ġcelebr ations +ĠS ou +ra ud +ĠKle in +Ġun real +con text +Ġpartners hips +Ġadop ting +t ical +Ġspl ash +ĠHe zbollah +c ategory +cycl op +xt on +ĠD ot +urd y +t z +Ġenvelop e +ĠN L +â ķ +Ġwhere in +Spe c +18 4 +Ġte lev +al iation +Ġmyth s +å ° +Ġrig orous +Ġcommun icating +Ġobser ver +Ġre he +ĠW ash +Ġapolog ized +ĠT in +Ġexpend itures +work ers +d ocument +Ġhes itate +ĠLen in +Ġunpredict able +Ġrenew al +cl er +ok ia +ĠCON T +Ġpost season +Tok ens +Ġex acerb +Ġbet ting +Ġ14 7 +Ġelev ation +W ood +ĠSol omon +19 4 +00 4 +out put +Ġredu nd +ĠM umbai +Ġp H +Ġreprodu ce +ĠD uration +MA X +Ġb og +C BS +ĠBal ance +ĠS gt +ĠRec ent +Ġc d +Ġpo pped +Ġincomp et +pro p +ay an +g uy +Pac ific +Ġty r +Ġ{ { +ĠMy stic +ĠD ana +Ġmast urb +Ġge ometry +à ¢ +ĠCor rect +Ġtraject ory +Ġdistract ed +Ġf oo +ĠW elsh +L uc +m ith +Ġrug by +Ġrespir atory +Ġtri angle +Ġ2 15 +Ġunder graduate +ĠSuper ior +ch anging +_ - +Ġright ly +Ġrefere e +Ġluc rative +Ġun authorized +Ġresemb les +ĠGN U +ĠDer by +Ġpath ways +ĠL ed +Ġend urance +Ġst int +Ġcollect or +F ast +Ġd ots +Ġnational s +ĠSec urities +Ġwh ip +Par am +Ġlearn s +M agic +Ġdetail ing +m oon +Ġbroadcast ing +Ġb aked +26 5 +hol m +ĠS ah +ĠHus sein +ĠCourt esy +17 4 +Ġ14 6 +Ġge ographic +pe ace +Ġjud ging +ĠS tern +B ur +Ġstory line +G un +ĠSt ick +24 5 +30 7 +ãĤ´ ãĥ³ +ĠAdminist rator +Ġbur nt +Ġp ave +ch oes +Ex ec +Ġcamp uses +Res ult +Ġmut ations +ĠCh arter +Ġcapt ures +Ġcomp ares +Ġbad ge +S cient +Ġer ad +ier y +o i +ett es +ĠE state +Ġst rap +Ġproud ly +Ġf ried +Ġwithd rawn +ĠV oy +ph ony +It ems +ĠP ierce +b ard +Ġann otation +ant on +ill on +Im pro +... ) +Ġhapp ier +---- -- +ad just +Ġstaff ers +Ġactiv ism +Ġper f +Ġal right +N eed +Ġcomm ence +Ġopio id +ĠAm anda +E s +ĠP ars +ĠK aw +W orks +24 8 +Ġind o +t c +end ant +ĠM oto +Ġlegal ization +OT E +Ġtask ed +Ġt sp +ĠACT IONS +16 6 +Ġrefres hing +ĠN R +ĠPere z +Ġinfring ement +S Y +List en +in ning +k u +Ġrot ate +pro gram +ar ah +Des ign +Ġ( £ +Ġst oring +Ġwar rants +Ġjud gement +ĠB rist +us ually +ph oto +ĠR an +ĠP ine +Ġoutrage ous +ĠValent ine +lu ence +ĠEvery body +Al tern +Ġrele vance +Ġtermin ated +Ġd essert +Ġfulf illed +Ġprosecut ed +ĠW ords +Ġm igrant +Ġcultiv ation +ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ +idel ity +ĠV ern +ĠLog in +Ġmetaph or +ĠT ip +Ġrecru its +ĠP ig +rib ing +Ġenthusi asts +ex per +Ġfright ening +ĠH air +ans on +str ate +Ġh i +He ight +Ġown ing +n one +Ġdis like +Ġkn ives +pher d +Ġloud ly +ĠAP Is +Dis play +ĠL ac +ĠUS S +ab l +ver ages +J ew +Ġ17 2 +ĠHist orical +at oon +ĠPhys ics +in tern +Ġwarm th +Ġto pp +D M +Ġgun man +Ġem peror +od i +ãĥ £ +in atory +ĠR ib +Ġ13 1 +ĠSat urn +ĠSh ining +Ġw aking +Qu otes +Ġcomed ian +en berg + ½ +Ġbelie vers +Ġpaper work +c ustom +Ġle v +Ġl ament +Ġpour ing +22 2 +p olitical +ĠSupp lement +m aid +Ġcruel ty +Ġt read +ys ics +A w +rit es +Ġmod ifier +ĠP osition +Ad am +l b +ub s +Ġimper fect +Ġcl usters +ĠEngine er +ĠC herry +Ġinaug uration +ĠS au +Ġembod iment +ĠUn cle +Ġover r +Ġexplos ions +c ule +ĠPrinc eton +ĠAndre a +Ġincorrect ly +Ġearn est +Ġpil gr +ĠS print +Ġslee ve +Ġhe ars +ĠAm azing +Ġbrow sing +ag in +Ġhom eland +Ġha w +Ġd iving +ist ered +17 8 +Ġbarg aining +ĠArc ade +Ġdeleg ate +ters on +................................ ................................ +ĠJackson ville +27 5 +Ġst agn +Ġad am +ĠSher man +C B +Ġsub urb +ĠFood s +Ġconver ting +ĠAr ist +Ġch ambers +l ove +Ġam ino +ĠG an +Ġmad ness +m c +ĠUS E +def ined +Ġul tr +ind ust +Ġw olves +l ance +Add itionally +Ġcr acks +as ia +ĠRe ason +ĠP ump +Ġaccident al +ĠL aser +ĠR id +Ġinitial ized +ell i +Ġun named +Ġn oun +ĠPass ed +Ġhost age +ĠEth iop +sh irts +Ġun rel +ĠEmb assy +Ġ19 41 +Ġat oms +Ġpur ported +16 4 +ĠF i +Ġgall ons +ĠMon ica +Ġp g +en ment +Ġsort ed +ĠG ospel +Ġhe ights +Ġtr aced +Ġunder going +She ll +Ġs acks +Ġproport ions +Ġhall uc +F ont +ac et +Ġwar mer +ĠIN TER +Ġgrab bing +Pl ug +Ġreal ization +ĠBur ke +Ġen chant +AT ER +ĠSe ed +Ġabund ant +F M +Ġc ivic +V s +is i +Ġv ow +Ġre per +ĠPartners hip +Ġpenet ration +Ġax e +Ġsh attered +ĠZ ombies +Ġv inyl +ĠAl ert +e on +Ġoblig ed +ĠIll ust +ĠPl aza +ĠFront ier +Ġdavid jl +ĠSer ial +ĠH av +ĠNut rition +B i +Ġâĸ Ī +ĠJ ays +lin ux +Ġhur ry +Ġv oy +Ġhop eless +ĠSte alth +Ġ ãģ +ess ors +tt le +b org +ĠSaf ari +f ell +Ġw ary +d ue +ĠAb ove +H a +E LL +Ġnot or +ĠW on +T oo +Ġoccup ations +Ġposs essions +Ġinv iting +Ġpred ators +Ġacceler ated +Ġ15 7 +uter te +ĠC ube +e ast +acc ount +G ive +Ġtrans plant +red ients +id able +Ġscreens hots +ĠG und +ĠF S +Ġtravel ers +Ġsens ory +ĠF iat +ĠRock ets +İ ĭ +_ { +F riend +Ġchar ming +AL S +Ġenjoy ment +m ph +Ġ5 000 +ĠRE G +Ù Ĩ +b ia +Ġcomp ilation +ro st +ĠV P +ĠSch ne +201 9 +Ġcop ying +M ORE +ĠFl ore +f alls +2 15 +t otal +Ġdis ciples +d ouble +Ġexceed ing +Ġsm ashed +Ġconcept ual +ĠRom ania +ĠB rent +ĠI CE +ĠT ou +Ġg rap +Ġn ails +18 9 +ãĥ ĺ +Ġproc ure +e ur +Ġconfir ming +ĠC ec +aw i +ĠEd en +Ġn g +Ġengine ered +at ics +Ġhook ed +Ġdisgust ing +ĠMur der +ãĤ ¿ +L ibrary +Ġ16 8 +Al most +hem atic +Men u +ĠNot re +ĠJ ur +Ġkidn apped +Ġhack er +ĠJ ade +Ġcreep y +Ġdraw ings +ĠSpons or +Ġcycl ists +ĠGob lin +Ġoptim ized +Ġst aged +ĠMc D +bet ween +A ge +en o +S ex +ĠW ide +n ings +av is +Ġincap able +ĠK ob +Ġreward ing +ĠL one +oles cent +Ġcontract ed +Ġstick y +J ose +B all +f est +ĠIn put +ĠRec ently +Ġto mat +squ are +App lication +Ġnit rogen +Ġdupl icate +ĠRec on +ĠD ear +L ondon +Ġint ra +Ġd ock +Ġout reach +ĠM illion +Ġmamm als +am pton +V AL +Ġsn aps +Ġd os +ĠWh ole +ĠRead y +T ry +ĠWinn ipeg +ear ance +Ġinc urred +ren ched +ĠNS W +il ot +rain e +Ġc ube +g ot +Ġrun way +etermin ed +ĠHaw ks +Ġsurviv or +ĠW ish +ĠD in +ĠDE F +ĠV ault +18 7 +Ġmush rooms +Ġcris p +be y +ĠDisco very +Ġdevelopment al +Ġparad igm +Ġcha otic +ĠT su +Ġ3 33 +b ons +Ġbacter ial +Ġcomm its +Ġcos mic +Ġme ga +oc ative +ĠP aint +ophob ic +Ġv ain +Ġcar ved +ĠTh ief +ĠG ul +ows hip +Ġc ites +ĠEd inburgh +Ġdimin ished +Ġacknowled ges +ĠK ills +Ġmic row +ĠHer a +Ġsen iors +Ġwhere by +H op +at ron +Ġun available +ĠN ate +Ġ4 80 +Ġsl ated +ĠRe becca +ĠB attery +Ġgram mar +Ġhead set +Ġcurs or +Ġex cluding +any e +aunder ing +eb in +Ġfeas ible +ĠPub lishing +ĠLab s +ĠCl iff +ĠFerr ari +Ġp ac +vis ible +mark ed +pe ll +Ġpol ite +Ġstagger ing +ĠGal actic +Ġsuper st +Ġpar an +ĠOffic ers +ãĢ ģ +Ġspecific s +ul us +23 9 +ĠP aste +AM P +ĠPan ama +ĠDe lete +angu ard +rest rial +Ġhero ic +ĠD y +ا ÙĦ +Ġincumb ent +Ġcr unch +t ro +Ġsc oop +Ġblog ger +Ġsell ers +ure n +Ġmedic ines +ĠC aps +ĠAnim ation +ox y +Ġout ward +Ġinqu iries +22 9 +Ġpsych ologist +ĠS ask +ev il +Ġcontam inated +ãĤ ¨ +he rence +Ġbrand ed +ĠAbd ul +z h +Ġparagraph s +Ġmin s +Ġcor related +er b +Ġimp art +Ġmil estone +ĠSol utions +ot le +Ġunder cover +Ġmar ched +ĠCharg ers +f ax +ĠSec rets +Ġr uth +we ather +Ġfemin ine +Ġsh am +Ġprest igious +igg ins +Ġs ung +hist ory +ett le +gg ie +Ġout dated +ol and +Ġper ceptions +ĠS ession +ĠDod gers +u j +ĠE ND +D oc +Ġdefic iency +Gr and +ĠJ oker +Ġretro spect +Ġdiagn ostic +Ġharm less +Ġro gue +ĠA val +E qu +Ġtrans c +ĠRoberts on +ĠDep ending +ĠBurn s +iv o +Ġhost ility +F eatures +ĵ ĺ +Ġdis comfort +ĠL CD +spec ified +ĠEx pect +3 40 +Ġimper ative +ĠReg ular +Ch inese +Ġstate wide +Ġsy mm +Ġlo ops +Ġaut umn +N ick +Ġsh aping +Ġqu ot +Ġc herry +ĠCross ref +è¦ ļéĨĴ +Stand ard +he ed +ĠD ell +ĠViet namese +Ġo st +ĠV alkyrie +O A +Ass ad +Ġreb ound +ĠTra ffic +pl aces +æ ĺ +ĠB uc +17 2 +Ġshel ters +Ġins isting +ĠCertain ly +ĠKenn eth +ĠT CP +Ġpen al +ĠRe play +he ard +Ġdial ect +iz a +ĠF Y +it cher +ĠD L +Ġspir al +Ġquarterback s +Ġh ull +Ġgo ogle +Ġto dd +ĠSter ling +ĠPl ate +Ġsp ying +mb ol +ĠReal m +ĠPro ced +ĠCr ash +Ġtermin ate +Ġprotest ing +C enter +gu ided +Ġun cover +Ġboy cott +Ġreal izes +s ound +Ġpret ending +ĠV as +19 80 +Ġfram ed +Ġ13 9 +Ġdesc ended +Ġrehab ilitation +Ġborrow ing +ĠB uch +Ġbl ur +R on +ĠFro zen +en za +Ch ief +ĠP oor +Ġtransl ates +M IN +Ġ2 12 +J ECT +Ġerupt ed +Ġsuccess es +S EC +Ġpl ague +Ġg ems +d oms +Ġstret ches +ĠSp y +Ġstory telling +C redit +ĠP ush +Ġtra ction +Ġin effective +ĠL una +Ġt apes +Ġanaly tics +erc ise +Ġprogram mes +ĠCar bon +Ġbeh old +he avy +ĠConserv ation +ĠF IR +Ġs ack +ter min +ric ks +Ġhous ed +Ġunus ually +I ce +Ġexecut ing +ĠMor oc +ed ay +Ġed itions +Ġsm arter +ĠB A +Ġout law +Ġvan ished +ib a +AL SE +ĠSil va +23 8 +C ould +Ġphilos opher +Ġevac uated +Sec ret +14 2 +Ġvis as +ãĤ ¬ +ĠM alt +ĠClear ly +ĠN iger +ĠC airo +ĠF ist +3 80 +ĠX ML +aut o +it ant +Ġrein forced +Rec ord +ĠSurviv or +G Hz +Ġscrew s +parent s +Ġo ceans +ma res +Ġbra kes +vas ive +Ġhell o +ĠS IM +rim p +Ġo re +ĠArm our +24 7 +Ġterr ific +Ġt ones +14 1 +ĠMin utes +Ep isode +Ġcur ves +Ġinflamm atory +Ġbat ting +ĠBeaut iful +L ay +Ġunp op +v able +Ġr iots +ĠTact ics +b augh +ĠC ock +Ġorg asm +ĠS as +Ġconstruct or +et z +G ov +Ġant agon +Ġthe at +Ġde eds +ha o +c uts +ĠMc Cl +Ġu m +ĠScient ists +Ġgrass roots +ys sey +"] => +Ġsurf aced +Ġsh ades +Ġneighb ours +Ġad vertis +oy a +Ġmer ged +Up on +Ġg ad +Ġanticip ate +Any way +Ġsl ogan +Ġdis respect +I ran +ĠT B +act ed +Ġsubp oen +medi ately +OO OO +Ġwa iver +Ġvulner abilities +ott esville +ĠHuff ington +J osh +ĠD H +M onday +ĠEll en +K now +x on +it ems +22 8 +Ġf ills +ĠN ike +Ġcum ulative +and als +I r +Ġ ì +Ġfr iction +ig ator +Ġsc ans +ĠVi enna +ld om +Ġperform ers +P rim +Ġb idding +M ur +Ġlean ed +ĠPri x +al ks +Ġ[ âĢ¦] +ĠTw itch +ĠDevelop er +ĠG ir +Ġcall back +Ab stract +Ġacc ustomed +Ġfreed oms +ĠP G +ur acy +Ġl ump +is man +,, ,, +19 92 +ĠR ED +Ġwor m +M atch +ĠPl atinum +I J +ĠOwn er +Tri via +com pl +Ġnew born +Ġfant as +O wn +Ġ19 59 +Ġsymp ath +Ġub iqu +Ġoutput s +Ġal lev +Ġpr ag +K evin +Ġfav ors +Ġbur ial +Ġn urt +so lete +c ache +Ġ15 6 +Ġunl ocks +te chn +M aking +Ġcon quer +ad ic +æ ĸ +Ġel f +Ġelect orate +ĠKurd s +ĠSt ack +ĠSam urai +Ġâ ĺħ +Ġ{ } +ĠS aid +ĠFall out +Ġkind ness +ĠCustom s +ĠBou levard +Ġhelicop ters +ot ics +ĠVe get +com ment +Ġcritic ised +Ġpol ished +ĠRem ix +ĠC ultural +Ġrec ons +Ġdo i +at em +Sc reen +Ġbar red +Com ments +ĠGener ally +Ġsl ap +7 20 +V ari +p ine +Ġem pt +Ġh ats +ĠPlay ing +l ab +a verage +form s +ĠC otton +Ġcan s +ĠD ON +ĠSom alia +C rypt +ĠIncre ases +E ver +mod ern +Ġsur geon +3 000 +Ġrandom ized +================================ ================================ +B ern +im pl +ĠC OR +Ġpro claim +th ouse +Ġto es +Ġam ple +Ġpres erving +Ġdis bel +gr and +B esides +Ġsil k +ĠPat tern +h m +Ġenter prises +Ġaffidav it +ĠAdvis ory +Ġadvert ised +ĠRel igious +se ctions +psy ch +ĠField s +aw ays +Ġhasht ag +ĠNight mare +Ġv ampire +Ġfore nsic +rosso ver +n ar +Ġn avy +Ġvac ant +ĠD uel +Ġhall way +Ġface book +ident ally +ĠN RA +Ġm att +Ġhur ricane +ĠKir by +ĠP uzzle +Ġsk irt +ou st +du llah +Ġanal ogy +in ion +Ġtomat oes +ĠN V +ĠPe ak +ĠMe yer +Ġappoint ments +Ġm asc +Ġal ley +re hend +Ġchar ities +Ġund o +Ġdest inations +ĠTest ing +"> " +c ats +* . +Ġgest ures +gener al +Le ague +Ġpack ets +ĠInspect or +ĠBer g +Ġfraud ulent +Ġcritic ize +F un +Ġbl aming +nd ra +Ġsl ash +ĠE ston +Ġpropos ing +Ġwh ales +Ġtherap ist +Ġsub set +Ġle isure +EL D +ĠC VE +ĠAct ivity +Ġcul min +sh op +ĠD AY +is cher +ĠAdmir al +ĠAtt acks +Ġ19 58 +Ġmem oir +Ġfold ed +Ġsex ist +Ġ15 3 +ĠL I +Ġread ings +Ġembarrass ment +ĠEmploy ment +w art +ch in +Ġcontin uation +l ia +Rec ently +Ġd uel +Ġevac uation +ĠKash mir +Ġdis position +ĠR ig +Ġbol ts +Ġins urers +4 67 +M ex +Ġret aliation +Ġmis ery +Ġunre asonable +r aining +I mm +ĠP U +em er +Ġgen ital +ãĤ ³ +ĠC andy +Ġon ions +ĠP att +lin er +Ġconced ed +Ġf a +Ġfor c +ĠH ernandez +ĠGe off +deb ian +ĠTe ams +Ġc ries +Ġhome owners +23 7 +A BC +Ġst itch +Ġstat istic +Ġhead ers +ĠBi ology +Ġmot ors +ĠG EN +ĠL ip +Ġh ates +Ġhe el +S elf +i pl +ED IT +ort ing +Ġann ot +ĠSpe ech +old emort +ĠJ avascript +ĠLe Bron +Ġfoot print +Ġf n +Ġseiz ures +n as +h ide +Ġ19 54 +ĠBe e +ĠDecl aration +ĠKat ie +Ġreserv ations +N R +f emale +Ġsatur ated +Ġb iblical +Ġtroll s +Dev ice +ph otos +Ġdr ums +ãĥīãĥ© ãĤ´ãĥ³ +N ight +f ighter +ĠH ak +ri ber +Ġc ush +Ġdiscipl inary +ba um +ĠG H +ĠSch midt +ilib rium +Ġs ixty +ĠKush ner +ro ts +Ġp und +ĠR ac +Ġspr ings +Ġcon ve +Bus iness +F all +Ġqual ifications +Ġvers es +Ġnarc iss +ĠK oh +ĠW ow +ĠCharl ottesville +ed o +Ġinterrog ation +ĠW ool +36 5 +B rian +Ġâľ ĵ +Ġalleg es +ond s +id ation +ĠJack ie +y u +Ġl akes +Ġworth while +Ġcryst als +ĠJud a +Ġcomp rehend +Ġfl ush +Ġabsor ption +ĠO C +Ġfright ened +ĠCh ocolate +Mart in +Ġbu ys +Ġbu cks +Ġapp ell +ĠChampions hips +Ġlist ener +ĠDef ensive +Ġc z +ud s +ĠM ate +Ġre play +Ġdecor ated +Ġs unk +ĠV IP +ĠAn k +Ġ19 5 +aa aa +Nob ody +ĠMil k +ĠG ur +ĠM k +ĠS ara +Ġse ating +ĠW id +Tr ack +Ġemploy s +Ġgig antic +AP P +ãĤ § +in ventory +Ġtow el +at che +l asting +ĠT L +Ġlat ency +Ġkn e +B er +me aning +Ġup held +Ġplay ground +Ġm ant +S ide +Ġstere o +Ġnorth west +Ġexception ally +Ġr ays +Ġrec urring +D rive +Ġup right +Ġab duct +ĠMar athon +Ġgood bye +Ġal phabet +h p +Ġcourt room +ring ton +ot hing +T ag +Ġdiplom ats +Ġbar bar +ĠAqu a +18 3 +33 33 +Ġmat urity +Ġinst ability +ĠAp ache +Ġ= == +Ġfast ing +ĠGr id +Mod Loader +Ġ15 2 +A bs +ĠOper ating +ett i +Ġacqu aint +Don nell +ĠK em +ĠFor ge +Ġarm ored +M il +Ġphilos ophers +in vest +Pl ayers +â Ī +Ġmy riad +Ġcomr ades +R ot +Ġremember ing +Ġcorrespond s +Ġprogram mers +ĠLyn n +Ġo lig +Ġco herent +yn chron +ĠChem ical +Ġj ugg +p air +post s +E ye +ĠIn ner +Ġsem ester +ott est +ĠEmir ates +ric anes +or ously +m its +ĠW is +Ġd odge +l ocation +Ġf aded +Am azon +ĠPro ceed +ĠIN FO +j ournal +ĠTru ck +T en +Ġ2 17 +Ġstat utes +m obile +ĠT ypes +Rec omm +b uster +pe x +Ġleg ends +Ġhead ache +f aced +ĠWi Fi +if ty +ĠH ER +Ġcirc uits +ER ROR +22 6 +ol in +Ġcyl inder +osp ace +ik ers +P rem +Qu ant +Ġconflic ting +Ġslight est +Ġfor ged +ion age +Step hen +ĠK ub +ĠOpp ortun +ĠHe al +Ġbl o +Ġrul ers +Ġh uh +Ġsubmar ine +f y +ass er +Ġallow ance +ĠKas ich +ĠT as +ĠAustral ians +Forge ModLoader +ĠâĨ ij +ĠMat rix +am ins +Ġ12 00 +ĠAc qu +23 6 +D ocument +ĠBre aking +19 3 +ĠSub st +ĠRoll er +ĠPro perties +ĠN I +t ier +Ġcr ushing +Ġadvoc ating +Further more +keep ers +Ġsex ism +x d +Ġcall er +ĠS ense +chie ve +ĠT F +Ġfuel ed +Ġreminis cent +Ġobs ess +ur st +Ġup hold +ĠF ans +het ics +Ġâ Ĺ +ĠB ath +Ġbe verage +Ġo scill +25 4 +Ġpol es +Ġgrad ual +Ġex ting +ĠS uff +ĠS uddenly +Ġlik ing +Ġ19 49 +un ciation +am ination +ĠO mar +ĠL V +ĠCon sequently +Ġsynt hes +ĠG IF +Ġp ains +Ġinteract ing +u ously +inc re +Ġrum or +ĠScient ology +19 7 +ĠZ ig +Ġspe lling +ĠA SS +Ġexting u +ms on +Ġg h +Ġremark ed +ĠStrateg ic +ĠM ON +å ¥ +g ae +ĠWH AT +E ric +ĠCamp us +Ġmeth ane +Ġimag in +J UST +ĠAl m +X T +i q +ĠR SS +Ġwrong doing +att a +Ġbig ot +Ġdemonstr ators +ĠCal vin +ĠV illa +Ġmembr ane +ĠAw esome +Ġbenef ic +26 8 +Ġmagn ificent +ĠL ots +G reg +ĠBor is +Ġdetain ees +ĠH erman +Ġwhis pered +Ġa we +Prof essor +fund ing +Ġphys iological +ĠDest ruction +Ġlim b +Ġmanip ulated +Ġbub bles +Ġpse ud +Ġhyd ra +ĠBrist ol +Ġst ellar +ĠExp ansion +ĠK ell +ĠInterest ingly +Ġm ans +Ġdrag ging +Ġec ological +ĠF it +Ġg ent +Ġbenef ited +ĠHait i +Ġpoly g +ãĥ İ +Ġ20 30 +Ġpro w +Ġrecon struction +Ġwas t +Ġpsych ic +ĠGree ks +Hand ler +16 2 +ĠP ulse +Ġsol icit +Ġsy s +Ġinflu x +ĠG entle +per cent +Ġprolifer ation +Ġtax able +Ġdisreg ard +Ġesc aping +Ġg inger +Ġwith stand +Ġdevast ated +ĠD ew +ser ies +Ġinject ed +ela ide +Ġturn over +he at +Ļ Ĥ +H appy +ĠSil ent +ãĤ Ń +iv ism +Ġir rational +AM A +Ġre ef +r ub +Ġ16 2 +Ġbank ers +ĠEth ics +v v +Ġcritic isms +K n +18 6 +M ovie +ĠT ories +Ġno od +Ġdist ortion +F alse +od ore +Ġt asty +Res earch +ĠU ID +- ) +Ġdivor ced +ĠM U +ĠHay es +ĠIs n +ian i +ĠH Q +Ġ" # +ign ant +Ġtra umatic +ĠL ing +H un +Ġsab ot +on line +r andom +Ġren amed +ra red +K A +d ead +é t +ĠAss istance +Ġse af +++++ ++++ +Ġse ldom +ĠWeb b +Ġbo olean +u let +Ġref rain +ĠDI Y +ru le +Ġshut ting +Ġutil izing +load ing +ĠPar am +co al +oot er +Ġattract ing +ĠD ol +Ġher s +ag netic +ĠRe ach +im o +Ġdisc arded +ĠP ip +01 5 +ü r +Ġm ug +Im agine +C OL +Ġcurs ed +ĠSh ows +ĠCurt is +ĠSach s +spe aking +ĠV ista +ĠFram ework +ong o +Ġsub reddit +Ġcr us +ĠO val +R ow +g rowing +Ġinstall ment +Ġgl ac +ĠAdv ance +EC K +ĠLGBT Q +LE Y +Ġac et +Ġsuccess ive +ĠNic ole +Ġ19 57 +Qu ote +Ġcircumst ance +ack ets +Ġ14 2 +ort ium +Ġguess ed +ĠFr ame +Ġperpet rators +ĠAv iation +ĠBen ch +Ġhand c +A p +Ġ19 56 +25 9 +r and +Net Message +d in +urt les +h ig +ĠV III +ff iti +ĠSw ords +b ial +Ġkidn apping +dev ice +Ġb arn +ĠEl i +auc as +S end +Con structed +Ġ ½ +Ġneed les +Ġad vertisements +Ġv ou +Ġexhib ited +ĠFort ress +As k +B erry +TY PE +Ġcan cers +ump ing +ĠTerrit ory +Ġpr ud +Ġn as +Ġathe ist +Ġbal ances +ãģ Ł +ĠSh awn +& & +Ġland sc +ĠR GB +Ġpet ty +Ġex cellence +Ġtransl ations +Ġpar cel +ĠChe v +E ast +ĠOut put +im i +Ġamb ient +ĠTh reat +Ġvill ains +Ġ5 50 +IC A +Ġtall er +Ġle aking +c up +Ġpol ish +Ġinfect ious +ĠK C +Ġ@ @ +back ground +Ġbureaucr acy +ĠS ai +un less +it ious +ĠSky pe +At l +ID ENT +00 8 +Ġhyp ocr +Ġpit chers +Ġguess ing +ĠF INAL +Bet ween +Ġvill agers +Ġ25 2 +f ashion +ĠTun is +Be h +ĠEx c +ĠM ID +28 8 +ĠHas kell +19 6 +ĠN OR +Ġspec s +Ġinv ari +Ġgl ut +ĠC ars +Ġimp ulse +Ġhon ors +g el +Ġjurisd ictions +ĠBund le +ul as +Calif ornia +ĠIncre ase +Ġp ear +Ġsing les +Ġc ues +Ġunder went +ĠW S +Ġexagger ated +Ġdub ious +Ġfl ashing +L OG +) ]. +J ournal +t g +V an +ĠI stanbul +ĠIn sp +ĠFrank en +D raw +Ġsad ness +Ġiron ic +ĠF ry +x c +Ġ16 4 +is ch +W ay +ĠProtest ant +h orn +Ġun aff +ĠV iv +ill as +ĠProduct ions +ĠH ogan +Ġper imeter +ĠS isters +Ġspont aneous +Ġdown side +Ġdescend ants +Ġor n +w orm +Japan ese +Ġ19 55 +Ġ15 1 +ĠDo ing +els en +umb les +Ġrad ically +ĠDr um +ĠB ach +Ġli abilities +ĠO B +ĠElement ary +Ġmem e +yn es +Ġfinger print +ĠGr ab +Ġundert ake +Mem bers +ĠRead er +ĠSim s +g od +Ġhypot hetical +s cient +ĠA J +Ġchar ism +Ġad missions +ĠMiss ile +tr ade +Ġexerc ising +ĠBack ground +W ritten +Ġvoc als +whe ther +Ġv i +ĠW inner +Ġl itter +ĠSh ooting +ST EM +ãĤ ¡ +ĠA FL +Ġvari ability +Ġe ats +ĠD PS +b row +Ġeleph ants +Ġstr at +Ġ Å +Ġsett lers +Matt hew +Ġin advert +H I +ĠIM F +ĠGo al +Ġnerv es +John son +ey e +ablish ment +Th ursday +BIL ITY +H ad +am oto +het amine +ep s +Ġmit ochond +Ġcomp ressed +ĠTre vor +ĠAnim als +T ool +L ock +Ġtwe ak +Ġpin ch +Ġcancell ation +P ot +Ġfoc al +ĠAst ron +17 3 +ĠA SC +ĠO THER +umn i +Ġdem ise +d l +Ù ħ +Sem itism +Ġcr acking +Ġcollabor ative +Ġexpl ores +s ql +Ġher bs +Ġconfig urations +m is +ĠRes ult +ace y +ĠSm oke +Ġsan ct +el ia +Ġdeg ener +Ġdeep est +Ġscream ed +Ġn ap +Soft ware +ĠST AR +E F +ĠX in +spons ored +mans hip +23 3 +Ġprim aries +Ġfilter ing +Ġas semble +m il +ĠMy ers +b ows +Ġpun ched +M ic +Ġinnov ations +Ġfun c +and o +Ġfr acking +ĠV ul +о Ð +osh op +ĠIm mun +Ġsett ling +Ġadolesc ents +Ġreb uilding +Ġtransform ing +Ġpar ole +Ġhar bor +Ġbook ing +ot ional +onge vity +ĠY o +b ug +Ġemer ges +ĠMethod s +ĠCh u +P res +ĠDun geons +Ġtra iling +ĠR um +ĠH ugh +å¤ © +ĠE ra +ĠBatt les +Res ults +ĠTr ading +Ġvers a +c ss +ax ies +he et +Ġgre ed +19 89 +Ġgard ens +Ġconting ent +P ark +ĠLeaf s +h ook +ro be +Ġdiplom acy +ĠF uel +ĠInv asion +Ġupgr ading +M ale +Ġe lic +Ġrelent less +ĠCo venant +ap esh +ĠT rop +T y +pro duction +art y +Ġpun ches +ak o +cyclop edia +ĠR abbit +ĠHD MI +Ġ14 1 +Ġf oil +Item Image +ĠF G +Ġimplement ations +ĠP om +ixt ures +Ġaw ait +Ġ3 30 +am us +Ġumb rella +Ġfore see +se par +Ġcircum cision +Ġperipher al +S ay +ĠExper t +In c +Ġwithd rew +ĠAnd ers +f ried +Ġradio active +ĠOp ening +Ġboard ing +ĠN D +Ġover throw +Act iv +W P +ĠAct s +× Ļ +Ġmot ions +v ic +ĠM ighty +ĠDef ender +a er +Ġthank ful +ĠK illing +ĠBr is +mo il +Ġpredict ing +26 6 +ch oice +Ġkill ers +Ġinc ub +ĠChe st +ather ing +Ġpro claimed +fl ower +oss om +umbled ore +ĠCy cling +ĠOccup y +AG ES +P en +ĠY ug +Ġpack aged +Ġheight ened +c ot +st ack +C ond +Ġst amps +m age +Ġpersu aded +Ġens l +ĠCard inal +Ġsol itary +Ġpossess ing +ĠC ork +Ġev id +ĠT ay +Ġbl ues +Ġextrem ism +Ġlun ar +Ġcl own +Te chn +Ġfest ivals +ĠPv P +ĠL ar +Ġconsequ ently +p resent +Ġsom eday +ç İĭ +ĠMet eor +Ġtour ing +c ulture +Ġbe aches +S hip +c ause +ĠFl ood +ãĥ ¯ +Ġpur ity +th ose +Ġem ission +b olt +Ġch ord +ĠScript ure +L u +Ġ$ { +cre ated +Other s +25 8 +Ġelement al +Ġannoy ed +ĠA E +d an +ĠS ag +Res earchers +Ġfair y +âĢĵ âĢĵ +======== ==== +Sm art +GG GG +Ġskelet ons +Ġpup ils +link ed +Ġur gency +en abled +ĠF uck +Ġcoun cill +r ab +U AL +T I +Ġlif es +Ġconf essed +B ug +Ġharm on +ĠCON FIG +ĠNe utral +D ouble +Ġst aple +ĠSH A +Brit ish +ĠSN P +AT OR +oc o +Ġswing ing +ge x +ole on +pl ain +ĠMiss ing +ĠTro phy +v ari +ran ch +Ġ3 01 +4 40 +00000000 00000000 +Ġrest oring +Ġha ul +uc ing +ner g +Ġfut ures +Ġstrateg ist +quest ion +Ġlater al +ĠB ard +Ġs or +ĠRhod es +ĠD owntown +????? - +ĠL it +ĠB ened +Ġco il +st reet +ĠPort al +FI LE +ĠG ru +* , +23 1 +ne um +Ġsuck ed +Ġr apper +Ġtend encies +ĠLaure n +cell aneous +26 7 +Ġbrow se +Ġover c +head er +o ise +Ġbe et +ĠG le +St ay +Ġm um +Ġtyp ed +Ġdiscount s +T alk +ĠO g +ex isting +ĠS ell +u ph +C I +ĠAust rian +ĠW arm +Ġdismiss al +Ġaver ages +c amera +Ġalleg iance +L AN +=" # +Ġcomment ators +ĠSet ting +ĠMid west +Ġpharm ac +ĠEX P +Ġstain less +Ch icago +Ġt an +24 4 +Ġcountry side +ĠV ac +29 5 +Ġpin ned +Ġcr ises +Ġstandard ized +T ask +ĠJ ail +ĠD ocker +col ored +f orth +" }, +Ġpat rons +Ġsp ice +Ġm ourn +ĠM ood +Ġlaund ry +Ġequ ip +ĠM ole +y ll +ĠTH C +n ation +ĠSher lock +Ġiss u +ĠK re +ĠAmeric as +ĠA AA +Ġsystem atically +Ġcont ra +ĠS ally +Ġrational e +Ġcar riage +Ġpe aks +Ġcontrad iction +ens ation +ĠFail ure +Ġpro ps +Ġnames pace +Ġc ove +field s +ãĤ ĭ +Ġw ool +ĠC atch +Ġpresum ed +ĠD iana +r agon +ig i +Ġh amm +Ġst unt +ĠG UI +ĠObserv atory +ĠSh ore +Ġsmell s +ann ah +Ġcock pit +ĠD uterte +8 50 +Ġopp ressed +bre aker +ĠCont ribut +ĠPer u +ĠMons anto +ĠAtt empt +Ġcommand ing +Ġfr idge +ĠR in +ĠChe ss +ual ity +Ġo l +Republic an +ĠGl ory +ĠW IN +.... ... +ag ent +read ing +Ġin h +J ones +Ġcl icks +al an +Ġ[ ]; +ĠMaj esty +ĠC ed +op us +ate l +à ª +AR C +ĠEc uador +ãĥ ł +ĠK uro +Ġritual s +Ġcapt ive +Ġoun ce +Ġdisag reement +Ġsl og +f uel +P et +M ail +Ġexerc ised +Ġsol ic +Ġrain fall +Ġdev otion +ĠAss essment +Ġrob otic +opt ions +ĠR P +ĠFam ilies +ĠFl ames +Ġassign ments +00 7 +aked own +Ġvoc abulary +Re illy +Ġc aval +g ars +Ġsupp ressed +ĠS ET +ĠJohn s +Ġwar p +bro ken +Ġstat ues +Ġadvoc ated +Ġ2 75 +Ġper il +om orph +ĠF emin +per fect +Ġh atch +L ib +5 12 +Ġlif elong +3 13 +Ġche eks +Ġnum bered +ĠM ug +B ody +ra vel +We ight +ĠJ ak +ĠHe ath +Ġkiss ing +ĠJ UST +Ġw aving +u pload +Ġins ider +ĠPro gressive +ĠFil ter +tt a +ĠBe am +Ġviol ently +ip ation +Ġskept icism +Ġ19 18 +ĠAnn ie +ĠS I +Ġgen etics +Ġon board +at l +ĠFried man +ĠB ri +cept ive +Ġpir ate +ĠRep orter +27 8 +Ġmyth ology +Ġe clipse +Ġsk ins +Ġgly ph +ing ham +F iles +C our +w omen +Ġreg imes +Ġphotograp hed +K at +ĠMA X +Offic ials +Ġunexpected ly +Ġimpress ions +F ront +;;;; ;;;; +Ġsuprem acy +Ġs ang +Ġaggrav ated +Ġabrupt ly +ĠS ector +Ġexc uses +Ġcost ing +ide press +St ack +ĠR NA +ob il +Ġghost s +ld on +at ibility +Top ics +Ġreim burse +ĠH M +ĠDe g +Ġth ief +y et +ogen esis +le aning +ĠK ol +ĠB asketball +Ġf i +ĠSee ing +Ġrecy cling +Ġ[ - +Cong ress +Ġlect ures +P sy +Ġne p +Ġm aid +Ġori ented +A X +Ġrespect ful +re ne +fl ush +ĠUn loaded +re quest +gr id +ĠAltern atively +ĠHug o +Ġdec ree +ĠBuddh ism +and um +And roid +ĠCong o +ĠJoy ce +Ġacknowled ging +hes ive +ĠTom orrow +ĠH iro +th ren +ĠM aced +Ġho ax +ĠIncre ased +ĠPr adesh +W ild +____ __ +16 1 +Ġa unt +Ġdistribut ing +ĠT ucker +ĠSS L +ĠW olves +B uilding +ou lt +ĠLu o +ĠY as +ĠSp ir +ĠSh ape +ĠCamb od +ĠIP v +Ġm l +Ġext rad +39 0 +ĠPenn y +d ream +Ġstation ed +opt ional +ew orthy +. +ĠWorks hop +ĠRet ail +ĠAv atar +6 25 +N a +ĠV C +ĠSec ure +M Y +19 88 +oss ip +Ġpro state +Ġund en +Ġg amer +ĠCont ents +ĠWar hammer +ĠSent inel +3 10 +Ġse gregation +ĠF lex +ĠM AY +Ġdr ills +ĠDrug s +Islam ic +Ġsp ur +Ġca fe +Ġimag inary +Ġgu iding +Ġsw ings +ĠThe me +ob y +Ġn ud +Ġbe gging +Ġstr ongh +Ġreject ing +Ġpedest rians +ĠPro spect +R are +s le +Ġconcess ions +ĠConst itutional +Ġbe ams +Ġfib ers +p oon +Ġinstinct s +pro perty +ĠB IG +Sand ers +im ates +Ġco ating +Ġcorps es +ĠTR UE +check ed +Ġ16 6 +A sh +ĠJ S +ĠF iction +Ġcommun al +Ġener getic +oooo oooo +Ġnow adays +IL D +ib o +ĠSU V +R en +Ġdwell ing +Sil ver +Ġt ally +ĠM oving +Ġcow ard +Ġgener als +Ġhorn s +Ġcirc ulated +Ġrob bed +ĠUn limited +Ġharass ed +Ġinhib it +Ġcomp oser +ĠSpot ify +Ġspread s +3 64 +Ġsu icidal +Ġno ises +ĠSt ur +Ġs aga +ĠK ag +is o +Ġtheoret ically +M oney +Ġsimilar ity +Ġslic ed +ut ils +ing es +" - +Ġan th +Ġimp ed +Mod ule +Through out +Ġmen us +comm ittee +and i +ob j +in av +f ired +ĠAb dullah +Ġund ead +Ġfont s +H old +EN G +Ġsustain ability +Ġfl ick +Ġr azor +ĠF est +ĠChar acters +Ġword ing +Ġpopul ist +Ġcritic izing +Ġm use +v ine +Ġcard board +Ġkind ly +Ġfr inge +ĠThe ft +icult ural +Ġgovern ors +Ġ ���� +Ġ16 3 +Ġtime out +ĠA uth +Child ren +A U +Ġred emption +ĠAl ger +Ġ19 14 +Ġw aved +Ġastron auts +og rams +Ġsw amp +ĠFinn ish +Ġcand le +Ġton nes +ut m +Ġr ay +Ġsp un +Ġfear ful +art icles +Ġca us +or ically +ĠRequ ires +ĠG ol +Ġpop e +Ġinaug ural +Ġg le +AD A +ĠIS IL +ĠOff ensive +Ġwatch dog +Ġbal con +ent ity +ĠH oo +Ġgall on +AC C +Ġdoub ling +Ġimpl ication +ĠS ight +Ġdoct r +---- --- +Ġ\ \ +Ġm alt +R oll +Ġâī ¥ +Ġrec ap +add ing +u ces +ĠB end +fig ure +Ġtur key +Ġsoc ietal +ĠT ickets +Ġcommer cially +Ġsp icy +Ġ2 16 +ĠR amp +Ġsuperior ity +à ¯ +ĠTr acker +C arl +ĠC oy +ĠPatri ot +Ġconsult ed +Ġlist ings +Ġsle w +reens hot +ĠG one +Ġ[ ...] +30 9 +Ġh ottest +Ø ± +Ġrock y +ĠD iaz +Ġmass age +Ġpar aly +Ġp ony +A z +Ġcart ridge +ĠN Z +Ġsn ack +ĠLam ar +ple ment +ĠLes lie +Ġm ater +Ġsn ipp +24 6 +Ġjoint ly +ĠBris bane +ĠiP od +Ġpump ing +Ġgo at +ĠSh aron +eal ing +Ġcor on +Ġan omal +rah im +ĠConnect ion +Ġsculpt ure +Ġsched uling +ĠD addy +at hing +Ġeyeb rows +Ġcur ved +Ġsent iments +Ġdraft ing +D rop +( [ +Ġnom inal +ĠLeaders hip +ĠG row +Ġ17 6 +Ġconstruct ive +iv ation +Ġcorrupt ed +ger ald +ĠC ros +ĠChe ster +ĠL ap +ãģ ª +OT H +D ATA +Ġal mond +pro bably +I mp +Ġfe ast +ĠWar craft +F lor +Ġcheck point +Ġtrans cription +Ġ20 4 +Ġtwe aks +Ġrel ieve +S cience +Ġperform er +Z one +Ġtur moil +ig ated +hib it +ĠC afe +the med +Ġflu or +ben ch +Ġde com +ĠU nt +ĠBar rett +ĠF acts +Ġt asting +ĠPTS D +ĠSe al +ĠJuda ism +ĠDynam ic +ĠC ors +V e +ĠM ing +ĠTrans form +v on +ĠDef enders +ĠTact ical +ĠV on +ĠUn ivers +Ġdist orted +ĠB reath +?' " +Ġag on +ĠDead ly +Ġl an +ĠCy cle +orn ed +Ġrel iably +Ġgl or +ĠMon key +ãĥ ¡ +Ġad ren +Ġmicrow ave +ĠAl ban +irc raft +dig it +sm art +ĠD read +¯¯¯¯¯¯¯¯ ¯¯¯¯¯¯¯¯ +{ { +ĠRoc hester +Ġsimpl ified +Ġinf licted +Ġtake over +Ġyour selves +ad itional +Ġmus cular +K S +Ġing en +T ax +ĠFe ature +27 7 +Ġcru c +Ġcr ate +Ġun identified +Ġacclaim ed +ĠM anga +ĠFr ances +ĠNep al +ĠG erald +ĠKu wait +Ġsl ain +ĠHe b +ĠG oku +ãģ® æ +28 6 +M rs +ĠC ody +ĠSan ctuary +01 6 +Ġdism ant +Ġdatas et +ĠH ond +b uck +ĠPat terson +Ġpal ette +ĠG D +ic ol +ĠL odge +Ġplanet ary +ak in +ĠRegist ered +ab we +ĠPeters burg +Ġha iled +ĠP iece +S che +ĠDO J +Ġen umer +18 1 +ĠObs erver +ĠB old +f ounded +com merce +Ġexplo its +ĠF inding +UR N +ĠS ne +ĠAc id +ay ette +ĠVal ues +Ġdr astic +Ġarchitect ural +Ġ" . +× ķ +ump ed +Ġwra pping +Ġwid ow +ĠSl ayer +l ace +on ce +German y +av oid +Ġtem ples +P AR +à ´ +ĠLuc ifer +ĠFl ickr +l ov +for ces +Ġsc outing +Ġlou der +tes y +Ġbefore hand +Ä ĵ +ĠNe on +ĠW ol +ĠTyp ically +ĠPolit ico +-+ -+ +Ġbuild er +Ġder ive +K ill +Ġp oker +Ġambig uous +Ġlif ts +Ġcy t +Ġrib s +ood le +ĠS ounds +h air +ĠSynd rome +t f +Ġproport ional +u id +Ġper taining +ĠKind le +ĠNeg ro +Ġreiter ated +ĠTon ight +oth s +ĠCorn ell +Ġo wing +Ġ20 8 +elf are +oc ating +ĠB irds +Sub scribe +Ġess ays +Ġburd ens +Ġillust rations +ar ious +ER AL +ĠCal cul +Ġx en +ĠLink edIn +ĠJ ung +Ġredes ign +Con nor +29 6 +Ġrevers al +ĠAd elaide +ĠL L +Ġs inking +Ġg um +US H +c apt +ĠGr imm +Ġfoot steps +ĠCB D +isp ers +Ġpro se +Wed nesday +ĠM ovies +ed in +Ġoverturn ed +Ġcontent ious +US B +~~~~~~~~ ~~~~~~~~ +ĠCo pper +Ġpoint less +N V +val ues +olph in +d ain +Ġdepos ited +ĠG W +Ġpreced ed +ĠCl a +ĠGo lem +ĠN im +ĠÎ ² +ĠEngine ers +m iddle +Ġfl att +oper ative +Ġcouncil s +imb abwe +el in +Ġstress ful +ĠL D +Ġres h +l ake +Ġwheel chair +ĠAltern ative +Ġoptim ize +oper ation +Ġpe ek +Ġones elf +ig il +Ġtrans itions +op athy +bl ank +Ġ16 9 +17 1 +________________________________ ________________________________ +Ġl aundering +En c +ĠD EC +Ġwork outs +Ġsp ikes +Ġdin osaurs +Ġdiscrim inatory +P ool +R ather +38 5 +R NA +tes ters +et o +ĠIdent ity +Ġve in +ĠBur ton +Ġarc ade +4 20 +Ult imately +ĠSad ly +à ° +p ill +Ġcub ic +ĠSpect rum +the se +st ates +Ġun official +h awks +ĠEVER Y +Ġrain bow +Ġincarcer ation +and ing +Ġsy ll +ĠEver ton +Ġ17 9 +ĠSer bia +Ġ18 9 +m eter +ĠMic key +Ġant iqu +Ġfact ual +ne ck +ĠN are +n orm +m ust +Ġhigh ways +Ġgl am +Ġdivid ing +ĠSquad ron +ĠMar tha +Ġbirth s +C over +//////// //////// +ĠW ong +Ph ot +ĠA LS +ri o +ĠNon etheless +ĠL emon +Ġ20 6 +ĠE E +Ġderiv ative +ĠWW II +v ote +Ġthere in +Ġsepar ating +44 6 +sy nc +ĠStre ets +Ġr att +Ġmunicip ality +ĠShort ly +Ġmon k +) ," +Ġscr ub +Ġoper atives +Ne ither +Pl ace +ĠLim it +F emale +ĠAct or +Char acter +Ġconstit uted +35 7 +Ġprotest ed +ĠSt raw +ĠHe ight +ild a +ĠTy ph +Ġflood s +Ġcos metic +W AY +pert ure +up on +t ons +ess ing +ĠP ocket +Ġro oft +ĠC aucas +Ġant idepress +Ġincomp atible +EC D +Ġoper a +ĠCont est +Ġgener ators +l ime +Def ense +19 87 +for um +Ġsav age +ĠHung arian +n z +Ġmet allic +Ġex pelled +Ġres idency +Ġdress es +66 6 +ĠC lement +f ires +C ategory +Ġge ek +al is +Ġc emetery +educ ated +Ġc rawl +ĠUn able +ĠT yson +ak is +Ġp ardon +ĠW ra +Ġstrengthen ed +ĠF ors +33 5 +ĠH C +ĠM ond +Ġvisual s +ĠBeat les +ett lement +Ġ ï +g ro +Ġb ash +Ġpo orest +Ġex cel +Ġaspir ations +ĠM unicip +ens ible +Ġceremon ies +Ġintimid ation +ĠCON TR +be ck +ĠK ap +as u +Ġtradem arks +ĠS ew +ĠComp etition +net work +ĠAr ri +ĠT et +Ro aming +W C +D at +Ġso b +Ġpair ing +Ġoverd ose +SA Y +ab er +Ġrev olt +ĠF ah +act ing +e q +est ation +F ight +ĠMar ks +27 3 +Ġ17 8 +R aw +ãģ ĭ +34 9 +bl ocks +Ġver ge +est ine +ĠPod esta +Ġinv asive +Ġprofound ly +ĠA o +e ach +Ġl est +inter pret +Ġshr inking +Ġerr one +Ġche es +ly s +ĠI vy +ĠDirect ory +Ġhint ed +V ICE +Ġcontact ing +ĠG ent +he i +Ġlabel ing +Ġmerc ury +ĠL ite +Ġexp ires +Ġdest abil +rit is +c u +Ġfeather s +Ġste er +Ġprogram med +ĠV ader +Go ing +ĠE lim +Ġy o +ĠMic he +Ġ20 3 +Ġslee ves +Ġb ully +ĠHum ans +36 8 +Ġcomp ress +ĠBan ner +AR S +Ġa while +Ġcal ib +Ġspons orship +ĠDiff iculty +ĠP apers +Ġident ifier +} . +Ġy og +ĠSh ia +Ġclean up +Ġvib e +int rodu +im ming +Austral ia +Ġout lines +ĠY outube +tr ain +ĠM akes +Ġde ported +Ġcent r +ĠD ug +ĠB oulder +ĠBuff y +Ġinj unction +ĠHar ley +ĠG roups +ĠD umbledore +ĠCl ara +Ġ" - +Ġsacrific ed +ep h +Sh adow +ib ling +Ġfreel ance +Ġevident ly +ph al +Ġret ains +M ir +Ġfin ite +d ar +ĠC ous +Ġrep aired +Ġperiod ic +Ġchampions hips +Ġaster oid +bl ind +Ġexpress ly +ĠAst ros +Ġsc aled +Ġge ographical +ĠRap ids +En joy +Ġel astic +ĠMoh amed +Mark et +be gin +Ġdisco vers +Ġtele communications +Ġscan ner +Ġen large +Ġsh arks +Ġpsy chedel +ĠRou ge +Ġsnap shot +is ine +X P +Ġpestic ides +ĠL SD +ĠDist ribution +re ally +Ġde gradation +Ġdisgu ise +Ġbi om +ĠEX T +Ġequ ations +Ġhaz ards +ĠComp ared +) * +Ġvirt ues +Ġeld ers +Ġenh ancing +ĠAc ross +er os +ang ling +Ġcomb ust +ucc i +Ġconc ussion +Ġcontrace ption +ĠK ang +Ġexpress es +Ġa ux +ĠP ione +Ġexhib its +Deb ug +OT AL +ĠAl ready +ĠWheel er +Ġexp ands +? : +Ġreconc iliation +Ġpir ates +Ġpur se +Ġdiscour age +Ġspect acle +R ank +Ġwra ps +ĠTh ought +Ġimp ending +O pp +ĠAng lo +ĠE UR +Ġscrew ed +ret ched +Ġencour agement +mod els +Ġconf use +mm m +ĠVit amin +âĸij âĸij +C ru +Ġkn ights +Ġdisc ard +Ġb ishops +ĠW ear +ĠGar rett +k an +ãĥ Ł +Ġmascul ine +cap ital +ĠA us +Ġfat ally +th anks +ĠA U +ĠG ut +12 00 +Ġ 00000000 +Ġsur rog +ĠBI OS +ra its +ĠWat ts +Ġresur rection +ĠElect oral +ĠT ips +4 000 +Ġnut rient +Ġdepict ing +Ġspr ink +Ġm uff +ĠL IM +ĠS ample +ps c +ib i +gener ated +Ġspec imens +Ġdiss atisf +Ġtail ored +Ġhold ings +ĠMonth ly +ĠE at +po ons +Ġne c +ĠC age +ĠLot us +ĠLan tern +Ġfront ier +Ġp ensions +Ġj oked +ĠHard y +=-=- =-=- +r ade +U ID +Ġr ails +Ġem it +Ġsl ate +Ġsm ug +Ġsp it +ĠCall s +ĠJac obs +f eat +ĠU E +Ġrest ruct +Ġregener ation +Ġenerg ies +ĠCon nor +OH N +ĠChe ese +Ġg er +Ġresur rect +man agement +N W +Ġpres ently +ĠBru ins +M ember +ĠM ang +id an +Ġboost ing +w yn ++ . +requ isite +ĠNY PD +ĠMe gan +ĠCond itions +Ġp ics +nes ium +ĠR ash +Ġ17 4 +ĠD ucks +Ġemb ro +z u +on ian +rel igious +Ġc raz +ĠAC A +ĠZ ucker +EM A +ĠPro s +We apon +ĠKn ox +ĠAr duino +Ġst ove +Ġheaven s +ĠP urchase +Ġher d +Ġfundra iser +Dig ital +5 000 +Ġprop onents +/ âĢĭ +Ġj elly +ĠVis a +Ġmon ks +Ġadvance ment +ĠW er +Ġ18 7 +e us +ert ility +Ġfet al +Ġ19 36 +L o +Ġout fits +Ġstair case +b omb +Ġcustom ized +cl air +T ree +Ġm apped +ĠConsider ing +ĠTor res +Ġmeth yl +Ġapprox imate +Ġdo om +ĠHans en +Ġc rossover +Ġstand alone +ä ¼ +Ġinv ites +Ġgra veyard +Ġh p +Donald Trump +Ġesc ort +G ar +Ġpredec essors +Ġh ay +Ġen zyme +ĠStra ight +vis ors +I ng +ane ously +ĠApp lied +Ġf ec +ĠDur ant +Ġout spoken +or b +Ġz eal +Ġdisgr ace +' ). +ĠChe ng +28 9 +ĠRen a +ĠSu icide +29 4 +Ġout raged +ĠNew man +ĠN vidia +ĠA ber +ĠB ers +Ġrecre ation +Wind ow +ĠD P +x e +Ġped oph +Ġfall out +ambo o +Ġpresent ations +ĠApp s +Ġh tml +3 45 +ĠX XX +Ġrub bing +ĠLe ather +Ġhum idity +se ys +est ablished +ĠUn its +64 6 +Ġrespect able +A uto +Ġthri ving +ĠInn ovation +ang s +Ext ra +reg ulation +29 8 +p ick +Ex amples +ĠC J +Att ack +Ġdr acon +L T +Ġstick er +re rs +Ġsun ny +I ss +reg ulated +d im +ĠAb stract +Ġhus bands +Off ice +om ination +it ars +AN GE +asc al +ĠK ris +ĠInf antry +Ġm alf +ĠA the +ĠR ally +bal anced +................ ........ +OU P +Ġmole cule +met ics +ĠSpl it +ĠInstruct ions +ĠN ights +c ards +Ġt ug +Ġcon e +å Ń +Ġt x +ĠDisc ussion +Ġcatast rophe +pp e +g io +Ġcommun ism +Ġhal ted +ĠGu ant +cle an +ĠSc hed +ĠK anye +Ġw ander +ĠSer iously +Ġ18 8 +enn ial +f ollow +product ive +ĠFl ow +ĠS ail +Ġc raw +Ġsim ulations +or u +ang les +ĠN olan +Ġmen stru +4 70 +Ġ20 7 +aj a +Ġcas ually +board ing +Ġ2 22 +ov y +ĠN umbers +um at +O E +28 7 +ĠCle mson +Ġcert s +Ġsl id +ĠT ribe +Ġto ast +Ġfort unes +Ġf als +ĠComm ittees +Ġg p +Ġf iery +ĠN ets +ĠAn ime +Pack age +ĠComp are +l aughter +in fect +Ġatroc ities +Ġjust ices +Ġins ults +ĠVern on +Ġsh aken +Ġperson a +est amp +36 7 +br ain +Ġexperiment ing +K en +ĠElect ronics +Ġ16 1 +dom ain +Ġgraph ical +b ishop +Ġwho pping +ĠEv angel +Ġadvertis ers +ĠSpe ar +Ġb ids +Ġdestro ys +ut z +Ġunders c +ĠAD D +Ġan ts +ĠC um +ipp les +ĠF ill +Ġgl anced +Ġind icted +ĠE ff +Ġmis con +ĠDes ktop +Ġab ide +ãĥ Ģ +ĠI o +ĠC oul +Ġcaps ule +ĠCh rys +M ON +Ġund es +ĠI RA +Ġc itation +Ġdict ate +ĠNet works +ĠConf lict +ĠSt uff +x a +is ec +ĠChem istry +Ġquarter ly +William s +an an +O pt +ĠAlexand ria +out heastern +ĠSpring field +ĠBlack s +Ġge ography +24 2 +Ġut most +ĠEx xon +ab outs +E VA +ĠEn able +ĠBar r +Ġdisag reed +ĠCy prus +Ġdement ia +Ġlab s +Ġubiqu itous +ĠLO VE +Ġconsolid ated +s r +Ġcream y +ĠTim ber +Reg ardless +ĠCert ificate +Ġ" ... +ogen ous +Capt ain +Ġinsult ing +ĠSor os +ĠInst r +ĠBulgar ia +bet ter +Ġsuck ing +ĠDavid son +at z +Ġcoll ateral +g if +Ġplag ued +ĠC ancel +ĠGard ner +R B +Ġsix teen +Rem ove +ur istic +c ook +R od +Ġcompr ising +f le +) âĢĶ +ĠVik ing +g rowth +agon al +Ġsr f +af ety +m ot +N early +st own +ĠF actor +Ġautom obile +Ġproced ural +m ask +amp ires +Ġdisapp ears +j ab +3 15 +Ġ19 51 +ne eded +Ġd aring +le ader +Ġp odium +Ġun healthy +Ġm und +Ġpy ramid +oc re +Ġkiss ed +Ġdream ed +ĠFant astic +ĠG ly +å Ĭ +Ġgreat ness +Ġsp ices +Ġmet ropolitan +Ġcomp uls +i ets +101 6 +ĠSh am +ĠP yr +fl ies +ĠMid night +Ġswall owed +Ġgen res +ĠL ucky +ĠRew ards +Ġdisp atch +ĠI PA +ĠApp ly +Ġa ven +al ities +3 12 +th ings +Ġ( ). +Ġm ates +ĠS z +ĠC OP +ol ate +O FF +Ġre charge +c aps +ĠYork er +ic one +Ġgal axies +ile aks +D ave +ĠP uzz +ĠCelt ic +ĠA FC +27 6 +ĠS ons +Ġaffirm ative +H or +Ġtutorial s +ĠC ITY +ĠR osa +ĠExt ension +Ser ies +Ġf ats +Ġr ab +l is +Ġun ic +Ġe ve +ĠSp in +Ġadul thood +ty p +Ġsect arian +Ġcheck out +ĠCy cl +S ingle +Ġmart yr +Ġch illing +88 8 +ou fl +Ġ] ; +Ġcongest ion +m k +ĠWhere as +Ġ19 38 +ur rencies +er ion +Ġbo ast +ĠPat ients +Ġch ap +ĠB D +real DonaldTrump +Ġexam ines +h ov +Ġstart ling +ĠBab ylon +w id +om ew +br ance +ĠOd yssey +w ig +Ġtor ch +ĠV ox +ĠMo z +ĠT roll +ĠAn s +Similar ly +ĠF ul +00 6 +Un less +ĠAl one +st ead +ĠPub lisher +r ights +t u +ĠDoes n +Ġprofession ally +Ġcl o +ic z +Ġste als +Ġ á +19 86 +Ġst urdy +ĠJoh ann +Ġmed als +Ġfil ings +ĠFr aser +d one +Ġmult inational +Ġf eder +Ġworth less +Ġp est +Yes terday +ank ind +Ġg ays +Ġb orne +ĠP OS +Pict ure +Ġpercent ages +25 1 +r ame +Ġpot ions +AM D +ĠLeban ese +Ġr ang +ĠL SU +ong s +Ġpen insula +ĠCl ause +AL K +oh a +ĠMac Book +Ġunanim ous +Ġl enders +Ġhang s +Ġfranch ises +ore rs +ĠUp dates +Ġisol ate +and ro +S oon +Ġdisrupt ive +ĠSur ve +Ġst itches +ĠSc orp +ĠDomin ion +Ġsupp lying +Ar g +Ġtur ret +ĠL uk +Ġbr ackets +* ) +ĠRevolution ary +ĠHon est +Ġnot icing +ĠSh annon +Ġafford ed +Ġth a +ĠJan et +! -- +ĠNare ndra +ĠPl ot +H ol +se ver +e enth +Ġobst ruction +Ġ10 24 +st aff +j as +or get +sc enes +l aughs +ĠF argo +cr ime +Ġorche str +Ġde let +ili ary +rie ved +Ġmilit ar +ĠGreen e +âĹ ı +ãģ ¦ +ĠGu ards +Ġunle ashed +ĠWe ber +Ġadjust able +Ġcal iber +Ġmotiv ations +Ġà ł +m Ah +ĠL anka +hand le +Ġp ent +ĠR av +ĠAng ular +ĠK au +umb ing +Ġphil anthrop +Ġde hyd +Ġtox icity +e er +ĠY ORK +w itz +å ¼ +ĠI E +commun ity +ĠA H +Ġret ali +Ġmass ively +ĠDani els +ĠD EL +Ġcar cin +Ur l +Ġrout ing +ĠNPC s +ĠR AF +ry ce +Ġwa ived +ĠGu atem +Every body +Ġco venant +Ġ17 3 +Ġrelax ing +Ġqu art +al most +Ġguard ed +ĠSold iers +ĠPL AY +Ġout going +L AND +Ġre write +ĠM OV +ĠIm per +ĠS olution +Ġphenomen al +Ġl ongevity +Ġimp at +ĠN issan +ir ie +Ġod or +ĠZ ar +ok s +Ġmilit ias +ĠSP EC +Ġtoler ated +ars er +ĠBrad ford ++ , +Ġsur real +s f +Can adian +Ġresemb lance +Ġcarbohyd rate +VI EW +Ġaccess ory +me al +larg est +ieg el +Some one +Ġtoug hest +os o +Ġfun nel +Ġcondemn ation +lu ent +Ġw ired +ĠSun set +Jes us +ĠP ST +ĠP ages +ĠTy coon +ĠP F +Ġselect ions +Ġ ठ+part isan +Ġhigh s +ĠR une +Ġcraft s +le ad +ĠParent s +Ġre claim +ek er +ĠAll ied +ae per +Ġlo oming +Ġbenefic iaries +ĠH ull +Stud ents +Jew ish +d j +Ġp act +tem plate +ĠOffic ials +ĠBay lor +Ġhe mp +Ġyouth s +ĠLevel s +ĠX iao +ĠC hes +Ġende avor +ĠRem oved +Ġhipp ocamp +H ell +ãĤ Ĭ +80 5 +Ġd inosaur +ĠWr ath +ĠIndones ian +Ġcalcul ator +ĠD ictionary +Ġ4 20 +ĠM AG +( _ +! , +t arians +Ġrestrict ing +rac use +Ġweek day +OU NT +Ġsh rugged +leg round +Ġb ald +ĠDo ctors +Ġt outed +ĠMax well +Ġ2 14 +Ġdiplom at +Ġrep ression +Ġconstitu ency +v ice +r anked +ĠNap oleon +g ang +ĠFore ver +t un +Ġbul b +ĠPD T +ĠC isco +V EN +Ġres umed +Ste ven +ĠManit oba +Ġfab ulous +ĠAg ents +19 84 +Ġam using +ĠMyster ies +Ġor thodox +fl oor +Ġquestion naire +Ġpenet rate +Ġfilm makers +ĠUn c +Ġst amped +Ġth irteen +Ġout field +Ġforward ed +Ġapp ra +Ġa ided +t ry +Ġunf ocused +ĠL iz +ĠWend y +ĠSc ene +Ch arg +Ġreject s +Ġleft ist +ĠProv idence +ĠBr id +reg n +Ġprophe cy +ĠL IVE +4 99 +Ġfor ge +ĠF ML +Ġintrins ic +ĠF rog +Ġw ont +ĠH olt +Ġfam ed +CL US +aeper nick +ĠH ate +ĠC ay +Ġregister ing +ort ality +rop y +ocaly ptic +a an +n av +Ġfasc ist +IF IED +Ġimpl icated +ĠRes ort +ĠChand ler +ĠBr ick +P in +ys c +Us age +ĠHel m +us ra +âĺħ âĺħ +ĠAb bas +Ġunanim ously +Ġke eper +Ġadd icted +?? ? +Ġhelm ets +Ġant ioxid +aps ed +80 8 +gi ene +Ġwa its +Ġmin ion +ra ved +ĠP orsche +Ġdream ing +Ġ17 1 +ĠC ain +Ġun for +ass o +ĠConfig uration +k un +hard t +Ġn ested +ĠL DS +L ES +Ġt ying +en os +Ġc ue +ĠMar qu +sk irts +Ġclick ed +Ġexp iration +ĠAccording ly +ĠW C +Ġbless ings +Ġaddict ive +ĠN arr +y x +ĠJagu ars +Ġrent s +ĠS iber +Ġt ipped +ous se +ĠFitz gerald +Ġhier arch +out ine +Ġwa velength +> . +ch id +ĠProcess ing +/ + +r anking +E asy +ĠConst ruct +Ġt et +ins ured +H UD +Ġqu oting +Ġcommun icated +in x +Ġin mate +Ġerect ed +ĠAbs olutely +ĠSure ly +Ġun im +ĠThr one +he id +Ġcl aws +Ġsuper star +ĠL enn +ĠWh is +U k +ab ol +Ġsk et +ĠN iet +Ġper ks +Ġaff inity +Ġopen ings +phas is +Ġdiscrim inate +T ip +v c +Ġgr inding +ĠJenn y +Ġast hma +hol es +ĠHom er +Ġreg isters +ĠGl ad +Ġcre ations +Ġlith ium +Ġappl ause +unt il +Just ice +ĠTur ks +Ġsc andals +Ġb ake +t ank +M ech +ĠMe ans +ĠM aid +Republic ans +is al +wind ows +ĠSant os +Ġveget ation +33 8 +t ri +Ġfl ux +ins ert +Ġclar ified +Ġmort g +ĠCh im +ĠT ort +Ġdiscl aim +met al +ĠAs ide +Ġindu ction +Ġinf l +Ġathe ists +amp h +Ġe ther +ĠV ital +ĠBu ilt +M ind +Ġweapon ry +S ET +Ġ18 6 +ad min +g am +cont ract +af a +Ġderiv atives +Ġsn acks +Ġch urn +E conom +Ġca pped +ĠUnder standing +ĠH ers +ĠI z +Ġd uct +I ENT +augh ty +Ġâľ Ķ +ĠN P +Ġsa iling +In itialized +Ġt ed +Ġreact ors +ĠL omb +Ġcho ke +ĠW orm +Ġadm iration +Ġsw ung +ens ibly +Ġr ash +ĠGo als +ĠImport ant +Sh ot +ĠR as +Ġtrain ers +ĠB un +Work ing +Ġhar med +ĠPand ora +ĠL TE +Ġmush room +ĠCH AR +ĠF ee +ĠM oy +B orn +ol iberal +ĠMart ial +Ġgentle men +Ġling ering +Offic ial +Ġgra ffiti +ĠN ames +D er +Ġqu int +ist rate +aze era +ĠNOT ICE +ĠFlore nce +Ġpay able +Ġdep icts +ĠSpe cies +He art +âĶĢâĶĢâĶĢâĶĢ âĶĢâĶĢâĶĢâĶĢ +Ġencl osed +Incre ases +D aily +ĠL is +Ġenact ment +ĠB acon +ĠSt eele +dem and +Ġ18 3 +Ġmouth s +Ġstr anded +Ġenhance ment +01 1 +ĠWh ats +Ġhe aled +en y +ĠR ab +Ġ3 40 +ĠLab yrinth +ro ach +ĠY osh +ĠCl ippers +Ġconcert s +Intern et +35 5 +Ġstick ers +Ġter med +ĠAx e +Ġgrand parents +Fr ance +ĠCl im +ĠU h +ul ic +Ġthr ill +cent ric +ĠOver view +ĠCond uct +Ġsubstant ive +Ġ18 2 +m ur +Ġstr ay +ĠCo ff +Ġrep etitive +ĠFor gotten +Ġqual ification +ew itness +ĠZ imbabwe +Ġsim ulated +ĠJ D +25 3 +ĠW are +Ġun sc +T imes +Ġsum mons +Ġdis connected +Ġ18 4 +ci us +ĠGu jar +od ka +Ġer ase +ĠTob acco +elect ed +Ġun cont +ĠShe pard +ĠL amp +Ġalert ed +Ġoper ative +arn a +u int +Ġneglig ence +ac ements +Ġsup ra +Ġprev ail +ĠSh ark +Ġbel ts +ãģ « +Ġt ighter +Engine ers +Ġin active +Ġexp onent +ĠWill ie +a ples +Ġhe ir +ĠH its +ian n +ĠS ays +Ġcurrent s +ĠBeng al +Ġar ist +B uffer +Ġbree ze +ĠWes ley +Col a +Ġpron oun +Ġde ed +ĠK ling +Ġof t +Ġinf lict +Ġpun ishing +Ġn m +ik u +OD UCT +01 4 +Ġsubsid y +ĠDE A +ĠHer bert +ĠJ al +B ank +Ġdef erred +Ġship ment +B ott +Ġal le +b earing +HT ML +Off line +Ġ2 13 +Ġscroll ing +Ġsc anned +ĠLib yan +ĠT OP +ch rom +d t +col umn +Psy NetMessage +Z ero +Ġtor so +0 50 +âķ IJ +Ġimp erson +ĠSchw artz +ud ic +Ġpiss ed +ĠS app +25 7 +ĠIS Ps +og l +Ġsuper vised +Ġad olescent +Ġatt ained +ĠDel ivery +ĠB unny +Ġ19 37 +Ġmini ature +Ġo s +Ġ3 70 +60 8 +ĠMour inho +Ġinn ate +Ġtem po +ĠN M +ĠFall en +00 9 +Ġprov ocative +Stream er +ĠBened ict +ĠBol she +Ġt urtle +ĠPC B +ĠEqu al +Direct or +ĠR end +Ġflu ids +Author ities +Ġcous ins +requ ency +ĠNeigh bor +s ets +sh ared +Char les +pass word +Ġg ears +Ġ2 11 +ĠHard ware +ri ka +Ġup stream +H om +Ġdisproportion ately +iv ities +Ġund efined +Ġelect rons +Ġcommem or +Event ually +Ġ> < +Ġir responsible +2 18 +ĠRe leased +ĠO VER +ĠI GN +ĠB read +st ellar +ĠS age +tt ed +dam age +ed ition +ĠPre c +Ġl ime +Ġconf inement +Ġcal orie +we apon +Ġdiff ering +ĠS ina +m ys +am d +Ġintric ate +k k +ĠP AT +ã o +st ones +lin ks +Ġr anch +Sem itic +Ġdifferent iate +ĠS inger +occup ied +Ġfort ress +c md +Ġinter ception +ĠAnk ara +Ġre pt +ĠSol itaire +Ġrem ake +p red +Ġd ared +aut ions +ĠB ACK +Run ning +Ġdebug ging +Ġgraph s +3 99 +ĠNig el +Ġb un +Ġpill ow +Ġprog ressed +fashion ed +Ġob edience +ER N +Ġrehe ars +C ell +t l +S her +Ġher ald +ĠPay ment +ĠC ory +ĠDe pt +Ġrep ent +ĠWe ak +uck land +Ġple asing +Ġshort ages +Ġjur ors +ĠK ab +q qa +Ant i +Ġw ow +ĠRC MP +Ġt sun +ĠS ic +Ġcomp rises +Ġsp ies +Ġprec inct +n u +Ġur ges +Ġtim ed +Ġstrip es +ĠB oots +Ġy en +Adv anced +Ġdisc rete +ĠArch angel +employ ment +D iff +Ġmon uments +Ġ20 9 +work er +Ġ19 6 +ĠI g +utter stock +T PS +J ac +Ġhomeless ness +Ġcomment ator +Ġrac ially +f ing +se ed +E le +ell ation +Ġeth anol +Ġpar ish +ĠD ong +ĠAw akening +Ġdev iation +ĠB earing +ĠTsu k +Ġrec ess +Ġl ymph +ĠCann abis +å ľ +ĠNEW S +Ġd ra +ĠStef an +ĠWr ong +ĠS AM +Ġloose ly +Ġinterpre ter +ĠPl ain +Go vernment +Ġbigot ry +Ġgren ades +ave z +pict ured +Ġmand ated +ĠMon k +ĠPed ro +Ġl ava +27 4 +Ġcyn ical +ĠScroll s +l ocks +M p +Ġcon gregation +orn ings +ph il +ĠI bid +Ġf erv +Ġdisapp earing +Ġarrog ant +sy n +ĠMa ver +ĠSu it +24 1 +Ġab bre +ack ers +P a +ĠY el +Whe never +Ġ23 5 +ĠV ine +ĠAn at +Ġext inct +LE T +Ġexecut able +V ERS +ox ide +D NA +ĠP rel +Ġresent ment +Ġcompr ise +ĠAv iv +Ġinter ceptions +Ġprol ific +IN A +ĠEr in +though t +2 19 +ĠPsychiat ry +un ky +chem ist +H o +ĠMcC oy +Ġbr icks +L os +ri ly +ĠUS SR +Ġr ud +Ġl aud +ĠW ise +ĠEmer ald +Ġrev ived +Ġdam ned +ĠRep air +id em +ct ica +Ġpatri arch +ĠN urs +me g +Ġcheap est +re ements +empt y +ĠCele br +Ġdepri vation +ch anted +ĠTh umbnails +E nergy +ĠEth an +ĠQ ing +Ġopp oses +W IND +v ik +ĠM au +ĠS UB +66 7 +G RE +ĠVol unte +nt on +C ook +å IJ +es que +Ġplum met +Ġsu ing +Ġpron ounce +Ġresist ing +ĠF ishing +ĠTri als +Ġy ell +Ġ3 10 +Ġin duct +Ġpersonal ized +oft en +R eb +EM BER +Ġview point +Ġexist ential +() ) +rem ove +MENT S +l asses +Ġev apor +Ġa isle +met a +Ġreflect ive +Ġentit lement +Ġdev ised +mus ic +asc ade +Ġwind ing +off set +Ġaccess ibility +ke red +Bet ter +ĠJohn ston +th inking +S now +ĠCroat ia +ĠAt omic +27 1 +34 8 +Ġtext book +ĠSix th +Ġ اÙĦ +Ġsl ider +ĠBur ger +b ol +S ync +Ġgrand children +Ġc erv ++ ) +Ġe ternity +Ġtweet ing +Ġspec ulative +Ġpiv otal +ĠW P +ĠT ER +ynam ic +Ġu pl +ĠC ats +per haps +Ġclass mates +Ġblat ant +' - +Ġl akh +ant ine +ĠB org +i om +/ ( +ĠAthlet ic +Ġs ar +OT A +ĠHoff man +Never theless +Ġad orable +Ġspawn ed +Ass ociated +ĠDom estic +Ġimpl ant +ĠLux em +ĠK ens +Ġp umps +ĠS AT +Att ributes +50 9 +av our +Ġcentral ized +ĠT N +Ġfresh ly +ĠA chieve +Ġouts iders +her ty +ĠRe e +ĠT owers +ĠD art +ak able +Ġm p +ĠHeaven ly +Ġr ipe +ĠCarol ine +ry an +Ġclass ics +Ġret iring +Ġ2 28 +Ġa h +Ġdeal ings +Ġpunch ing +ĠChap man +O ptions +max well +vol ume +Ġst al +Ġex ported +ĠQu ite +Ġnumer ical +B urn +F act +ĠKey stone +Ġtrend ing +Ġalter ing +ĠAfric ans +47 8 +ĠM N +ĠKn ock +Ġtempt ation +Ġprest ige +Over view +ĠTrad itional +ĠBah rain +Priv ate +ĠH OU +Ġbar r +ĠT at +C ube +US D +ĠGrand e +ĠG at +ĠFl o +Ġres ides +Ġind ec +vol ent +Ġperpet ual +ub es +Ġworld view +ĠQuant um +Ġfil tered +Ġen su +orget own +ERS ON +ĠM ild +37 9 +OT T +à ¥ +Ġvit amins +Ġrib bon +Ġsincere ly +ĠH in +Ġeight een +Ġcontradict ory +Ġgl aring +Ġexpect ancy +Ġcons pir +Ġmon strous +Ġ3 80 +re ci +Ġhand ic +Ġpump ed +Ġindic ative +Ġr app +Ġav ail +ĠLEG O +ĠMar ijuana +19 85 +ert on +Ġtwent ieth +################ ################ +ĠSw amp +Ġval uation +Ġaffili ates +adjust ed +ĠFac ility +26 2 +Ġenz ymes +itud inal +Ġimp rint +S ite +Ġinstall er +ĠT RA +m ology +lin ear +ĠCollect ive +ig ating +ĠT oken +Ġspec ulated +K N +ĠC ly +or ity +Ġdef er +Ġinspect ors +appro ved +R M +ĠSun s +Ġinform ing +ĠSy racuse +ib li +7 65 +Ġgl ove +Ġauthor ize +âĢ¦âĢ¦âĢ¦âĢ¦ âĢ¦âĢ¦âĢ¦âĢ¦ +ĠCru ise +Ġcontract ing +she ll +IF E +ĠJew el +p ract +ĠPhot oshop +ĠKnow ing +h arm +Ġattract ions +ad an +et us +01 8 +w agen +Al t +Ġmultip ly +Ġequ ilibrium +: { +ĠF ighters +ĠEd gar +Ġfour teen +Go vern +Ġmis use +Ġab using +Ġancest ry +ram er +64 4 +Ġwor ms +Ġthick er +ĠComb ine +Ġpeas ants +Ġv ind +Ġcon quest +Ġm ocked +Ġc innamon +ĠC ald +ĠGall up +Ġavoid ance +Ġincarn ation +ĠStr at +Ġt asted +ent a +ĠN eal +p ared +Ġtermin ology +ject ion +Scient ists +ĠIN S +ĠDe e +Ġdirect ories +R oad +ĠSh ap +br ight +ĠDirect ors +ĠCol umn +Ġb ob +Ġprefer ably +Ġgl itch +f urt +Ġe g +id is +C BC +Ġsur rendered +Ġtest ament +33 6 +ug gest +ĠN il +an other +Ġpat hetic +ĠDon na +Ġ2 18 +ĠA very +Ġwhis key +Ġf ixture +ĠCon quest +Ġbet s +O cc +ĠLe icester +] ." +Ġ) ); +Ġfl ashes +45 6 +Ġmask ed +ge bra +Ġcomput ed +che l +aud er +Ġdefe ats +ĠLiber ation +ĠOs ama +ĠV ive +Ch anges +Ch annel +Ġtar iffs +Ġm age +ĠS ax +Ġinadvert ently +ĠC RE +ĠRe aper +ink y +gr ading +Ġstere otyp +Ġcur l +ĠF ANT +Ġfram eworks +M om +ĠAn ch +Ġflav our +car bon +Ġperm itting +let cher +ĠMo zilla +ĠPark ing +ĠCh amp +Sc roll +Ġmurd erer +Ġrest ed +Ġow es +ĠP oss +AD D +IF F +res olution +ĠMin ing +Ġcompar ative +D im +Ġneighbour ing +ĠA ST +ĠT oxic +Ġbi ases +Ġgun fire +ur ous +ĠMom ent +19 83 +Ġper vasive +tt p +ĠNorm ally +r ir +S arah +ĠAlb any +Ġun sett +ĠS MS +ip ers +l ayer +ĠWh ites +up le +Ġtur bo +ĠLe eds +Ġthat s +ĠMin er +M ER +ĠRe ign +Ġper me +ĠBl itz +Ġ19 34 +Ġintimid ating +t ube +Ġecc entric +ab olic +box es +ĠAssoci ates +v otes +Ġsim ulate +um bo +aster y +Ġship ments +FF FF +an th +Ġseason ed +Ġexperiment ation +âĸ ł +law s +Me et +idd les +ant ics +R ating +IS IS +h ift +Ġfront s +b uf +01 7 +Ġun att +ĠD il +le ases +ĠGard ens +77 7 +t ouch +ve ll +45 8 +Ġ= ==== +s aving +Ġer osion +ĠQu in +Ġearn s +Ġaccomplish ment +ĠWe i +Ġ< [ +____ _ +Ġir rig +ĠT eddy +Ġconqu ered +ĠArm ored +Ġassert s +Ġmanip ulating +r é +Ġtranscript s +G allery +Ġplot ting +Ne il +Ġbetray al +load er +ĠS ul +Ġdispl acement +Ġroy alty +ĠW I +he it +ĠDev ices +alle l +Ġmunicipal ities +Ġcan al +St ars +ĠU AE +Ġ" âĢ¦ +ĠC U +ab ove +Ġreson ance +ĠguiActive Un +add ed +ĠBra ves +ĠI bn +Ġhere by +ĠB RE +Ġshare holder +ĠH ir +ĠJ i +Ġstrange ly +Ġadm ired +Ġpl ight +Ġb achelor +ĠP ole +cipl inary +T ony +ĠArmen ian +Ġun man +ĠZion ist +St age +isco ver +Ġautom otive +Ġs idelines +Ġsl ick +ĠRena issance +ĠF UN +Im ages +ĠH aj +Ġp ing +Ġshort cut +ĠBl vd +ĠLook s +Ġbur sts +Ġcl amp +Ġm ish +Ġsort ing +Ġpatri ot +Ġcorrect ness +ĠScand inav +ĠCaval iers +p ython +az ar +Ġ3 75 +ĠJa une +40 9 +Ġdetrim ental +Ġstab bing +Ġpoison ed +Ġf ountain +oc ent +or st +ĠMar i +Ġr ains +ĠO vers +ĠInst itution +ud get +AM Y +t ale +ĠK R +ĠPr ices +Ġhead aches +Ġlands l +ĠA ura +Bon us +ĠZ hao +ĠH ip +Ġhop s +ĠKurd istan +Ġexplo iting +ry n +Ġhypocr isy +op ening +Ġgun shot +Ġw ed +inter stitial +Inter stitial +Ġam en +Bre aking +Ġmarket ed +W ire +ĠC rowd +Contin ue +ĠK nown +ĠEffect ive +ore an +iz ons +Jose ph +Ġescal ation +us ername +Ġcur tain +AT ES +ĠP AR +ĠM iy +Ġcounter fe +l ene +Ġcont enders +d aily +ĠAs c +ĠPhill ip +most ly +Ġfil ename +he ne +Ġresemb ling +Ġst aging +ĠCh loe +Ġw iring +H on +ĠRen ew +ott age +ĠHy brid +m uch +Ġstro kes +Ġpolicy makers +AP TER +ĠArk ham +pl ot +Ġassist ants +Ġde port +ĠSe ga +Ġinflu enza +ĠC ursed +ĠK obe +Ġskin ny +Prov ider +ĠR ip +Ġincrement al +product s +B F +Ġd ome +ĠC redits +Ġlos ers +int s +ĠBet ty +ĠTal ent +ĠD AM +L v +E ss +Ġd ens +tem p +J udge +od ic +Ġ' ( +UR ES +ets k +V O +Ġretrie ved +Ġarchitect s +Ù ĩ +Ġeth ic +ĠSecond ary +st ocks +ad ia +Ġ3 25 +ĠOp inion +Ġsimultane ous +Ġd izz +ul p +Ġsmugg ling +ipp ery +R andom +f acing +ĠD as +Ġstock p +Ġdiscl osures +po inter +Ġcor al +ĠSe lection +ĠP ike +ival ent +Ġruth less +ĠR im +Ġensu ing +ĠExper iment +Ġcongress man +Ġbelie ver +Ġun specified +ĠM ord +Ġknowledge able +ĠV ERY +T X +Ġstra ps +Ġtur f +apesh ifter +Ġmar ital +Ġfl ock +ãģ Ĩ +26 3 +AM ES +ĠOpp osition +Ġtre asures +ĠG OD +Ġmodel ed +ĠWOR LD +Ġ( [ +ĠUs age +H F +Ġ$ ( +uss ed +Ġpione er +E ight +par se +b read +rit z +ĠMir anda +ĠK ant +++ ) +ore n +Ġprov oked +Ġbre eds +ĠIn cludes +ĠPast ebin +ĠFl ip +J ava +Ġbr ink +Ġrum ored +Ġun seen +Ġgar nered +ĠDef in +al ted +Ġtatt oos +Ġhes itation +is itions +ĠWe aver +ĠReport ing +Ġtherap ies +Ġconsult ants +Ġresid ual +ĠMal i +ĠRom a +i ago +ĠRes idents +ub i +Ġremed ies +Ġadapt ive +ĠAl ive +ĠBar cl +Ġwal lets +c rypt +etermin ation +ĠPel osi +Ġsl ipping +oton in +Ġall iances +pat rick +ir is +Ġor th +ĠPer kins +ĠDe V +ĠG ets +Ġdry ing +ge e +fore st +ĠFor get +ore m +33 9 +Ġvague ly +ĠD ion +ĠP orn +ĠH OW +Ġp neum +Ġrub ble +ĠT aste +enc ia +ĠG el +Ġd st +Ġ24 5 +ĠMoroc co +inf lamm +ĠTw ins +Ġb ots +d aughter +ĠB alk +Ġbre thren +Ġlog os +Ġgo bl +f ps +Ġsub division +Ġp awn +Ġsquee zed +Ġmor ale +ĠD W +' " +Ġkn ot +ook y +Ġdiv isive +Ġboost ed +ch y +ãĥ IJ +if act +Ġnewcom ers +ĠWrest ling +Ġsc outs +w olves +R at +Ġnin eteenth +ĠOs borne +St ats +Ġem powered +Ġpsych opath +ĠO EM +ugg age +ĠP K +ĠMoh ammad +P ak +Ġanarch ists +ĠExt ract +est hes +ĠStock holm +l oo +ĠG raph +Ġdeploy ing +ĠStr anger +ĠM old +Ġstaff er +Ġdiscount ed +uck le +ple ase +ĠLand ing +ÃŃ a +Ġ19 3 +Ġan te +Ġrep etition +Ġ+ /- +Ġpar ody +Ġlive ly +AA A +ĠHor us +Ġp its +ind ers +L OC +ĠVen ice +40 6 +ĠDis cover +â Ĩ +ellect ual +Ġp ens +Ġey el +ig uous +Im pl +Ġj oking +Ġinv al +ĠBel fast +Ġcredit ors +ĠSky walker +ov sky +Ġcease fire +Ġse als +is oft +) ). +ĠFel ix +IT S +Ġt resp +ĠBlock chain +ew are +ĠSch war +en ne +mount ed +ĠBe acon +les h +Ġimmense ly +Ġche ering +Em ploy +sc ene +ish ly +atche wan +ĠNic olas +Ġdr ained +ĠEx it +ĠAz erb +j un +Ġflo ated +u ania +De ep +Ġsuper v +Ġmyst ical +ĠD ollar +ĠApost le +ĠR EL +ĠProv ided +ĠB ucks +ãĥ ´ +cut ting +Ġenhance ments +ĠPengu ins +ĠIsa iah +Ġj erk +ĠW yn +Ġst alled +Ġcryptoc urrencies +ĠR oland +sing le +Ġl umin +ĠF ellow +ĠCap acity +ĠKaz akh +W N +Ġfin anced +38 9 +Ġt id +Ġcoll usion +ĠMy r +î Ģ +Sen ator +Ġped iatric +Ġneat ly +Ġsandwic hes +ĠArchitect ure +Ġt ucked +Ġbalcon y +Ġearthqu akes +qu ire +F uture +Ġhe fty +é Ĺ +Ġspecial izes +Ġstress es +Ġs ender +Ġmisunder standing +Ġep ile +Ġprov oke +ĠCol ors +Ġdis may +uk o +[ _ +58 6 +ne utral +Ġdon ating +ĠRand all +Mult i +Ġconvenient ly +ĠS ung +ĠC oca +Ġt ents +ĠAc celer +Ġpart nered +27 2 +ir ming +ĠB AS +s ometimes +Ġobject ed +ub ric +p osed +LC S +gr ass +Ġattribut able +V IS +Israel i +Ġrepe ats +ĠR M +v ag +ut a +in ous +Ġin ert +ĠMig uel +æ Ń +ĠHawai ian +B oard +Ġart ific +ĠAzerb ai +as io +ĠR ent +A IN +Ġappl iances +Ġnational ity +Ġass hole +ĠN eb +Ġnot ch +h ani +ĠBr ide +Av ailability +Ġintercept ed +Ġcontin ental +Ġsw elling +ĠPers pect +b ies +. < +ith metic +ĠL ara +Ġtempt ing +add r +Ġoversee ing +cl ad +ĠD V +ĠGing rich +Ġm un +ĠApp ropri +Ġalter ations +ĠPat reon +Ġha voc +Ġdiscipl ines +Ġnotor iously +aku ya +ier i +? ). +ĠW ent +Ġsil icon +Ġtre mb +Cont ainer +K nown +Ġmort ar +est e +ick a +Ar thur +ĠPre viously +ĠMart y +Ġsp arse +g ins +Ġin ward +ĠParticip ant +C opy +ĠM isc +Ġantib iotic +ĠRet ro +Ġel usive +Ġass ail +ĠBatt alion +ĠB ought +Ġdimin ish +ĠEuro pa +s ession +ĠDanger ous +ies el +Ġdisbel ief +Ġbl asts +ext reme +ĠBoy d +ĠProject s +ĠGu ys +Ġunder gone +Ġgr ill +ĠDw ight +Ġ19 7 +US ER +Ġfiles ystem +Ġcl ocks +T aylor +Ġwra pper +Ġfold ing +ous and +ĠPhilipp ine +ATION AL +ĠPer th +Ġas hes +Ġaccum ulate +ĠGate way +Sh op +orks hire +H an +ĠBar rel +ĠLe h +ĠX V +Ġwh im +Ġrep o +ĠC G +ĠM am +Ġincorpor ating +Ġbail out +Ġlingu istic +Ġdis integ +C LE +Ġcinem atic +ĠF iber +S yn +il ion +ĠCom pos +c hens +Ġne oc +Ġbo iled +F INE +on o +un cle +ik en +ĠB M +Î ¹ +Ġreceipt s +Ġdisp osed +ĠTh irty +ĠR ough +ĠA BS +Ġnot withstanding +oll en +# $ +Ġunrel iable +Ġbl oom +Ġmedi ocre +Ġtr am +ĠTas man +Ġsh akes +Ġmanifest o +ĠM W +Ġsatisf actory +Ġsh ores +Ġcomput ation +Ġassert ions +orm ons +ar ag +ab it +Dem ocrats +ĠL oot +ĠVol ks +ha ired +Ġgrav itational +S ing +ĠM iz +Ġthro ttle +Ġtyr anny +ĠView s +Ġrob ber +ĠMinor ity +Ġsh rine +sc ope +pur pose +Ġnucle us +our cing +ĠUS DA +ĠD HS +w ra +ĠBow ie +Sc ale +ĠB EL +x i +I ter +Ġ( ), +w right +Ġsail ors +ous ed +NAS A +ĠPro of +ĠMin eral +t oken +ĠF D +R ew +Ġe ll +6 30 +Ġchance llor +ĠG os +Ġamount ed +ĠRec re +ome z +ĠOpt im +ĠOl ive +Ġtrack er +ow ler +ĠUn ique +R oot +Ġmar itime +ĠQur an +ĠAd apt +Ġecosystem s +ĠRe peat +ĠS oy +ĠI MP +Ġgrad uating +and em +P ur +ĠRes et +ĠTr ick +ĠPh illy +ĠT ue +ĠMalays ian +Ġclim ax +Ġb ury +Ġcons pic +ĠSouth ampton +ĠFl owers +Ġesc orted +ĠEduc ational +ĠI RC +Ġbrut ally +e ating +Ġpill ar +ĠS ang +ĠJ ude +ar ling +ĠAm nesty +Ġrem inding +ĠAdminist rative +hes da +Ġfl ashed +ĠP BS +per ate +fe ature +Ġsw ipe +Ġgra ves +oult ry +26 1 +bre aks +ĠGu er +Ġsh rimp +ĠV oting +qu ist +Ġanaly tical +Ġtables poons +ĠS OU +Ġresear ched +Ġdisrupt ed +Ġj our +Ġrepl ica +Ġcart oons +b ians +} ) +c opy +G ot +ou ched +P UT +Ġsw arm +not ations +s aid +Ġreb uilt +Ġcollabor ate +Ġr aging +Ġn ar +Ġdem ographics +ĠD DR +Ġdist rust +oss ier +ĠK ro +Ġpump kin +Ġreg rets +Ġfatal ities +ĠL ens +ĠO le +p d +Ġpupp et +ĠOut look +ĠSt am +O l +F air +U U +Ġre written +Ä ± +Ġfasc inated +Ġve ctors +Ġtrib unal +u ay +ĠM ats +ĠCo ins +[ [ +Ġ18 1 +Ġrend ers +ĠK aepernick +Ġesp ionage +Ġsum m +Ġd itch +Acc ount +Ġspread sheet +Ġmut ant +p ast +40 7 +Ġd ye +Ġinit iation +Ġ4 000 +Ġpunish able +Ġth inner +ĠKh al +Ġinter medi +D un +ĠGoth am +Ġeager ly +Ġvag inal +p owers +V W +ĠWATCH ED +Ġpred ator +ams ung +Ġdispar ity +Ġ[ * +Ġam ph +Ġout skirts +ĠSpir its +Ġskelet al +Ð » +ĠR ear +Ġissu ance +ĠLog ic +re leased +Z Z +ĠB ound +Ent ry +Ġex its +is ol +ĠFound er +Ġw re +ĠGreen land +ĠM MO +t aker +IN C +ãģ ¾ +Ġhour ly +hen ko +Ġfantas ies +Ġdis ob +Ġdemol ition +ãĥ ĭ +Ġen listed +rat ulations +Ġmis guided +Ġens ured +Ġdiscour aged +m ort +Ġfl ank +Ġc ess +Ġreact s +ĠS ere +s ensitive +ĠSer pent +ass ad +Ġ24 7 +Ġcalm ly +b usters +Ġble ed +ĠSt ro +Ġamuse ment +ĠAntar ctica +Ġs cept +ĠG aw +a q +ason ic +Ġsp rawling +n ative +atur ated +ĠBattle field +IV ERS +E B +ĠG ems +ĠNorth western +ĠFil ms +ĠAut omatic +Ġappre hend +ãģ ¨ +Ġgui Name +Ġback end +Ġevid enced +ge ant +01 2 +ĠS iege +Ġexternal To +Ġunfocused Range +ĠguiActiveUn focused +Ġgui Icon +ĠexternalTo EVA +ĠexternalToEVA Only +F ri +ch ard +en aries +Ġchief s +Ġc f +ĠH UD +Ġcorro bor +Ġd B +ĠT aken +ĠPat ricia +ra il +ĠCh arm +ĠLiber tarian +rie ve +Person al +ĠO UR +ger ies +Ġdump ing +Ġneurolog ical +it imate +ĠClint ons +raft ed +ĠM olly +Ġtermin als +reg ister +Ġfl are +Ġenc oded +Ġautop sy +p el +m achine +Ġexempt ions +ĠRoy als +d istance +Ġdraft s +Ġl ame +ĠC unning +Ġsp ouses +ĠMark ets +ĠCar rier +Ġimp lying +ĠY ak +s id +Ġl oser +Ġvigil ant +Ġimpe achment +Ġaug mented +ĠEmploy ees +Ġunint ended +tern ally +ĠW att +Ġrecogn izable +ess im +æ Ŀ +Ġco ated +r ha +Ġlie utenant +ĠLegisl ation +pub lished +44 4 +01 3 +Ġide ally +ĠPass word +Ġsimpl ify +ĠMet a +ĠM RI +Ġple ading +organ ized +hand ler +Ġun ravel +cor rect +Ġ icy +Ġparan oid +Ġpass er +Ġinspect ions +of er +ĠHealth care +28 3 +ĠBr ut +iol a +for ge +ĠMed ieval +MS N +ie vers +ĠProgram ming +å ī +Ġ2 23 +m u +ĠC LE +ug a +Ġsho ppers +Ġinform ative +ĠPl ans +Ġsupplement ation +ĠT ests +ty ard +ocy tes +ĠVeg a +ĠGujar at +erman ent +Ex cept +ĠL OT +all a +ĠC umm +ĠO sw +Ġven om +ĠDeb t +ĠD OWN +Ġreun ion +Ġm uc +ĠRel ief +Ġge op +ĠðŁ ĺ +al ogue +An th +ech o +Ġcor ros +Ġrepl ication +ĠBl azing +ĠD aughter +Ġinf lic +ĠLind sey +Ù Ī +28 4 +Ex it +Ġgl oom +TA IN +Ġundermin ing +Ġadv ising +h idden +Ġover flow +Ġg or +urd ue +Ġe choes +enh agen +Ġimp uls +d rug +c ash +Ġas ync +Ġmir ac +at ts +p unk +Ġpiv ot +ĠLegisl ative +Ġblog gers +ĠCl aw +s burg +d yl +ĠRecomm end +Ġver te +Ġprohib iting +ĠPant her +Jon athan +Ġo min +Ġhate ful +28 1 +ĠOr che +ĠMurd och +down s +Ġas ymm +G ER +Al ways +Ġinform s +ĠW M +ĠP ony +ĠApp endix +ĠAr lington +J am +Ġmedic inal +ĠS lam +IT IES +Ġre aff +ĠR i +F G +S pring +b ool +Ġthigh s +Ġmark ings +ĠRa qqa +ĠL ak +p oll +ts ky +ĠMort y +ĠDef inition +Ġdeb unk +end ered +ĠLe one +a vers +Ġmortg ages +App arently +N ic +ha us +ĠTh ousands +au ld +Ġm ash +sh oot +Ġdi arr +Ġconscious ly +H ero +e as +ĠN aturally +ĠDestroy er +Ġdash board +serv ices +R og +Ġmillenn ials +Ġinv ade +- ( +Ġcomm issions +ĠA uckland +Ġbroadcast s +Ġfront al +Ġcr ank +ĠHist oric +Ġrum ours +CT V +Ġster il +Ġboost er +rock et +ãĤ ¼ +ut sche +ĠP I +Ġ2 33 +ĠProdu cer +ĠAnaly tics +Ġinval uable +Ġunint ention +ĠC Y +Ġscrut in +Ġg igg +Ġeng ulf +Ġprolet ariat +Ġh acks +ĠH ew +ar ak +ĠSl ime +ield ing +ag her +ĠEll iot +Ġtele com +Ġ2 19 +ult an +ĠAr bor +ĠSc outs +B an +Ġlifes pan +Ġbl asp +38 8 +Ġjud iciary +ĠContin ental +ask ing +Mc C +L ED +Ġbag gage +ĠSorce rer +Ġrem nants +ĠGriff ith +ets u +ĠSub aru +ĠPerson ality +des igned +ush ima +agn ar +Ġrec oil +Ġpass ions +\ ": +Ġte e +Ġabol ition +ĠCreat ing +j ac +Ġ19 4 +01 9 +Ġpill ars +ric hed +/ " +t k +Ġlive lihood +Ġro asted +ah on +ĠH utch +ass ert +Ġdivid end +Ġkn it +Ġd aunting +Ġdisturb ance +Ġsh ale +Ġcultiv ated +Ġrefriger ator +L B +ĠN ET +Ġcommercial s +Ġthink ers +45 5 +Ġch op +B road +Ġsuspic ions +Ġtag ged +l ifting +Ġsty lish +ĠShield s +Short ly +Ġt ails +A uth +ST E +ĠG AME +Ġse ism +ĠK is +olog ne +Ġcow ork +Ġforc ibly +Ġthy roid +ĠP B +AN E +mar ried +h orse +Ġpoly mer +ĠCh al +od or +DE BUG +ĠCon text +Ġbl iss +Ġpin point +ĠMat hemat +leg ram +ĠWeek end +Ġlab elled +Ġb art +it les +Ġest rogen +âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ +" ' +Ġvis ibly +Ġouts ider +aid a +Are a +Ġdisse min +Ġdish onest +ĠCl osed +ĠBullet in +ĠRam sey +sw ord +ĠX I +our ced +S ame +34 6 +ĠRe pe +ĠK ou +c ake +em is +C ache +ĠMe aning +ĠEn light +onom y +Ġmanifest ation +sw orth +J ay +Ġch ore +ö r +D ream +Ġsanction ed +Ġcult urally +ĠA ra +N av +Ġthe ological +Ġstr ut +ĠV O +ĠHand book +Ġconstruct ing +Ġ ¶ +ĠBenef its +ĠPsych ological +s ac +å ¸ +p olicy +ĠMat ters +ĠReport ed +ĠBy te +Ġvit ro +ĠM aiden +Ġl am +ĠJenn ings +Ġgar ment +ĠRut gers +ĠStaff ord +ĠWell ington +Ġinter mitt +Ġn pm +Ġord eal +Ġplug ged +o oming +in ished +fram ework +Ġtim ber +Ġc ass +Ġ8 50 +il ess +ĠRed ux +7 68 +St re +Ġsurpass ed +w hel +Ġparalle ls +Ġve il +ĠG I +ĠR EST +Ġread iness +s ort +Ġmod ifying +ĠSl ate +ru ff +Ġmar ble +Ġinf rared +Ġaud itor +ĠFANT ASY +ĠP overty +ĠS PD +Ġ" ( +K y +RA Y +Ġexecut ions +ĠBever ly +ĠMarx ism +ĠBur st +ĠK ali +est ones +Clear ly +E ll +ãģ § +ĠProceed ings +T oken +IF IC +ñ a +Cent ral +ĠH aley +ĠD rama +Ġform ations +OR N +Book s +Ġdom inating +ĠFly ers +ĠCompan ion +Ġdiscipl ined +ĠYug oslav +ĠSpell s +Ġv engeance +Ġland lords +L en +ĠO gre +ano ia +Ġpier cing +Ġcon greg +Ġscore r +ob ia +Ġnic kel +ĠLear ns +Ġre jo +Ġmaster piece +Fl ash +Ġinhab ited +ĠOpen GL +ĠD ud +ĠI CO +Ġar ter +Ġpl ur +Ġmaster y +Ġlong standing +st ed +Ġw ines +Ġtelev ised +ĠSh rine +ĠBay ern +Ġâ ĵĺ +Ġencl osure +j ohn +Ġprophe ts +ĠRes urrection +ĠOrd ers +Ġun even +r als +Ġd wind +ĠL ah +ĠSl oven +37 8 +Ġins istence +aff le +ĠCl one +Ġhard ship +ĠCongress man +Ġple ad +Ġreview ers +Ġc ured +Ġ19 35 +as ley +f ake +ĠTh inking +yd ia +P ART +ĠD ota +o it +Ġwh ipped +Ġb ouncing +ĠHispan ics +com ings +Ġcann abin +ĠCh ambers +ĠZ ack +Option al +Ġco ats +Ġprow ess +ĠNort on +Ġplain ly +Ġfre ight +Ġinhib ition +Ġcl am +Ġ30 3 +ke f +ale igh +L uke +Ġpsych o +ator ium +M ED +Ġtreat ies +Ġind isc +Ġd c +OP S +Ġresil ient +ĠInter state +Ġsl ack +Ġmund ane +Ġestab lishes +35 9 +Ġstr ained +Ġn ond +S us +Ġcast e +ar ate +ie ving +Ġunfair ly +Ġpars er +on ial +urs ive +V ia +ĠOtt o +ĠAuthor ities +stro ke +K R +ĠMer cy +Ġfurn ished +Ġout set +Ġmet ic +19 82 +olith ic +ĠT ent +og ical +ĠA ircraft +Ġh ides +ĠBec ame +Ġeduc ators +re aching +Ġvol atility +Ġtodd ler +ĠNAS CAR +ĠTw elve +ĠHigh lights +Ġgra pe +Ġspl its +Ġpe asant +Ġre neg +ĠMS I +Tem p +st ars +Ġtre k +ĠHy de +b inding +Ġreal ism +Ġox ide +ĠH os +Ġmount s +Ġbit ing +Ġcollaps ing +Ġpost al +Ġmuse ums +Ġdet ached +Ġrespect ing +Ġmonop ol +Ġwork flow +ĠC ake +Tem plate +ĠOrgan isation +Ġpers istence +36 9 +C oming +B rad +Ġredund ant +ĠG TA +Ġb ending +Ġrev oked +Ġoff ending +Ġfram ing +Ġprint f +Comm un +mem bers +Out side +Ġconst rued +Ġc oded +F ORE +Ġch ast +Ch at +Ind ian +ĠY ard +? !" +ĠP orts +ĠX avier +ĠR ET +' ." +ĠBo at +iv ated +ich t +umer able +D s +ĠDun n +Ġcoff in +Ġsecure ly +ĠRapt ors +ĠB es +Install ation +Ġin ception +ĠHealth y +end ants +Ġpsych ologists +ĠShe ikh +c ultural +ĠBlack Berry +sh ift +F red +oc he +Ġc akes +ĠS EO +ĠG ian +ĠAs ians +og ging +e lement +Ġpund its +ĠV augh +ĠG avin +Ġh itter +Ġdrown ed +Ġch alk +ĠZ ika +Ġmeas les +80 2 +âĢ¦ .. +ĠAW S +] " +Ġdist ort +ĠM ast +Ġantib odies +ĠM ash +Mem ory +ĠUg anda +ĠPro b +Ġvom iting +ĠTurn s +Ġoccup ying +Ġev asion +ĠTher apy +Ġprom o +Ġelect r +Ġblue print +ĠD re +pr iced +ĠDep ot +Ġallev iate +ĠSom ali +m arg +n ine +Ġnostalg ia +ĠShe pherd +Ġcaval ry +Ġtor ped +ĠBlood y +x b +Ġs ank +Ġgo alt +report print +embed reportprint +clone embedreportprint +ĠIn itially +ĠF ischer +Ġnot eworthy +c ern +Ġin efficient +raw download +rawdownload cloneembedreportprint +c ation +ĠD ynasty +l ag +D ES +Ġdistinct ly +ĠEston ia +Ġopen ness +Ġg ossip +ru ck +W idth +ĠIb rahim +Ġpet roleum +Ġav atar +ĠH ed +ath a +ĠHog warts +Ġc aves +67 8 +Ġsafegu ard +ĠM og +iss on +ĠDur ham +sl aught +ĠGrad uate +Ġsub conscious +ĠEx cellent +ĠD um +---- - +Ġp iles +ĠW ORK +ĠG arn +ĠF ol +ĠAT M +Ġavoid s +ĠT ul +Ġble ak +EL Y +iv ist +light ly +P ers +ĠD ob +ĠL S +Ġins anity +Î µ +atal ie +En large +Ġtw ists +Ġfault y +Ġpir acy +Ġimp over +Ġrug ged +ĠF ashion +Ġs ands +' ? +sw ick +Ġn atives +Ġhe n +ĠNo ise +ãĥ Ĺ +Ġg reens +Ġfree zer +Ġd ynasty +ĠFather s +ĠNew ark +Ġarchae ological +Ġo t +ob ar +Ġblock ade +Ġall erg +L V +Ġdeb it +ĠR FC +ĠMil ton +ĠPress ure +Ġwill ingly +Ġdisproportion ate +Ġopp ressive +Ġdiamond s +Ġbelong ings +19 70 +Ġbell s +Ġimperial ism +Ġ2 27 +Ġexpl oding +ĠE clipse +Ġ19 19 +Ġr ant +Ġnom inations +34 7 +Ġpeace fully +ric a +ĠF UCK +Ġvib ration +mal ink +Ġro pes +ĠIv anka +ĠBrew ery +ĠBook er +ĠOw ens +go ers +Serv ices +ĠSn ape +Ġ19 1 +39 5 +Ġ2 99 +just ice +Ġb ri +Ġdisc s +Ġprom inently +Ġvul gar +Ġsk ipping +l ves +Ġtsun ami +37 4 +ĠU rug +ĠE id +rec ated +p hen +Ġfault s +ĠStart ed +9 50 +Ġp i +Ġdetect or +Ġbast ard +Ġvalid ated +Space Engineers +OUR CE +Ġ( ~ +Ġuns ur +Ġaff irmed +Ġfasc ism +Ġres olving +ĠCh avez +ĠC yn +Ġdet ract +L ost +Ġrig ged +Ġhom age +ĠBrun o +55 5 +ec a +Ġpress es +Ġhum our +Ġsp acing +Ġ' / +olk ien +C oun +OP ER +T re +S on +ĠCambod ia +ier re +m ong +o zy +Ġliquid ity +ĠSov iets +ĠFernand o +Ġ2 29 +Ġsl ug +ĠCatal an +elect ric +Ġsc enery +ĠH earth +Ġconst rained +Ġgoal ie +ĠGu idelines +ĠAm mo +ĠPear son +Ġtax ed +Ġfet us +Resp onse +ĠAlex is +th ia +G uy +Ġrecon struct +Ġextrem es +Ġconclud ing +ĠP eg +ook s +Ġded uctions +R ose +Ġground breaking +ĠT arg +ãĥ ģ +ĠRe ve +res ource +Ġmo ons +Ġelectrom agnetic +Ġamid st +ĠVik tor +N ESS +B ACK +Ġcomm ute +ĠAna heim +Ġfluct uations +6 40 +Ġnood les +ĠCop enhagen +ĠT ide +ĠGri zz +ĠS EE +Ġpip elines +Ġsc ars +end o +ag us +ĠE TF +/ # +ĠBec ome +44 8 +Ġvis c +ĠRecomm ended +Ġj umper +Ġcogn ition +Ġassass in +Ġwitness ing +ĠSet up +Ġl ac +v im +IS M +p ages +SS L +35 8 +Ġad ject +indust rial +l ore +cher y +Ġgl itter +Ġc alf +Flor ida +Ġspoil ers +Ġsucceed s +Ġch anting +Ġslog ans +ĠTr acy +Vis it +rol ogy +Ġm ornings +Ġline age +Ġs ip +Ġintense ly +Ġflour ish +ĠSle eping +ĠF em +or por +ĠK lan +ĠDar th +h ack +ĠNi elsen +Ġtum ors +Ġprocure ment +ĠY orkshire +Ġra ided +K Y +An na +Ġ// [ +ĠDis order +ĠMust ang +ĠW en +ĠTry ing +s q +Ġdeliver ies +Ġshut ter +Ġcere bral +Ġbip olar +ĠC N +l ass +j et +Ġdeb ating +> : +Ġe agle +gr ades +ĠD ixon +UG C +M AS +ĠDr aco +ĠMach ines +aff er +Ġem an + ² +pr on +ĠG ym +Ġcompar atively +ĠTrib unal +PR O +Ġle x +Ġfert ile +Ġdep ressing +Ġsuperf icial +ess ential +ĠHun ters +g p +Ġprom inence +L iber +ĠAn cest +ote chnology +Ġm ocking +ĠTra ff +ĸ ļ +Med ium +I raq +Ġpsychiat rist +Quant ity +ĠL ect +Ġno isy +5 20 +G Y +Ġsl apped +ĠM TV +Ġpar a +p ull +Mult iple +as her +Ġn our +ĠSe g +Spe ll +v ous +ord ial +Sen ior +ĠGold berg +ĠPl asma +ne ed +Ġmess enger +ere t +Ġteam ed +Ġliter acy +ĠLe ah +ĠD oyle +Ġem itted +U X +Ġev ade +Ġm aze +Ġwrong ly +ĠL ars +Ġstere otype +Ġpled ges +Ġarom a +ĠM ET +Ġac re +ĠO D +Ġf f +Ġbrew eries +ĠH ilton +und le +ĠK ak +ĠThank fully +ĠCan ucks +in ctions +ĠApp ears +Ġco er +Ġundermin ed +ro vers +And re +Ġbl aze +um ers +Ġfam ine +amp hetamine +ulk an +Am ount +Ġdesper ation +wik ipedia +develop ment +ĠCor inth +uss ia +Jack son +L I +N ative +R s +Oh io +ĠKath leen +F ortunately +Ġattend ant +ĠPre ferred +ĠDid n +ĠV s +M is +Ġrespond ent +Ġb oun +st able +Ġp aved +Ġunex pl +ĠChe ney +L M +ĠC ull +bl own +Ġconfront ing +oc ese +serv ing +W i +ĠLith uania +ann i +Ġst alk +h d +Ġv ener +AP H +ynchron ous +UR R +um ably +hist oric +H alf +H ay +Ġresil ience +spe ction +Ġabandon ing +O bs +ĠDeb bie +Ġgrad ient +ĠPl aint +ĠCan al +AR CH +Ġexpans ive +Ġfun g +Ġb ounced +U nd +Ġprec autions +Ġclar ification +Ġd agger +Ġgri ps +Ġ µ +ĠRiver a +ĠUnd ead +is ites +ĠFIR ST +ñ o +aud i +Ġhost ages +Ġcompl iant +Ġal umni +Se ven +Ġcyber security +e ither +Col lect +Ġinvari ably +ĠS oci +Ġlaw maker +Ġa le +ĠPerson ally +N azi +Ġcustom ization +ĠPro c +ĠSask atchewan +eat uring +Ġsp ared +Ġdiscontin ued +Ġcomput ational +ĠMotor ola +Ġsuprem acist +government al +Ġparad ise +ĠDown ing +ĠNik on +Ġcat alyst +ber ra +Tor onto +8 75 +bet a +ĠMac ron +Ġunreal istic +ve ctor +ĠVeh icles +it iveness +ĠR V +ĠCol bert +s in +o ji +ent in +ĠKr ish +hell o +ff ield +ok y +ĠT ate +Ġmap le +Ġa ids +chem ical +33 4 +n uts +ĠWar p +Ġx x +ĠRob b +umer ous +_- _ +ft ime +ĠV W +Ġw inger +ĠD ome +t ools +ĠP V +ĠGe orgetown +Ġg eared +Ġjihad ists +Ġc p +Ġster oids +M other +cler osis +ĠDR M +nes ia +Ġl inger +Ġimm ersive +ĠC OUN +Ġoutwe igh +ens ual +B and +Ġtransform s +mat ched +ps ons +ĠJud icial +f actor +Ġrefer ral +Ġodd ly +ĠW enger +B ring +ĠB ows +60 2 +IC LE +Ġl ions +ĠAcad emic +ĠTh orn +ĠRa ider +kef eller +St orage +L ower +ĠOr t +ĠEqu ality +AL T +ĠS OC +T ypes +Ġl yn +ĠAss et +co at +TP P +C VE +ĠPione er +app lication +Mod ern +ĠH K +En vironment +Al right +R ain +IP P +ĠShi ite +Ġm ound +ĠAb ilities +cond ition +St aff +Ġcompet ence +ĠM oor +ĠDi ablo +Ġwith held +Ġost ensibly +ĠB rom +Ġms g +Ġden omin +ĠRef erences +ĠF P +Ġplun ged +Ġp amph +m oving +cent ral +Ġdown right +Ġf ading +T al +T yp +ĠTh y +uk es +it he +Ġo ve +Ġbatt led +Ġseaf ood +Ġfig ur +ĠR D +c rop +Ġsqu ads +{ \ +à ¹ +ĠE h +Ġinterview ing +ĠQ in +Ġas piring +PL IC +Ġcla uses +ĠG ast +ĠN ir +Ġl uggage +Ġh ose +Ġsystem d +Ġdesc ending +ĠRev ised +ĠR ails +al ign +70 9 +33 7 +Ġf ug +charg ing +t ags +Ġut er +k ish +WAR NING +49 0 +prof its +Ġvoy age +Ġa ce +ĠV anguard +ĠT anks +ĠM uk +Ġ2 26 +S afe +Ar mor +Ġvolcan ic +Ġwom b +ĠM IL +Ġbegin ner +ĠRec ogn +ĠA AP +PL AY +) ! +Ġdetect ing +c n +Ġbre aches +Bas ically +ĠP ag +ĠMunicip al +ĠInd ie +ĠL af +ĠDis able +ĠOl son +Ġrest rained +Ġrul ings +Ġhum ane +ev ents +ĠCinem a +display Text +ĠH atch +action Date +onna issance +Ġassault ing +ĠL ug +CH AT +Ġvig orous +ĠPer se +Ġintoler ance +ĠSnap chat +ĠSh arks +Ġd ummy +ĠDi agn +ĠGu itar +im eters +40 3 +RE G +A x +Ġsepar ates +ĠMah m +Ġt v +j ah +O OL +C irc +ĠWinds or +uss ian +Ġintu ition +Ġdis dain +ĠDon ovan +Ġ2 21 +E mb +Ġcondem ning +Ġgener osity +zz y +Ġpant ies +ĠPre vent +Action Code +AN A +34 2 +external ActionCode +Ġspec ifying +Ġcryst all +J ere +Ġru pt +ĠApp rentice +Ġprof iling +Ð º +St rike +Ġsid eline +Ġoblig ated +Ġocc ult +Ġbureaucr atic +ant ically +rupt ed +neg ative +ĠEthiop ia +ĠC ivic +Ġins iders +el igible +ĠTV s +ĠB AR +ĠT I +i ologist +ĠA IR +Ġsubstit uted +Ar ab +ĠS aul +ĠY og +p rem +Ġbuild ers +Ġstation ary +Ġdoubt ful +Ġvig orously +Ġthr illing +Ph ysical +ĠCare y +ĠHyd ra +geon ing +ĠS ly +y ton +Ġborrow ers +ĠPark inson +Ġ ë +ĠJama ica +Ġsat ir +Ġinsurg ents +ĠF irm +Ġis ot +ĠK arn +our ning +ak ens +doc s +l ittle +ĠMon aco +CL ASS +Tur key +L y +ĠCon an +ass ic +Ġstar red +ĠPac ers +et ies +Ġt ipping +M oon +ĠR w +s ame +Ġcav ity +Ġgo of +ĠZ o +Sh ock +um mer +Ġemphas izes +Ġreg rett +Ġnovel ty +Ġen vy +ĠPass ive +r w +50 5 +Ġind ifferent +ĠR ica +ĠHim self +ĠFred die +Ġad ip +ä¸ Ģ +Ġbreak out +Ġhur ried +ĠHu ang +ĠD isk +Ġro aming +?????- ?????- +U V +ĠRick y +ĠS igma +Ġmarginal ized +Ġed its +Ġ30 4 +mem ory +Ġspec imen +29 3 +ãģ ¯ +Ġvert ically +Ġaud ition +ĠHe ck +Ġc aster +ĠHold ings +ad al +ĠC ron +ĠL iam +Ġdef lect +P ick +ĠDeb ug +RE F +Ġvers atility +ot hes +class ified +ĠMah ar +ĠH ort +C ounter +st asy +not iced +33 1 +ĠSh im +f uck +ĠB ie +Ġair ing +ĠPro tein +ĠHold ing +Ġspect ators +ili ated +ĠThat cher +n osis +ãĥ¼ ãĥ³ +Te le +B oston +ĠTem pl +st ay +Ġdecl arations +47 9 +Vol ume +ĠDesign er +ĠOver watch +id ae +Ġon wards +Ġn ets +ĠMan ila +part icularly +Ġpolit ic +o other +Ġport raits +Ġpave ment +c ffff +Ġs aints +Ġbegin ners +ES PN +Ġshort comings +âķIJ âķIJ +Ġcom et +ĠOrgan ic +qu el +Ġhospital ized +Bre ak +Ġpe el +dyl ib +asp x +ur ances +ĠT IM +P g +Ġread able +ĠMal ik +Ġm uzzle +Ġbench marks +d al +ĠV acc +ĠH icks +60 9 +ĠB iblical +he ng +Ġover load +ĠCivil ization +Ġimm oral +Ġf ries +ãĤ Ĵ +Ġreprodu ced +Ġform ulation +j ug +ire z +g ear +Ġco ached +Mp Server +ĠS J +ĠK w +In it +d eal +ĠO ro +ĠL oki +ĠSong s +Ġ23 2 +ĠLou ise +asion ally +Ġunc ond +olly wood +Ġprogress ives +ĠEn ough +ĠDo e +Ġwreck age +Ġbr ushed +ĠBase Type +Ġz oning +ish able +het ically +ĠC aucus +ĠH ue +Ġk arma +ĠSport ing +Ġtrad er +Ġseem ing +ĠCapt ure +4 30 +b ish +Ġt unes +Ġindo ors +ĠSp here +ĠD ancing +TER N +Ġno b +ĠG ST +m aps +Ġpe ppers +F it +Ġoverse es +ĠRabb i +ĠR uler +vert ising +off ice +xx x +Ġra ft +Ch anged +Ġtext books +L inks +ĠO mn +ãĢ ij +Ġinconven ience +ĠDon etsk += ~ +Ġimplicit ly +Ġboost s +ĠB ones +ĠBo om +Cour tesy +Ġsens ational +AN Y +Ġgre edy +ed en +Ġinex per +ĠL er +ĠV ale +Ġtight en +ĠE AR +ĠN um +Ġancest or +S ent +ĠH orde +urg ical +all ah +Ġsa p +amb a +ĠSp read +tw itch +Ġgrand son +Ġfract ure +Ġmoder ator +ĠSe venth +ĠRe verse +Ġestim ation +Cho ose +Ġpar ach +Ġbar ric +ãĢ IJ +Ġcomp ass +Ġall ergic +âĢ ķ +OT HER +err illa +Ġw agon +Ġz inc +Ġrub bed +ĠFull er +ĠLuxem bourg +ĠHoo ver +Ġli ar +ĠEven ing +ĠCob b +est eem +Ġselect or +ĠB rawl +is ance +ĠE k +Ġtro op +Ġg uts +ĠApp eal +ĠTibet an +Ġrout ines +ĠM ent +Ġsummar ized +steam apps +Ġtr anqu +Ġ19 29 +or an +ĠAut hent +Ġg maxwell +Ġappre hens +Ġpo ems +Ġsa usage +ĠWeb ster +ur us +Ġthem ed +Ġl ounge +Ġcharg er +Sp oiler +Ġsp illed +h og +ĠSu nder +ĠA in +ĠAng ry +Ġdis qual +ĠFrequ ency +ĠEther net +Ġhel per +Per cent +Ġhorr ifying +Ġa il +ĠAll an +EE E +ĠCross ing +44 9 +Ġh olog +ĠPuzz les +ĠGo es +eren n +60 4 +ãģ ı +ĠRaf ael +Ġatt en +ĠE manuel +Ġup ro +ĠSus p +P sych +ĠTr ainer +ĠN ES +ĠHun ts +bec ue +Ġcounsel or +R ule +Ġtox ins +Ġb anners +r ifice +Ġgreet ing +Ġfren zy +Ġall ocate +Ġ* ) +ex pr +50 3 +ĠCh ick +ĠT orn +Ġconsolid ation +ĠF letcher +sw itch +fr ac +cl ips +ĠMcK in +ĠLun ar +Mon th +IT CH +Ġscholar ly +rap ed +39 8 +Ġ19 10 +Ġe greg +Ġin secure +Ġvict orious +cffff cc +Ġsing led +Ġel ves +ĠW ond +bur st +Ġcam oufl +ĠBL ACK +Ġcondition ed +ç ī +ans wered +Ġcompuls ory +asc ist +Ġpodcast s +ĠFrank furt +bn b +Ġne oliberal +ĠKey board +ĠBel le +w arm +Ġtrust s +Ġins ured +ĠBu cc +us able +60 7 +ĠPl ains +Ġ18 90 +Ġsabot age +Ġlod ged +f elt +Ġg a +ĠN arc +ĠSal em +Ġsevent y +ĠBl ank +p ocket +Ġwhis per +Ġm ating +om ics +ĠSal man +ĠK ad +Ġan gered +Ġcoll isions +Ġextraord inarily +Ġcoerc ion +G host +b irds +è Ģ +k ok +Ġper missible +avor able +Ġpo inters +Ġdiss ip +ac i +Ġtheat rical +ĠCos mic +Ġforget ting +Ġfinal ized +å¤ § +y out +l ibrary +Ġbo oming +ĠBel ieve +ĠTe acher +ĠL iv +ĠGOOD MAN +ĠDomin ican +OR ED +ĠPart ies +Ġprecip itation +ĠSl ot +R oy +ĠComb ined +Ġinteg rating +Ġch rome +Ġintest inal +ĠRe bell +Ġmatch ups +Ġblock buster +ĠLore n +ĠLe vy +Ġpre aching +ĠS ending +ĠPur pose +ra x +f if +Ġauthor itative +ĠP ET +ast ical +Ġdish on +Ġchat ting +Ġ"$ :/ +Connect ion +Ġrecre ate +Ġdel inqu +Ġbro th +ĠD irty +ĠAd min +z man +Ġscholars hips +Ġ25 3 +cont act +als a +7 67 +c reen +abb age +Ġ19 15 +Ġbl ended +Ġal armed +L anguage +35 6 +Ġbl ends +ĠCh anged +W olf +Ġhe pat +Creat ing +Ġper secut +Ġsweet ness +art e +Ġforfe iture +ĠRober to +im pro +N FL +ĠMag net +Det ailed +Ġinsign ificant +ĠPOL IT +ĠBB Q +ĠC PS +Ġse aw +amin er +m L +end if +f inals +Ġ26 5 +u ish +Ġ} ) +ĠPro blems +Ġem blem +Ġserious ness +Ġpars ing +Ġsubst itution +Ġpress ured +Ġrecy cled +ale b +Rub y +Ġprof iciency +Dri ver +ĠW ester +: ' +AF TA +Ġm antle +ĠClay ton +fl ag +Ġpractition er +c overed +ĠSt ruct +add afi +4 25 +ĠTown ship +ĠHyd ro +Lou is +34 3 +Ġcond o +ĠT ao +Ġutil ization +Ġnause a +ĠDem s +rid ges +p ause +Ġform ulas +Ġchall enger +37 6 +Ġdefect ive +ĠRail way +ĠPub Med +Ġyog urt +l bs +ĠNor folk +OP E +ĠMood y +Ġdistribut or +Ġscroll s +Ġextract s +St an +Ġv iability +Ġexp oses +Ġstar vation +ĠStep s +ĠD odd +f ew +ST D +33 2 +Ġclos ures +Ġcomplement ary +ĠS asha +ump y +Ġmon et +Ġartic ulate +ĠDo ct +k iller +Ġsc rim +Ġ2 64 +Ġprost itutes +Ġse vered +Ġattach ments +Ġcool ed +L ev +ĠF alk +f ail +Ġpolic eman +ĠD ag +Ġpray ed +ĠK ernel +Ġcl ut +Ġc ath +Ġan omaly +St orm +em aker +ĠBreak fast +ul i +o ire +J J +h z +Oper ation +ĠS ick +35 4 +ĠGuatem ala +R ate +Ġexp osures +f aces +ĠArch ae +ra f +ĠM ia +Ġ20 25 +Ġop aque +Ġdisgu ised +ĠHead quarters +S ah +Ġp ots +9 78 +ĠM alf +Ġfrown ed +Ġpoison ous +ĠCon vers +ee ks +Ġcr ab +." " +Ġtre ason +Ġr anc +Ġescal ating +Ġwar r +Ġmob s +Ġl amps +ĠSun shine +ĠBrun swick +Ph ones +Ġspe lled +ĠSk ip +Ġ20 50 +Ġ19 11 +ĠPl uto +ĠAm end +Ġme ats +38 7 +Ġst omp +ĠZh ou +ĠLevi athan +ĠHaz ard +ad v +ĠOr well +Ġal oud +Ġb umper +ĠAn arch +ub untu +ĠSer ious +f itting +ĠOption al +ĠCec il +RE AM +Ġser otonin +Ġcultiv ate +ag ogue +} \ +Ġmos ques +ĠSun ny +Ġre active +rev olution +ĠL up +ĠFed ora +Ġdefense man +ĠV ID +ist ine +Ġdrown ing +ĠBroad casting +Ġthr iller +ĠS cy +Ġacceler ating +Ġdirect s +od ied +b ike +d uration +Ġpain fully +R edd +Ġproduct ions +Ġg ag +Ġwh ist +Ġs ock +Ġinf initely +ĠConc ern +ĠCit adel +Ġlie u +Ġcand les +ogene ous +arg er +Ġheaven ly +inflamm atory +Per formance +C s +ruct ose +az aki +Ġp essim +Ġinf erence +Ġpow d +ĠZ oe +Ġpain ts +Ġd azz +pt a +-------- --- +Ġins pir +ĠExper imental +ĠKn ife +reg or +b ors +Ġshow ers +rom eda +Ġs aint +Ġben ign +ĠJ iang +Ġenvision ed +Ġsh roud +IF T +H O +Ġsh uff +ĠI CC +Ġse greg +Ġrevis it +ighth ouse +L i +Ġsub strate +ĠSe as +ĠRew ard +ĠH ep +ĠBr ass +s bm +Ġelim inates +Ġst amina +ĠV AT +ĠLo an +Ġconst raint +Ġappropri ated +Ġp es +ĠA LE +r anging +Ġ40 4 +39 2 +Ġintellectual s +ach u +Ġrestruct uring +ĠLe vin +Ġrun es +Ġdelight ful +Ġcarbohyd rates +ĠMod els +ĠExp o +Ġtransport ing +all oc +Ġring ing +S amsung +Ġscarce ly +ĠURL s +ĠM AS +Ġprot otypes +Ġnarr ator +ĠCPU s +cd n +ĠBart on +Ġdecided ly +ĠSh u +ix ir +oc ious +ĠMy st +N intendo +Ġre use +Ġforg iven +F ew +in ical +n at +Ġseam less +ĠEv a +ĠE VE +ĠJ O +land ers +Ġso fter +neg ie +Ġtrans ient +Ġorb ital +Ġfulf il +ĠK om +Hop efully +Ġdynam ically +ĠHun ger +å Ľ +ĠArmen ia +el man +ber to +Ġp ige +ĠID s +lim it +Ġve ins +Ġso aring +p acks +Gold en +ĠCr ab +ist or +ĠR PM +Ġ$ $ +g ression +Ġjihad ist +Ġgam ble +Ġcare g +Ġinf lated +F ace +ĠFire arms +ĠEm manuel +â Ŀ +Ġsh ocks +gr ab +Ġspl end +ĠHP V +ab ortion +Ab ove +Ent ity +play ers +Ġcomm enced +ul ence +Ġfulfill ment +Ġembod iments +ĠW elfare +Ġha il +Ġ< @ +tt en +Ġcat cher +ĠJ azeera +Ġvolcan o +Ġstabil ize +ĠHand ler +Ġintens ified +ĠAb rams +Ġhum iliation +p aced +60 5 +ĠCent OS +Spe cific +Ġhe ed +ĠC AM +ĠGal ile +D ie +Ġabol ished +ĠThom son +ĠTe achers +ĠW ass +j ong +ĠIS BN +ĠAll ies +sh ake +å · +v ict +How ard +Ġde em +Ġexceed ingly +ĠSmart stocks +ib e +Ġdoor way +Ġcompet ed +ig mat +Ġnational ists +Ġg room +ĠKe en +Ġdispos able +de cl +ĠT olkien +ĠSche me +Ġb iod +Ġav id +ĠEl on +ag ar +ĠT SA +R oman +Ġartific ially +Ġadvis ors +X L +ĠInf erno +36 6 +Ġted ious +ĠPhot ography +ĠCar rie +Ġtro pe +ĠSand ra +Ġdec imal +Que en +ĠGund am +ĠO M +ote ch +N BA +Ġ19 32 +Ġent renched +ĠMar ion +Ġfr aternity +Lab our +Hen ry +Ġlat itude +E ither +Ġenh ances +ĠPot ential +Ġsh ines +id ad +Ġbread th +Ġcapac ities +ĠðŁ ĻĤ +ĠBron x +Ġsex es +Ġdifferent iation +Ġheavy weight +ĠT aj +d ra +Ġmigr ate +Ġexhaust ion +ĠR UN +els ius +ĠCu omo +Ġgu itars +Ġcl ones +ĠSom ew +ĠP ry +------------ - +Ġwarr anted +cy cles +Ġsalv age +Ġdis ks +R ANT +ĠNGO s +ĠMart ian +":[ {" +Ġadd icts +oj ure +il let +Ġamazing ly +art ments +p ixel +ĠGPU s +Lay out +è £ +ĠTam il +ĠBas il +Ġimpart ial +ĠSt ructure +f ork +b ryce +Ġr idge +ĠHamb urg +ri ous +Ġbl itz +cig arettes +Ġcan ned +40 2 +Ġiron ically +Ġcompassion ate +ĠHaw kins +. # +ĠCat hedral +Ġrall ied +in ternal +Ġqu ota +st akes +T EXT +m om +Ġcomple tes +Ġ23 8 +Ġsh rug +ãĥ ij +ĠN inth +Ġrev ise +ĠProv ider +Ġtre acher +Ġqu asi +ĠPR ES +Ġdep osition +Ġconfidential ity +iss ors +Ġim balance +Ġspan ning +Ġang ular +ĠC ul +commun ication +ĠNor a +ĠGen ius +op ter +Ġs acked +Sp ot +Ġfine ly +ĠCH R +28 2 +w aves +Pal est +ĠRo hing +N L +è ¿ +Ġsh itty +ĠSc alia +4 75 +Pro gress +Ġreferen cing +Ġclass rooms +ab ee +Ġs od +hes ion +70 8 +ĠZucker berg +ĠFin ish +ĠScot ia +ĠSav ior +ĠInstall ation +an tha +( - +Ġ30 2 +ĠP unk +Ġcr ater +yout u +Ġro ast +Ġinflu encing +Ġd up +ĠJ R +ĠG rav +Ġstat ure +Ġbath rooms +A side +W iki +me an +ĠZ ak +ĠOn es +ĠN ath +Ġhyper t +Ġcommence ment +C ivil +Ġmoder ately +Ġdistribut ors +Ġbreast feeding +Ġ9 80 +ĠS ik +ĠC ig +ĠAM ER +R IP +ĠCare er +ust ing +Ġmess ed +Ġe h +ĠJ ensen +/ $ +Ġblack mail +Ġconvers ions +Ġscientific ally +Ġmant ra +p aying +Ġiv ory +ĠCour ts +OU GH +aunt let +Ser ial +B row +ĠH undreds +3 23 +Ġpe e +Ġlin ux +Ġsub mer +ĠPrinc ipal +48 5 +ĠD SL +ĠCous ins +Ġdoctr ines +ĠAthlet ics +Ġ3 15 +ĠK arma +Ġatt ent +ur ger +Ġpresc ribe +Ġenc aps +ĠC ame +Ġsecret ive +ĠCr imes +d n +C lean +ĠEgypt ians +ĠCar penter +Ġ ll +H um +ĠMil o +Ġcapital ists +Ġbrief ed +T we +ĠBas in +elve t +M os +Ġplun ge +ĠKa iser +ĠFu j +ill in +Ġsafegu ards +Ġo ste +ĠOpportun ity +ĠM afia +ĠCall ing +ap a +ur ban +br ush +ill ard +c é +int elligence +ĠL ob +ĠDru id +Ġsm oother +Ġfoot ing +Ġmotor ists +arc ity +Ġmascul inity +Ġm ism +Ġabdom inal +ĠTa vern +ĠR oh +Ġesc apes +s igned +Anth ony +Ġsacrific ing +Ġintim acy +Ġan terior +ĠK od +Ġmot if +Ġg raz +Ġvisual ization +Ġguitar ist +ĠTro tsky +m agic +D ar +ĠMor i +Ġw ards +Ġtoile ts +l est +Ġtele port +ĠSund ays +ĠPl at +ET S +Ġe Sports +Pat rick +ĠK atherine +en ko +Ġhas sle +ĠM ick +gg les +Ġh ob +aint ain +Ġair borne +Ġsp ans +Ġch ili +Ġa perture +Ġvolunte ered +ĠInc ident +ĠF res +ĠVeter an +augh tered +ing o +Ġun insured +CL OSE +Ġf use +Ġer otic +Ġadvert ise +ra ising +Text ure +Ġatt ends +ĠRE AL +udd led +Ġsm oot +Ġ30 5 +ĠWill is +Ġbl ond +An alysis +ĠV T +on ica +Ġstrongh old +R F +N M +. >> +Ġprosper ous +Ġbo asted +29 2 +ĠManufact uring +PR ESS +g ren +Ġpharm acy +ĠRoc kefeller +k ai +Ġth umbs +ĠH ut +Ġmother board +Ġguard ians +ĠAl ter +ll ular +Ġsh ack +Ġwise ly +Ġback bone +erv a +Ġsu icides +ĠMcG regor +ij ah +E mer +ĠB rav +Ġdesign ate +P OST +produ ced +Ġcleans ing +irl wind +ex istent +ĠHum ph +ĠPay ne +Ġv ested +Å ¡ +Ġstring ent +ion a +Ġuns ub +Ġsum med +ĠHer cules +sub ject +ĠR agnar +ĠN os +Ġcharacter ization +Ġsav vy +ĠDaw son +ĠCas ino +Ġf ri +ĠBar rier +Ġmis information +Ġins ulation +Ġcorrid ors +Ġair planes +ĠNo ct +ah i +Ġ19 16 +k b +arm ac +Ġsh un +Ġsche ma +Ġhorr ified +Ġ23 9 +aund ers +N B +i ates +er ity +ĠSh ard +Ġr arity +Ġgroup ed +ĠGh ana +again st +ĠBi ological +ĠA ware +ow ell +Ï Ħ +ĠBe au +sh aw +H ack +ĠJul ius +US S +ol son +aun a +c ru +ĠMaur ice +ĠI k +Ġsequ encing +Ġradical s +Ġ( ?, +v irtual +Ġany ways +Ġreper c +Ġhand lers +Ġhes itant +é ĥ +ĠM F +ple mentation +ass ociated +Ġcampaign ed +ĠY ue +ut ations +ĠY oga +Ġsim mer +Ġro ds +Ġmel ody +Ġconv oy +v ideos +Ġscreen ed +N eg +ochem ical +Ġ( )) +Ġultr as +Ġant ip +ĠIsland ers +70 4 +Ġfet ish +Ġridic ulously +ĠK art +Ġmitochond rial +Ġinterf ering +Build er +Ġover fl +Ġac ne +ĠM ud +ĠK err +f lex +ĠPost al +ĠBalt ic +47 7 +ĠPers ons +our age +H B +ĠM use +ĠImm ortal +ĠDri ving +Ġpet itions +Ġsubsc ript +Ġs orce +ĠProcess or +ut on +S ony +Ġph on +Ġr aced +ĠAnth rop +Ġday time +ĠEx ercise +Add ing +Ġeng ages +ĠQual comm +Ġmir acles +Ġmem es +ĠDr ink +ĠOri oles +Ġhair s +ĠPol ar +ath om +Ġsl ippery +ĠR emy +Ġcar amel +ĠY EAR +Ġal k +I gn +a ution +ĠMer lin +ĠC ran +Ġap ologies +Ġ4 10 +Ġout ing +ĠMem ories +app ointed +Ġcount ered +u ld +pos ing +Ġfire wall +ĠW ast +ĠW et +work ed +se ller +Ġrepe aled +ere o +ass uming +BL IC +m ite +ĠCEO s +ĠChap el +ellig ent +________________ ________ +D og +Ġw art +Ġsubsc riber +s ports +Ġbe gged +ĠM V +Ġsem if +eth ical +Ġpre ach +Ġrev ital +Ġpun itive +Ġshort cuts +Ġinstit uted +ĠWars aw +Ġabdom en +ĠK ING +Ġsuper intendent +Ġf ry +ĠGe o +T OR +Ġcontrad ictions +apt ic +Ġlandsc apes +b ugs +Ġcl ust +Ġvol ley +c ribed +Ġt andem +Ġrob es +WH AT +Ġpromot er +Ġel oqu +review ed +ĠD K +ĠPl ato +Ġf ps +T ank +ĠDer rick +Ġpriorit ize +as per +ĠHond uras +ĠCom pleted +ne c +Ġm og +n ir +ĠMay o +DE F +st all +in ness +ĠVolks wagen +Ġprec aution +ĠM ell +i ak +ist ries +Ġ24 8 +Ġoverl apping +Sen ate +ĠEnh ance +res y +rac ial +OR TS +ĠM ormons +Str ong +ĠCo ch +Mex ico +ĠMad uro +Ġj ars +Ġcan e +W ik +oll a +iff erence +Ġphysic ist +ĠMag gie +Ġ28 5 +Ġdep iction +ĠMcL aren +J u +Ġsl ows +Ġcommission ers +ĠWill ow +ĠExpl os +hov ah +Ġtechn ician +Ġhom icides +ĠFl av +ĠTr uman +Ġ100 00 +u ctor +Ġsh ader +News letter +45 7 +Ġre ver +Ġhard ened +Ġwhere abouts +Ġrede velop +Ġcar bs +Ġtra vers +Ġsqu irrel +Ġfoll ower +Ġs ings +50 8 +Ġrabb its +emon ium +Ġdocument ing +Ġmisunder stood +) ' +R ick +gg ies +Ġprem ie +Ġsk ating +Ġpass ports +Ġf ists +aged don +H aw +AC P +0 80 +ĠThough ts +ĠCarl son +Ġpriest hood +h ua +Ġdun geons +ĠLo ans +Ġant is +Ġfamiliar ity +ĠS abb +op al +ĠIn k +st rike +Ġc ram +Ġlegal ized +Ġcu isine +Ġfib re +Tra vel +ĠMon ument +OD Y +eth y +Ġinter state +ĠP UR +em porary +ĠArab ian +develop ed +Ġsadd le +Ġg ithub +ĠOff er +ĠIS P +ro let +ĠSUP ER +ĠDen is +Ġmultipl ier +Ġstir red +Interest ingly +Ġcustom ary +Ġbill ed +he x +Ġmultipl ied +Ġfl ipping +ĠCros by +Ġfundament als +ia e +ĠPlay ed +ĠAt om +am azon +ĠFl am +ee z +activ ated +Ġtables poon +Ġliberal ism +ĠPal in +ĠP atel +N um +ĠT AM +Ġs urn +ĠRel oaded +Ġco ined +" ], +ĠCl ash +ĠAg u +Ġprag matic +ĠActiv ate +Ġ8 02 +Ġtrail ers +Ġsil hou +Ġprob es +Ġcirc us +ĠB ain +ĠLind say +ĠAb bey +Del ivery +Ġconcess ion +Ġgast ro +ĠSpr ite +Ä Ł +and el +Ġg imm +Ġaut obi +ĠT urtle +Ġwonder fully +ĠHar am +ĠWorld wide +ĠHand le +Ġtheor ists +Ġsle ek +ĠZh u +ograph ically +EG A +ĠOwn ers +ath s +ĠAntar ctic +n atal +=" " +fl ags +`` `` +Ġs ul +K h +Ġpot assium +Ġlinem an +Ġcere al +ĠSe asons +Ġ20 22 +Ġmat hematic +Ġastron omers +prof essional +Ġf ares +cknow led +Ġch i +Ġyoung sters +Ġmistaken ly +Ġhem isphere +ĠDiv inity +r one +Ġ" , +r ings +Ġattract s +v ana +å ¹ +C AP +Ġplay list +Ġpor ch +ãģ £ +Ġincorpor ates +Ġso ak +Ġassert ing +ĠTerror ism +ĠP ablo +J a +ces ter +Ġfear ing +ĠPr ayer +Ġescal ated +G W +Ġro be +ĠBright on +ac ists +ĠSym phony +ĠDwar f +ĠPar ade +ĠLe go +Ġinex pl +Ġl ords +le af +RA G +l iber +Ġcig ars +ĠJe hovah +60 6 +WIND OWS +ĠLiber ia +eb us +He avy +Ġl ubric +ĠR W +angu ages +Ġnarrow ed +com puter +ĠE mber +Ġmurder ing +Ġdown stream +ĠT uls +ĠT ables +Top ic +ĠAcc uracy += / +l ost +ĠRe i +Ġprogress es +b ear +Ġestablish ments +Just in +ĠPe ach +ĠG omez +å ¿ +ĠTri angle +Id ent +ĠH ive +Res ources +Ġmix es +ĠAss uming +M u +Ġhyp oc +Ġs ane +ĠW an +id ious +Su ccess +Ġ io +Ang el +Ġdanger ously +ĠCreat ure +W ORK +: [ +ĠKat rina +List ener +M iller +ĠId lib +h ang +Ġcircum vent +h ref +Ġcel estial +ĠWe eks +ĠP ug +ĠDal ton +Ġsubpoen a +uk u +Ġpers isted +pe i +old ing +ĠDoc uments +ĠH ast +ĠC ENT +Ġprim er +Ġsyn onymous +Ġn ib +om bs +Ġnot ation +ĠD ish +ĠAt mosp +Ġforb id +ĠAN G +pat tern +l os +Ġproject iles +b rown +." , +ĠVen om +Ġfierce ly +ub lished +ĠU ran +ĠNic arag +4 10 +ĠC AL +OT OS +ĠMir acle +ĠEn chant +Ġguard ing +app end +Att ach +Ġlevel ed +Ġcond oms +ih ilation +64 9 +Ġnight mares +ĠTHE Y +ĠST ART +ĠK inn +Ġroomm ate +Ġhy giene +o pping +J ob +Ġl vl +ĠV ER +ĠKe eping +ab etic +Ġformat ting +eral a +Ġrev isions +Ġres urg +T el +ĠGood man +35 3 +p od +Ġind isp +ĠTrans lation +Ġg own +ĠM und +Ġc is +Ġby stand +col lect +ĠPun jab +act ively +ĠG amb +te ll +Ġimport ing +g encies +Ġloc om +ĠBr ill +H oly +ĠBer ger +Ġshow down +Ġrespond ers +IL Y +Ġt akedown +le ted +Ġmat tered +Ġpredict ive +Ġover lay +G PU +ĠV ick +Ġconvey ed +T ab +pe er +Sc an +Ġdefensive ly +v ae +Ġappro ving +Ġt iers +ĠV ia +quer ade +ĠSaud is +Ġdemol ished +ĠProp he +Ġmon o +Ġhospital ity +H AM +ĠAri el +M OD +ĠTor ah +Ġbl ah +ĠBel arus +erent ial +ĠT uc +Ġbank er +39 7 +Ġmosqu it +ĠScient ist +ĠMus ical +Ġh ust +Sh ift +Ġtor ment +Ġstand off +E duc +ĠF og +Ġampl ifier +Sh ape +Inst ance +ĠCrit ics +Ġda emon +H ouston +Ġmatt ress +ĠID F +Ġobsc ene +ĠA mer +hett i +Ġcomp iling +35 2 +vere tt +ĠRed uction +ist ration +ĠBl essed +ĠB achelor +3 16 +Ġpr ank +ĠVul can +dd ing +Ġm ourning +ĠQu int +ĠBl aster +test ing +Ġsed iment +>> > +ĠE ternity +ĠWH ERE +ĠM aze +Ġreact ing +ĠAl v +oms day +ĠC RA +Ġtransl ator +Ġbog us +at u +We bsite +oll s +Ġbapt ism +Ġs ibling +ĠAut umn +ve z +ãģ® é +gu ards +Ge org +assad ors +ĠFre ud +Ġcontin ents +ĠReg istry +Bern ie +ĸļ 士 +Ġtoler ant +ĠU W +Ġhor ribly +99 5 +ĠMID I +Ġimpat ient +oc ado +er i +ĠWor st +ĠNor ris +ĠTalk ing +Ġdef ends +ens able +Ġ20 21 +Ġanat omy +L ew +Ġdraw er +ĠCan berra +Ġpatri otic +é¾įå ĸļ士 +ĠAv g +AR M +Ġundis closed +Ġfare well +45 9 +b able +ĠAll ison +OL OG +Ġcon co +t ight +ĠAC PI +ĠM ines +l ich +ĠâĶ ľ +represent ed +200 000 +Ġenthusi ast +OT S +b il +ĠIng redients +Ġinvent or +ĠMy SQL +³³ Âł +ĠAB OUT +with in +Ġm k +B ul +ĠF ake +Ġdracon ian +W a +hel m +ĠTer ran +erv ille +Ġcommon place +SI ZE +Ġ" < +re place +ograph s +ĠSE LECT +inc ible +ĠMost ly +ĠShe ffield +ĠID E +ugg le +Ġcit ations +h urst +ĠUn ix +Ġunle ash +ĠP iper +ĠN ano +Ġsucc umb +Ġreluct ance +Ġ25 00 +ĠMer chant +Ġwire t +Ġcomb os +ĠBirth day +Ġchar coal +ĠU PS +ĠFair fax +Ġdrive way +ĠT ek +ĠP itch +ove re +Ġtechn icians +ĠAct ual +fl ation +ĠF iscal +ĠEm pty +an amo +Ġmag nesium +Ġsl ut +Ġgrow ers +Invest igators +( ): +ĠS atellite +ĠKe ynes +miss ive +l ane +Ġb orough +3 44 +ĠTE AM +ĠBet hesda +C V +h ower +ĠR AD +Ġch ant +ĠR iy +Ġcompos itions +Ġmild ly +Ġmedd ling +Ġag ility +ane ers +5 01 +Ġsyn th +ling er +29 1 +Ġex claimed +Part y +Ġcont amin +ĠMan or +ĠResp ond +Ġpra ising +Ġman ners +fle et +Sum mer +ĠLy nd +ĠDef initely +gr im +Ġbow ling +st ri +ç Ľ +y nt +Ġmand ates +D IV +Ġreconc ile +view s +ĠDam on +vet te +F lo +ĠGreat est +il on +ic ia +Ġportray al +Ġcush ion +50 4 +19 79 +oss al +App lic +sc ription +Ġmit igation +AT S +p ac +Ġer ased +Ġdefic iencies +ĠHolland e +ĠX u +Ġb red +Ġpregn ancies +f emin +Ġem ph +Ġpl anners +Ġout per +utter ing +Ġperpet rator +Ġm otto +ĠEll ison +ĠNE VER +Ġadmitted ly +AR I +ĠAzerbai jan +Ġmill isec +Ġcombust ion +ĠBott le +ĠL und +ĠP s +ĠD ress +Ġfabric ated +Ġbat tered +Ġs idel +ĠNot ting +Fore ign +ĠJer ome +0 20 +ĠAr bit +Ġkn ots +ĠR IGHT +M oving +ãģ Ļ +Ġsur geries +Ġcour thouse +Ġm astered +Ġhover ing +ĠBr an +ĠAl ison +Ġsaf est +m ilitary +Ġbull ied +Ġbar rage +Read er +ES E +ĠGe ographic +T ools +3 14 +ĠGe ek +ro th +gl ers +ĠF IN +Ï ģ +ĠA ston +al tern +48 8 +Ġveter in +G amer +Ġint el +ren ches +Sh ield +Ġam nesty +ĠB har +Ġp iled +Ġhonor able +ĠInst itutes +Ġso aked +Ġcom a +ĠE FF +34 1 +by tes +ĠG mail +le in +ĠCanad iens +m aterial +I l +Ġinstruct ors +ĠK Y +Ġconce ive +ub b +ĠP ossible +Ġeas ing +ĠChrist ina +Ġcar ic +ĠHD R +R OM +Ġsho vel +de lete +Ġp uff +ĠCh anging +Ġseam lessly +Att ribute +Ġacqu isitions +ak ery +ĠE F +Ġaut istic +ĠT akes +ĠPow der +ĠSt ir +5 10 +ĠBub ble +sett ings +ĠF owler +Ġmust ard +Ġmore over +Ġcopyright ed +ĠLED s +15 00 +æ ī +ĠH IS +en f +Ġcust od +ĠH uck +G i +Ġim g +An swer +C t +j ay +ĠInf rastructure +Ġfeder ally +L oc +Ġmicro bes +Ġover run +dd s +ot ent +adi ator +>>>> >>>> +Ġtorn ado +Ġadj ud +Ġintrig ued +Ġs i +ĠRevel ation +pro gress +Ġburgl ary +ĠSai yan +ĠK athy +Ġser pent +ĠAndre as +Ġcomp el +ess ler +ĠPl astic +ĠAd vent +ĠPos itive +ĠQ t +ĠHind us +reg istered +ular ity +Ġrighteous ness +Ġdemon ic +u itive +ĠB DS +ĠGre gg +c ia +ĠCrus ade +ĠSina i +W ARE ++ ( +Ġme ll +Ġder ail +y ards +A st +Ġnotice ably +ĠO ber +R am +Ġun noticed +Ġse q +av age +T s +Ġ6 40 +Ġconced e +Ġ] ) +F ill +Ġcapt ivity +ĠImprove ment +ĠCrus ader +ara oh +M AP +æ Ĺ +Ġstr ide +al ways +F ly +N it +Ġal gae +ĠCook ing +ĠDo ors +Mal ley +Ġpolic emen +ãģ į +Ġastron aut +access ible +49 5 +ĠR AW +cl iffe +udic rous +Ġdep ended +al ach +Ġvent ures +ra ke +Ġt its +ĠH ou +Ġcond om +ormon al +Ġind ent +Ġupload ing +Foot note +Import ant +Ġ27 1 +Ġmind ful +Ġcont ends +C ra +Ġcal ibr +ĠO ECD +plug in +F at +ĠIS S +ĠDynam ics +ans en +68 6 +' ), +Ġsp rite +Ġhand held +ĠH ipp +=~ =~ +Tr ust +Ġsem antics +ĠBund es +ĠRen o +ĠLiter ature +s ense +G ary +ĠA eg +ĠTr in +EE K +Ġcler ic +ĠSS H +Ġch rist +Ġinv ading +ib u +Ġen um +aur a +Ġal lege +ĠInc redible +B BC +Ġth ru +Ġsa iled +Ġem ulate +Ġin security +Ġc rou +Ġaccommod ations +Ġincompet ent +Ġsl ips +ĠEarth qu +s ama +IL LE +Ġi Phones +as aki +Ġby e +Ġar d +Ġext ras +Ġsl aughtered +Ġcrowd funding +res so +Ġfil ib +ĠER ROR +ĠT LS +e gg +ĠIt al +Ġen list +ĠCatal onia +ĠSc ots +Ġser geant +Ġdiss olve +N H +Ġstand ings +ri que +I Q +Ġbenef iciary +Ġaqu arium +You Tube +ĠPower Shell +Ġbright est +ĠWar rant +S old +Writ ing +Ġbegin nings +ĠRes erved +ĠLatin os +head ing +Ġ4 40 +Ġrooft op +AT ING +Ġ3 90 +VP N +G s +k ernel +turn ed +Ġprefer able +Ġturn overs +ĠH els +S a +ĠShin ji +ve h +ĠMOD ULE +V iol +Ġex iting +Ġj ab +ĠVan illa +Ġac ron +ĠG ap +ber n +A k +ĠMc Gu +Ġend lessly +ĠFar age +ĠNo el +V a +M K +Ġbr ute +ĠK ru +ĠES V +ĠOl ivia +âĢ ł +ĠK af +Ġtrust ing +Ġh ots +3 24 +Ġmal aria +Ġj son +Ġp ounding +ort ment +Count ry +Ġpostp oned +Ġunequ iv +? ), +ĠRo oney +udd ing +ĠLe ap +ur rence +sh apeshifter +ĠH AS +os ate +Ġca vern +Ġconserv atism +ĠB AD +Ġmile age +Ġarrest ing +V aults +Ġmix er +Dem ocratic +ĠB enson +Ġauth ored +8 000 +Ġpro active +ĠSpirit ual +t re +Ġincarcer ated +ĠS ort +Ġpe aked +Ġwield ing +re ciation +×Ļ × +P atch +ĠEm my +Ġex qu +tt o +ĠRat io +ĠP icks +ĠG ry +ph ant +Ġf ret +Ġeth n +Ġarch ived +% - +c ases +ĠBl aze +Ġim b +c v +y ss +im ony +Ġcount down +Ġaw akening +ĠTunis ia +ĠRe fer +ĠM J +Ġun natural +ĠCar negie +iz en +ĠN uggets +he ss +Ġev ils +64 7 +Ġintrodu ctory +l oving +ĠMcM ahon +Ġambig uity +L abel +ĠAlm ighty +Ġcolor ing +ĠCl aus +set ting +N ULL +ĠF avorite +ĠS IG +> ( +ĠSh iva +ĠMay er +Ġstorm ed +ĠCo verage +we apons +igh am +Ġun answered +Ġle ve +Ġc oy +c as +b ags +as ured +Se attle +ĠSant orum +ser ious +Ġcourage ous +ĠS oup +Ġconfisc ated +Ġ// / +Ġuncon ventional +Ġmom s +ĠRohing ya +ĠOrche stra +ĠPot ion +Ġdisc redit +ĠF IL +f ixed +ĠDe er +do i +ĠDim ension +Ġbureaucr ats +et een +Ġaction Group +oh m +Ġb umps +ĠUt ility +Ġsubmar ines +ren heit +re search +ĠShap iro +Ġsket ches +Ġde ceptive +ĠV il +es ame +ĠEss entially +Ġramp age +isk y +Ġmut tered +th ritis +Ġ23 6 +f et +b ars +Ġpup il +ĠTh ou +o S +s ong +Ġfract ured +Ġre vert +pict ure +Ġcrit erion +us her +Ġreperc ussions +ĠV intage +ĠSuper intendent +Offic ers +Ġflag ged +Ġbl ames +Ġin verse +ograp hers +Ġmakes hift +Ġdev oid +Ġfoss ils +ĠArist otle +ĠFund s +Ġde pleted +ĠFl u +ĠY uan +Ġw oes +Ġlip id +Ġsit u +requ isites +Ġfurn ish +ĠSam ar +Ġshame ful +Ġadverse ly +Ġad ept +Ġrem orse +Ġmurder ous +uck les +ĠE SL +Ġ3 14 +s ent +Ġred ef +ĠC ache +ĠP urs +ig ans +Ġ4 60 +Ġpres criptions +Ġf res +F uck +ocr ates +Tw enty +ĠWe ird +ĠT oggle +ĠC alled +itiz ens +Ġp oultry +Ġharvest ing +ãĤ¦ ãĤ¹ +Bott om +Ġcaution ed +t n +39 6 +ĠNik ki +Ġeval uations +Ġharass ing +Ġbind ings +ĠMon etary +Ġhit ters +Ġadvers ary +un ts +Ġset back +Ġenc rypt +ĠC ait +Ġl ows +eng es +ĠN orn +Ġbul bs +Ġbott led +ĠVoy ager +3 17 +Ġsp heres +p olitics +Ġsubt ract +Ġsens ations +Ġapp alling +Ġ3 16 +Ġenvironment ally +ĠST EM +Ġpub lishes +5 60 +Ġdilig ence +48 4 +Ġadv ises +Ġpet rol +Ġimag ining +Ġpatrol s +ĠInt eger +ĠAs hes +act us +ĠRad iant +ĠL T +it ability +ht aking +Set ting +Ġnu anced +ĠRe ef +ĠDevelop ers +N i +pie ces +99 0 +Lic ense +Ġlow ers +ĠOtt oman +3 27 +oo o +Ġqu itting +mark ets +Beh ind +Ġbas in +Ġdoc s +an ie +fl ash +ct l +Ġcivil ized +ĠFuk ushima +"] ," +ĠK S +ĠHonest ly +ar at +Ġconstruct s +ĠL ans +ĠD ire +ĠLI KE +ĠTrou ble +Ġwith holding +ĠOb livion +Ġsan ity +any a +Con st +Ġgro cer +ĠC elsius +Ġrecount ed +ĠW ife +B order +ate red +h appy +Ġspo iler +Ġlog ically +H all +Ġsucceed ing +Ġpoly morph +Ġax es +ĠShot gun +ĠS lim +ĠPrin ciples +ĠL eth +art a +Ġsc or +Sc reenshot +Ġrelax ation +#$ #$ +Ġdeter rent +idd y +Ġpower less +Ġles bians +Ġch ords +ĠEd ited +se lected +Ġseparat ists +000 2 +Ġair space +Ġturn around +Ġc unning +P ATH +P oly +Ġbomb ed +Ġt ion +x s +Ġwith hold +Ġw aged +ĠLiber ties +Fl ag +Ġcomfort ing +45 4 +ĠI ris +are rs +Ġr ag +Ġrel ocated +ĠGu arant +Ġstrateg ically +Ġgam ma +uber ty +ĠLock heed +g res +Ġgr illed +ĠLow e +st ats +ĠR ocks +Ġsens ing +Ġrent ing +ĠGe ological +ا Ø +ot rop +Ġse w +Ġimproper ly +48 6 +Ġâĸ ł +Ġstar ving +ĠB j +Disc ussion +3 28 +ĠCom bo +ĠFix es +N AT +Ġstri ving +th ora +Ġharvest ed +ĠP ing +Ġplay ful +Ġaven ues +Ġoccup ational +Ġw akes +ĠCou rier +Ġdrum mer +ĠBrow ser +ĠH outh +it u +Ġapp arel +p aste +Ġhun ted +ĠSecond ly +l ain +X Y +ĠP IN +ic ons +Ġcock tails +Ġs izable +Ġhurd les +est inal +ĠRecre ation +Ġe co +64 8 +ĠD ied +m int +Ġfinger prints +Ġdis pose +ĠBos nia +ts y +22 00 +Ġins pected +ĠF ou +Ġf uss +Ġamb ush +ĠR ak +Ġmanif ested +Pro secut +Ġsuff ice +ren ces +Ġcompens ated +ĠC yrus +Ġgen us +ĠWolver ine +ĠTrend s +Ġh ikes +ĠSe en +Ġen rol +C old +Ġpol itely +ĠSl av +ĠRu pert +Ġey ewitness +ĠAl to +Ġun comp +Ġposter ior +M ust +ĠHer z +Ġprogress ively +Ġ23 4 +Ġind ifference +ĠCunning ham +Ġacadem ia +Ġse wer +Ġast ounding +ĠA ES +r ather +Ġeld est +Ġclim bs +ĠAdd s +Ġout cry +Ġcont ag +ĠH ouses +Ġpe pt +ĠMel ania +interest ed +ĠU CH +ĠR oots +ĠHub bard +ĠT BD +ĠRoman ian +fil ename +St one +ĠIm pl +Ġchromos ome +C le +d x +Ġscram bled +ĠP t +Ġ24 2 +OP LE +Ġtremend ously +St reet +Ġcra ving +Ġbund led +ĠR G +p ipe +Ġinj uring +Ġarc ane +Part icip +ĠHero ic +st y +Ġto pping +ĠTemp est +rent ices +b h +Ġpar anoia +ĠUnic ode +Ġegreg ious +Ġ\ ' +ĠOsw ald +Ġgra vel +ĠSim psons +Ġbl and +ĠGuant anamo +Writ er +lin ers +ĠD ice +J C +Ġpar ity +Ġs ided +Ġ23 7 +ĠPyr rha +at ters +d k +F ine +comp an +Ġform ulated +ĠId ol +il ers +hem oth +ĠF av +Ġintr usion +Ġcar rots +ĠL ayer +ĠH acker +Ġ ---------------- +Ġmoder ation +é ģ +oc oc +Ġcharacter ize +ĠTe resa +Ġsocio economic +Ġper k +ĠParticip ation +tr aining +ĠPaul o +ph ys +Ġtrust worthy +Ġembod ied +ĠMer ch +c urrency +ĠPrior ity +Ġte asing +Ġabsor bing +Ġunf inished +ĠCompar ison +Ġdis ple +writ ers +Ġprofess ions +ĠPengu in +Ġang rily +ĠL INK +68 8 +ĠCor respond +Ġprev ailed +Ġcart el +l p +as ms +ĠRed emption +ĠIslam ists +effect s +d ose +ĠL atter +ĠHal ifax +Ġv as +ĠTop ics +ĠN amed +advert ising +zz a +IC ES +Ġret arded +ach able +ĠPupp et +ĠItem Level +Ġret ract +Ġident ifiable +A aron +ĠB uster +s ol +hel le +as semb +H ope +r anged +B a +ĠP urch +é Ģ +ĠSir i +Ġarri vals +Ġ19 12 +Ġshort ened +Ġ3 12 +Ġdiscrep ancy +ĠTem perature +ĠWal ton +Ġkind erg +p olit +Ġrem ix +Ġconnect ors +ãĥĺ ãĥ© +ĠKazakh stan +dom inated +Ġsu gars +im ble +ĠPan ic +ĠDem and +ĠCol ony +on en +ĠM ER +7 75 +ur ia +aza ar +ĠDeg ree +P ri +Ġsun shine +Ġ25 1 +Ġpsychedel ic +Ġdigit ally +ĠBra un +Ġsh immer +Ġsh ave +ĠTel esc +ĠAst ral +ĠVenezuel an +ĠO G +Ġc rawling +Int eg +ĠFe ather +Ġunfold ing +Ġappropri ation +Ġè£ı è +ĠMob ility +ĠN ey +- . +b ilt +L IN +ĠT ube +ĠCon versely +Ġkey boards +ĠC ao +Ġover th +Ġla ure +>> \ +ĠV iper +ach a +Off set +ĠR aleigh +ĠJ ae +J ordan +j p +Ġtotal itarian +Connect or +Ġobserv es +ĠSpart an +ĠIm mediately +ĠSc al +C ool +Ġt aps +Ġro ar +P ast +Ġch ars +ĠB ender +ĠShe ldon +Ġpain ter +Ġbe acon +ĠCreat ures +Ġdownt urn +Ġh inder +ĠAnd romeda +à Ľ +cc oli +ĠF itness +et rical +Ġutil izes +Ġsen ate +Ġen semble +Ġche ers +T W +Ġaff luent +k il +ry lic +ord ering +Com puter +Ġgru esome +ost ics +ĠUb isoft +ĠKel ley +Ġw rench +Ġbourgeois ie +IB LE +ĠPrest on +w orn +ar ist +reat ing +Ġst ained +ar ine +Ġsl ime +EN N +Ġche sts +Ġground water +ann ot +ĠTr ay +ĠLoc ke +ĠC TR +Ġd udes +ĠEx ternal +ĠDec oder +Ġpar amed +ĠMed line +80 9 +ĠD inner +rup al +g z +ĠG um +ĠDem o +j ee +Ġd h +ber man +arch s +Ġen qu +ĠEp stein +Ġdevast ation +Ġfriends hips +ĠAr d +Ġ23 1 +ĠRub in +ĠDist ance +Ġsp urred +Ġd ossier +Ġover looking +\\\\\\\\ \\\\\\\\ +Fore st +ĠCom es +\ ", +ĠIran ians +Ġf ixtures +L aughs +Ġcur ry +ĠKing ston +Ġsqu ash +Ġcat alogue +Ġabnormal ities +Ġdigest ive +.... ..... +Ġsubord inate +og ly +Ġ24 9 +M iddle +Ġmass ac +Ġburg ers +Ġdown stairs +Ġ19 31 +39 4 +ĠV G +Ġl asers +ĠS ikh +ĠAlex a +der ived +Ġcycl ist +ãģ® éŃĶ +onel iness +!!!! !!!! +Ġbuff s +leg ate +Ġrap ing +Ġrecomm ending +ro red +Ġmult icultural +un ique +Ġbusiness men +Ġune asy +ĠM AP +Ġdisp ersed +cipl ine +J ess +ĠK erala +å § +Ġabst raction +Sur v +U h +Ġprin ters +ij a +ow der +Ġanalog ous +ĠA SP +af er +Ġunfold ed +Ġlevel ing +Ġbre ached +ĠH earing +Ġn at +Ġtransl ating +crit ical +Ġant agonist +ĠYes terday +Ġfuzz y +w ash +m ere +Ġbe wild +ĠM ae +V irgin +ph rase +Ġsign aled +ĠH IGH +Ġprot ester +Ġgar ner +unk nown +Ġk ay +Ġabduct ed +Ġst alking +am n +Ġdes erving +ĠR iv +ĠJ orge +Ġscratch ing +ĠS aving +ip ing +Ġte ase +Ġmission ary +ĠMor row +T IME +P resent +Ġchem otherapy +tern ess +ĠH omes +ĠP urdue +Ġst aunch +ĠWhit ney +ĠTH ERE +Î ¼ +iat us +ĠErn est +ĠDe ploy +Ġcove ted +F ML +ĠDial ogue +Ġex ited +f ruit +Ġner d +":" "," +Ġv ivo +ru ly +4 60 +ĠAm en +rehens ible +Ġâ ĺ +D IR +Ġad herence +Ġche w +ĠCo ke +ĠSerge i +dig ital +ĠNe ck +g ently +enth al +/ ) +Ġwe ary +Ġgu ise +ĠConc ord +ĠOn ion +at cher +Ġb inge +ĠDirect ive +Ġman ned +ans k +Ġill usions +Ġbillion aires +38 3 +oly n +odynam ic +ĠWhe at +ĠA lic +Ġcol oured +ĠN AFTA +ab o +Ġmac ros +ind ependent +s weet +Ġsp ac +ĠK abul +Ġ Ä +em e +Ġdict ated +Ġsh outs += { +Ġr ipping +ĠSh ay +ĠCr icket +direct ed +Ġanalys ed +ĠWAR RANT +ag ons +ĠBlaz ers +Ġche ered +Ġar ithmetic +ĠTan z +37 3 +ĠFl ags +Ġ29 5 +Ġw itches +ĠIn cluded +ĠG ained +ĠBl ades +G am +ĠSam antha +ĠAtl antis +ĠPr att +Ġspo iled +ĠI B +ĠRam irez +Pro bably +re ro +ĠN g +ĠWar lock +t p +Ġover he +Ġadministr ations +Ġt int +Ġreg iment +Ġpist ols +Ġblank ets +Ġep ist +Ġbowl s +Ġhydra ulic +Ġde an +Ġj ung +Ġasc end +70 5 +ĠSant iago +à ® +Ġun avoid +ĠSh aman +re b +Ġstem ming +99 8 +ĠM G +st icks +esthes ia +ER O +Ġmor bid +ĠGr ill +ĠP oe +any l +Ġdele ting +ĠSurve illance +Ġdirect ives +Ġiter ations +ĠR ox +ĠMil ky +F ather +Ġpat ented +44 7 +Ġprec ursor +Ġm aiden +ĠP hen +ĠVe gan +ĠPat ent +K elly +Redd itor +Ġn ods +Ġvent ilation +ĠSchwar z +Ġw izards +Ġomin ous +ĠHe ads +ĠB G +Ġl umber +ĠSp iel +Ġis Enabled +Ġancest ral +ĠSh ips +Ġwrest ler +ph i +Ġy uan +ĠRebell ion +Ġice berg +Ġmag ically +Ġdivers ion +ar ro +yth m +ĠR iders +ĠRob bie +ĠK ara +ĠMain tenance +ĠHer b +Ġhar ms +p acked +ĠFe instein +Ġmarry ing +Ġbl ending +ĠR ates +Ġ18 80 +Ġwr ink +ĠUn ch +ĠTor ch +desc ribed +Ġhuman oid +ilit ating +ĠCon v +ĠFe ld +IGH TS +Ġwhistlebl ower +ort mund +ets y +arre tt +ĠMon o +ĠI ke +ĠC NBC +ĠW AY +ĠMD MA +ĠIndividual s +Ġsupplement al +Ġpower house +ĠSt ru +F ocus +aph ael +ĠCol leg +att i +Z A +Ġp erenn +ĠSign ature +ĠRod ney +Ġcub es +idd led +ĠD ante +ĠIN V +iling ual +ĠC th +Ġso fa +Ġintimid ate +ĠR oe +ĠDi plom +ĠCount ries +ays on +Ġextrad ition +Ġdis abling +ĠCard iff +Ġmemor andum +ĠTr ace +Ġ?? ? +se ctor +ĠRou hani +ĠY ates +ĠFree ze +Ġbl adder +M otor +ĠProm ise +ant asy +Ġforesee able +ĠC ologne +cont ainer +ĠTre es +ĠG ors +ĠSin clair +Ġbar ring +key e +Ġsl ashed +ĠStat istical +é ĩ +Ġâĸ º +All ows +Ġhum ility +Ġdr illed +ĠF urn +44 3 +Ġse wage +Ġhome page +Ġcour tyard +Ġv ile +Ġsubsid iaries +aj o +direct ory +Ġam mon +V ers +charg es +Ġ} } +ĠCh ains +Ġ24 6 +n ob +Ġper cept +Ġg rit +Ġfisher men +ĠIraq is +ĠDIS TR +ĠF ULL +ĠEval uation +g raph +at ial +Ġcooper ating +Ġmel an +Ġenlight ened +Ġal i +t ailed +Ġsal ute +Ġweak est +ĠBull dogs +U A +ĠAll oy +Ġsem en +oc ene +ĠWilliam son +s pr +, âĢĶ +ĠG F +itt ens +Be at +ĠJ unk +iph ate +ĠFarm ers +ĠBit coins +ig ers +d h +ĠL oyal +p ayer +Ġentert ained +Ġpenn ed +Ġcoup on +Que ue +Ġweaken ing +c arry +Ġunderest imate +Ġshoot out +Ġcharism atic +ĠProced ure +Ġprud ent +in ances +Ġric hes +Ġcort ical +Ġstr ides +Ġd rib +ĠOil ers +5 40 +ĠPer form +ĠBang kok +Ġe uth +S ER +Ġsimpl istic +t ops +camp aign +Q uality +Ġimpover ished +ĠEisen hower +Ġaug ment +ĠH arden +Ġinterven ed +Ġlist ens +ĠK ok +Ġs age +Ġrub bish +ĠD ed +Ġm ull +pe lling +Ġvide ot +Produ ction +D J +m iah +Ġadapt ations +Ġmed ically +Ġboard ed +Ġarrog ance +Ġscra pped +Ġopp ress +FORM ATION +Ġj unction +4 15 +EE EE +S kill +Ġsub du +ĠSug gest +ĠP ett +Ġle tt +ĠMan ip +ĠC af +ĠCooper ation +T her +Ġreg ained +¶ æ +ref lect +Ġth ugs +ĠShel by +Ġdict ates +ĠWe iner +ĠH ale +Ġbatt leground +s child +Ġcond ol +h unt +osit ories +Ġacc uses +Fil ename +Ġsh ri +Ġmotiv ate +Ġreflect ions +N ull +ĠL obby +¥ µ +ĠS ATA +ĠBack up +Ñ ĥ +n in +ĠCor rection +Ġju icy +ut ra +ĠP ric +Ġrest raining +ĠAir bnb +ĠAr rest +Ġappropri ations +Ġsl opes +Ġmans laughter +Ġwork ings +ĠH uss +ĠF rey +Le ave +ĠHarm ony +ĠF eder +Ġ4 30 +Ġt rench +Ġglad ly +Ġbull pen +ĠG au +b ones +Ġgro ove +Ġpre text +ã ħĭ +Ġtransm itter +ĠComp onent +Ġunder age +ĠEm pires +T ile +Ġo y +ĠMar vin +ĠC AS +Ġbl oss +Ġrepl icated +ĠMar iners +Marc us +ĠBl ocks +Ġliber ated +Ġbutter fly +Fe el +Ġfer mentation +Ġyou tube +Ġoff end +ĠTer m +res ist +Ġcess ation +Ġinsurg ency +Ġb ir +ĠRa ise +59 5 +Ġhypothes es +50 2 +Ġpl aque +ocr at +Ġjack ets +ĠHuff Post +am ong +Ġconf er +48 7 +ĠL illy +Ġadapt ing +ĠF ay +Ġsh oved +ve c +Ġref ine +Ġg on +Ġgun men +z ai +ĠShut tle +ĠI zan +Ġ19 13 +Ġple thora +· · +Ġ5 10 +Ġp uberty +Ġ24 1 +ĠWe alth +ĠAl ma +ĠM EM +ĠAd ults +C as +pr ison +R ace +Ġwater proof +Ġathlet icism +Ġcapital ize +ĠJu ice +Ġillum inated +ĠP ascal +Ġirrit ation +ĠWitness es +ad le +ĠAst ro +Ġf ax +ĠEl vis +Prim ary +ĠL ich +ĠEl ves +Ġres iding +Ġst umble +3 19 +ĠP KK +Ġadvers aries +D OS +ĠR itual +Ġsm ear +Ġar son +ident al +Ġsc ant +Ġmon archy +Ġhal ftime +Ġresid ue +Ġind ign +ĠSh aun +ĠEl m +aur i +A ff +W ATCH +ĠLy on +hel ps +36 1 +Ġlobby ist +Ġdimin ishing +Ġout breaks +Ġgo ats +f avorite +ĠN ah +son ian +ĠBo oster +Ġsand box +ĠF are +ĠMalt a +Ġatt Rot +ĠM OR +ld e +Ġnavig ating +T ouch +Ġunt rue +ĠDis aster +Ġl udicrous +Pass word +ĠJ FK +blog spot +4 16 +ĠUN DER +ern al +Ġdelay ing +T OP +Ġimpl ants +ĠAV G +ĠH uge +att r +Ġjournal istic +ĠPe yton +ĠI A +R ap +go al +ĠProgram me +Ġsm ashing +w ives +print ln +ĠPl ague +in us +EE P +Ġcru iser +ĠPar ish +umin ium +Ġoccup ants +ĠJ ihad +m op +Ġp int +Ġhe ct +ĠMe cca +direct or +ĠFund ing +ĠM ixed +Ġst ag +T ier +Ġg ust +Ġbright ly +ors i +Ġup hill +R D +Ġles ions +ĠBund y +liv ious +Ġbi ologist +ĠFac ulty +ĠAuthor ization +Ġ24 4 +All ow +ï ¸ +ĠGi ul +Ġpert inent +ot aur +es se +ĠRo of +Ġunman ned +35 1 +ĠSh ak +ĠO rient +Ġend anger +D ir +Ġrepl en +ed ient +Ġtail or +Ġgad gets +Ġaud ible +âĺ Ĩ +N ice +Ġbomb ard +ĠR ape +Ġdef iance +ĠTW O +ĠFilip ino +Ġunaff ected +erv atives +Ġso ared +ĠBol ton +Ġcomprom ising +ĠBrew ers +R AL +ĠA HL +icy cle +Ġv ampires +Ġdi pped +oy er +ĠX III +Ġsidew ays +ĠW aste +ĠD iss +ĠâĶľ âĶĢâĶĢ +$ . +Ġhabit ats +ĠBe ef +tr uth +tr ained +spl it +R us +And y +ĠB ram +RE P +p id +è£ ħ +ĠMut ant +An im +ĠMar ina +Ġfut ile +hig hest +f requency +Ġepile psy +Ġcop ing +Ġconc ise +Ġtr acing +ĠS UN +pan el +ĠSoph ie +ĠCrow ley +ĠAd olf +ĠShoot er +Ġsh aky +ĠI G +ĠL ies +ĠBar ber +p kg +Ġupt ake +Ġpred atory +UL TS +/ ** +Ġintox icated +ĠWest brook +od der +he ment +Ġbas eman +AP D +st orage +ĠFif ty +ed itor +G EN +UT ION +ir ting +Ġse wing +r ift +Ġag ony +ĠS ands +Ġ25 4 +C ash +Ġl odge +Ġp unt +N atural +ĠIde as +Ġerrone ous +ĠSens or +ĠHann ity +Ġ19 21 +Ġm ould +ĠG on +kay a +Ġanonym ously +ĠK EY +Ġsim ulator +W inter +Ġstream ed +50 7 +? ", +Ġte ased +Ġco efficient +Ġwart ime +ĠTH R +' '. +ĠBank ing +mp ire +Ġf andom +Ġl ia +G a +Ġdown hill +Ġinterpre ting +Ind ividual +N orm +Ġjealous y +bit coin +Ġple asures +ĠToy s +ĠChev rolet +ĠAd visor +IZ E +Ġrecept ions +70 6 +C ro +Ġ26 2 +Ġcit rus +ir u +Review er +ject ed +U ES +an z +19 81 +ĠWork er +Ġcompl ied +ores cent +contin ental +T on +ĠPr ism +ĠShe ep +Ġ28 8 +n ox +ĠV og +O rd +Ġreal ms +te k +Ġirrig ation +Ġbicy cles +Ġelectron ically +p oly +t all +() ); +Ġaest hetics +ĠInteg rated +Expl ore +Ġd unk +47 6 +p ain +ĠJac ques +ĠD mit +Fram es +Ġreun ited +Ġhum id +D ro +P olitical +Ġyouth ful +Ġent ails +Ġmosqu ito +36 3 +spe cies +Ġcoord inating +ĠMay hem +ĠMagn us +M ount +Impro ved +ĠST ATE +ATT LE +Ġflow ed +Ġtack led +Ġfashion ed +Ġre organ +iv ari +f inger +Ġreluct antly +et ting +ĠV and +you ng +ĠGar land +Ġpresum ption +Ġamen ities +ĠPle asant +on ential +ĠO xy +Ġmor als +ĠY ah +Read y +Sim on +En h +D emon +Ġcl ich +Mon itor +ĠD U +Ġwel comes +Ġstand out +Ġdread ful +Ġban anas +Ġball oons +h ooting +bas ic +Ġsuff ix +Ġd uly +can o +Ch ain +at os +Ġgeop olitical +Ġ( & +ĠGem ini +ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ ÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤÃĥÃĤ +Ġacqu itted +L uck +prot ect +10 24 +Ġsc arcity +Ġmind fulness +ec ided +D N +pr ime +ĠPres idents +ĠVID EO +Ġ( âĪĴ +add ock +N OR +ĠP ru +p un +ĠL OL +)) )) +ĠL iqu +ĠS AS +Ġsty ling +Ġpunish ments +Ġnum b +Ġasc ertain +ĠRock ies +f lu +Th umbnail +Ġperpet rated +ĠSem i +Ġdis arm +ĠOld er +ĠEx ception +Ġexponent ially +ĠCommun ities +Ġabol ish +ĠPart ner +pt oms +Ġ7 77 +ĠFo ley +ĠC ases +Ġgre ase +ĠReb irth +G round +Ġ; ) +ĠDoct rine +ik ini +Y e +ĠBl ossom +Ġpers ists +b ill +Ġinf usion +Ġbud dies +9 11 +ĠPat ient +Ġdem os +Ġacquaint ance +ĠP aw +at ari +Ġx ml +Ġfasc ination +ĠSer ve +Ï Ĥ +br anded +Ġa z +Return s +Ġover shadow +Ġro am +Ġspeed y +n umbered +hel ial +Ġdisc iple +Ġass urances +g iven +pect ing +ĠN atalie +çĶ ° +Ġmosquit oes +rote in +Ġnumer ic +Ġindepend ents +Ġtrans itional +Ġreaction ary +ĠMech dragon +do ctor +Ġshort est +Ġsequ ential +ĠB ac +ĠAccount s +ãģ Į +ach y +ract ive +ĠReg iment +Ġbreat htaking +ffic iency +ĠB ates +Ġ3 11 +Ġward robe +ft s +ĠBer k +Sim ply +ĠRivers ide +iver ing +ident ial +lu cent +Ġen riched +ĠCon ver +ĠG iving +ãĥ Ļ +Ġlegal ize +ĠF TC +Ġfre aking +M ix +Ġter restrial +es ian +ci ents +W ing +LO AD +Ġled ge +ĠViol ent +ĠMet all +Ġ30 8 +Ġs outheastern +hett o +M eat +Ġslow down +Ġret reated +Jere my +end as +**** * +er ic +Ġre ins +opp able +ĠHuman ity +ear ances +rig an +C amera +Ġwa ivers +s oc +Ġalter ation +trans form +ĠC emetery +50 6 +Ġindef inite +Ġstim ulating +y g +60 3 +ĠS op +Ġdescript ive +Ph ase +ĠEd mund +Ġpneum onia +vent us +A mb +Ġlabor atories +ĠEx clusive +ug ar +W ere +Ġmalf unction +Ġhomosexual s +Ġ---- --- +un i +Ġturb ines +ĠEqu ity +D u +Ġmind ed +ĠR H +ĠBlack hawks +Ġfe ats +Ġ17 00 +re pl +36 2 +lad en +Ġindisp ensable +ly ss +tt i +Ġre el +Ġdiver ted +Ġlik eness +Ġsubscript ions +Ġfing ert +Ġfil thy +dest ruct +d raft +ĠBernard ino +l aunch +Ġper plex +ĠS UM +car b +Ġswe ater +ĠVent ure +ĠJ ag +ĠCele b +ĠV oters +Ġstead fast +Ġathlet ics +ĠHans on +ĠDr ac +Tr acker +Ġcomm end +ĠPres idency +ĠD ID +in formed +Ġweb page +P retty +Ġforce fully +ãĥĥ ãĤ¯ +Ġrel ocation +Ġsat ire +â ī +ĠSunder land +æ Ħ +V oice +???? ???? +Ġinform ant +Ġbow el +ĠUn iform +Ġ ..." +Ġpur ge +Ġpic nic +ĠU mb +ĠU PDATE +ĠSapp hire +ĠSt all +le arn +Ġobject ively +Ġob liter +Ġlooph ole +Ġjour neys +Ġo mission +Pro s +ĠSid ney +pl oma +Ġspray ed +Ġg uru +Ġtra itor +Ġtim et +Ġsn apping +ĠSe vent +urn al +ĠUk ip +Ġb owed +por al +l iberal +R os +Quest ions +i OS +Ġsummar ize +ST AT +Ġ18 50 +ap est +Ġl ender +ĠVari able +br inging +ĠL ORD +, ) +Ġcollaps es +x iety +ĠN ed +Y D +ĠSch a +Ġantib ody +Ġdis band +y re +ill usion +Ġro ver +s hed +ĠHiro sh +cc i +Ġcal am +ĠMort on +P interest +Ġ19 28 +ĠE uras +ord es +Ġf ences +ĠIn ventory +ĠVal encia +ĠU d +ĠT iff +Ġsqu e +Ġqu otation +Ġtroubles ome +er ker +QU EST +ĠKing doms +s outh +Ġle vy +Pr ince +ĠSt ing +Ġnick named +Ġapp e +Ġphot ographic +Ġcorp us +re ference +ĠT rog +U nt +) =( +ĠLat via +Ġactiv ating +Ġlicense e +Ġdispar ities +ĠNews letter +ãĥĥ ãĥĪ +Ġfree ing +ĠJe ep +ĠPer ception +ins k +Ġsil icone +ĠHay den +Le an +ĠSuz uki +ibr arian +66 8 +Ġsp or +Ġcorrel ations +ag hetti +Ġtu ber +ĠIP CC +il us +ĠV u +Ġwealth iest +ĠCarb uncle +an za +Ġfool ed +ĠZ ur +Ġd addy +ran o +il ian +Ġknock out +f man +requ ired +ĠWik ileaks +ĠD uffy +ON T +Ġins ol +ĠObject s +Ġb ou +ĠNord ic +ĠIns ert +sc an +Ġd ancers +Ġid iots +major ity +ĠNev ille +ĠFree BSD +Ġt art +pan ic +69 0 +Ġcoc oa +Ġsam pled +Ġlook up +Ind ust +Ġinject ions +gen re +Ġa u +Ġroad way +Ġgen itals +K ind +ĠEx aminer +ĠY az +F resh +Ġpar alysis +ĠAl uminum +Ġre ap +ok é +Ġsl oppy +ĠTun nel +pos ium +ner y +en ic +Ġher bal +ĠOut er +ĠBuild er +Ġinc ur +Ġide ologies +Ġback ups +cons uming +ĠDet ect +de ck +ĠKN OW +ĠG ret +ĠM IC +Ġtough ness +ĠEx hibit +Ġh ive +L es +ĠSCH OOL +ĠAt ari +ald e +ĠN ull +and estine +m ouse +Ġbrig ade +48 9 +Ġrev ol +ĠLaw son +ĠW ah +op oly +eb ted +ĠS aunders +Ġ3 13 +ĠW inc +Ġtab oo +ĠHel met +Ġw edge +ch ip +ĠT ina +b g +Ġinf uri +r n +Ġanomal ies +ĠSy nc +ĠEx am +ĠComm it +ĠDi ary +ĠALS O +ĠDe bor +omed ical +Ġcomprehens ion +6 55 +Ġempower ing +Ġ ire +Ġju ices +ĠE TH +ĠBox ing +=" / +Ġfacilit ated +p oke +ĠPars ons +ĠMod er +tra vel +Ġcivil izations +Ġliber tarians +Ġrun e +ĠCl arks +at hed +Ġcampaign ers +ĠDis patch +ĠFah renheit +ĠCap com +-------- -- +Ġl ace +Ġdr aining +Ġl iner +ĠArt ificial +é n +t ask +] ). +ĠGM O +ĠOper ator +ord inary +ĠInf luence +ĠU ps +Ġpot ency +uss en +osp ons +ĠSw im +ĠDead line +Un ity +Ġcul inary +Ġenlight enment +Ġwe arer +Ġmin ed +Ġp ly +Ġinc est +ĠDVD s +W alk +B TC +Tr ade +Ġdev al +ib and +ĠOvers ight +Palest inian +Ġd art +Ġm ul +L R +Ġrem ovable +ĠReal ms +ì Ŀ +Ġmisc ar +ĠV ulkan +68 5 +è re +ĠS ap +Ġmer ging +ĠCar ly +che ster +Ġbr isk +Ġlux urious +ĠGener ator +Ġbit terness +Ġed ible +Ġ24 3 +T G +Ġrect angle +With No +bel ow +J enn +Ġdark est +Ġh itch +Ġdos age +Ġsc aven +ĠK eller +ĠIllust rated +Certain ly +ĠMaver icks +Marg inal +Ġdiarr hea +Ġenorm ously +Ġ9 99 +sh r +qu art +Ġadam ant +ĠM ew +Ġren ovation +Ġcerv ical +ĠPercent age +en ers +ĠKim ber +Ġflo ats +Ġde x +ĠW itcher +ĠSwan sea +d m +Ġsal ty +y ellow +Ġca pe +ĠDr ain +ĠPaul a +ĠTol edo +les i +Mag azine +ĠW ick +ĠM n +ĠA ck +ĠR iding +AS ON +Ġhom ophobic +AR P +Ġwand ered +C PU +ood oo +ĠP ipe +Ġtight ening +ĠBut t +3 18 +Ġdesert ed +S ession +Ġfacilit ating +J ump +Ġemer gencies +OW ER +Ġexhaust ive +ĠAF TER +Ġheart beat +ĠLab el +ack y +ĠCert ified +ilt ration +Z e +ĠU tt +Ġ13 00 +Ġpres ume +ĠDis p +Ġsur ged +Ġdoll s +Col umb +Ġchim pan +ĠR azor +Ġt icks +Ġcouncill or +Ġpilgr image +ĠReb els +ĠQ C +ĠA uction +x ia +ik k +b red +Ġinsert ion +Ġco arse +d B +SE E +ĠZ ap +ĠF oo +Ġcontem por +ĠQuarter ly +ot ions +ĠAl chemist +ĠT rey +ĠDu o +S weet +80 4 +ĠGi ov +Ġfun n +N in +h off +Ġram ifications +Ġ19 22 +ĠExper ts +az es +Ġgar ments +ar ial +ĠN ab +Ġ25 7 +ĠV ed +Ġhum orous +ĠPom pe +Ġn ylon +Ġlur king +ĠSerge y +ĠMatt is +Ġmisogyn y +ĠComp onents +ĠWatch ing +ĠF olk +ract ical +B ush +Ġt aped +Ġgroup ing +Ġbe ads +Ġ20 48 +Ġcon du +quer que +Read ing +Ġgriev ances +Ult ra +Ġend point +H ig +ĠSt atic +ĠScar borough +L ua +ĠMess i +a qu +ĠPsy Net +ĠR udd +Ġa venue +v p +J er +Ġsh ady +ĠRes ist +ĠArt emis +Ġcare less +Ġbro kers +Ġtemper ament +Ġ5 20 +T ags +ĠTurn ing +Ġut tered +Ġp edd +Ġimpro vised +Ġ: ( +Ġtab l +Ġpl ains +16 00 +press ure +ĠEss ence +marg in +friend s +ĠRest oration +Ġpoll ut +ĠPok er +ĠAugust ine +ĠC IS +ĠSE AL +or ama +Ġth wart +se ek +Ġp agan + º +cp u +Ġg arn +Ġass ortment +ĠI LCS +t ower +Recomm ended +Ġun born +ĠRandom Redditor +ĠRandomRedditor WithNo +Ġparaly zed +Ġeru ption +Ġinter sect +ĠSt oke +ĠS co +B ind +å ¾ +ĠP NG +ĠNeg ative +ĠNO AA +Le on +Ġall oy +ĠL ama +ĠD iversity +5 75 +Ġunderest imated +ĠSc or +Ġm ural +Ġb usted +so on +l if +Ġnone x +Ġall ergy +ĠUnder world +ĠR ays +ĠBl asio +Ġh rs +ĠD ir +Ġ3 27 +by ter +Ġrepl acements +Ġactiv ates +ri ved +M H +Ġp ans +ĠH I +Ġlong itudinal +Ġnu isance +al er +Ġsw ell +ĠS igned +s ci +ĠIs les +ĠA GA +Ġdef iant +Ġson ic +oc on +K C +ĠA im +t ie +ah ah +Ġm L +D X +Ġb isc +ĠBill board +ĠSY STEM +NE Y +ga ard +Ġdist ressed +former ly +Al an +Ġche fs +Ġopt ics +ĠC omet +ĠAM C +Ġredes igned +irm ation +Ġsight ings +38 2 +3 11 +ĠW B +Ġcont raction +ĠT OTAL +D ual +Ġstart led +Ġunderstand ably +Ġsung lasses +ETH OD +Ġd ocker +Ġsurf ing +ĠH EL +ĠSl ack +ton es +Ġsh alt +Vis ual +49 8 +Dep artment +c ussion +Ġunrest ricted +Ġt ad +Ġre name +employ ed +Ġeduc ating +Ġgrin ned +bed room +ĠActiv ities +ĠV elvet +ĠSW AT +Ġsh uffle +ig or +Ġsatur ation +F inding +c ream +ic ter +Ġv odka +tr acking +te c +Ġfore ground +iest a +Ġve hement +ĠEC B +ĠT ie +E y +Ġt urtles +ĠRail road +ĠKat z +ĠFram es +Ġmen ace +ĠFell owship +ĠEss ential +ugg ish +Ġdri p +ch witz +ĠKy oto +s b +ĠN ina +Param eter +Ġal arms +ĠCl aud +Ġpione ering +Ġchief ly +ĠSc ream +Col lection +Ġthank fully +ĠRonald o +åŃ IJ +st rip +ĠDisney land +com mercial +See ing +S oul +Ġevac uate +Ġc iv +ĠAs he +Ġdiv ides +ĠD agger +rehens ive +Ġber ries +ĠD F +Ġs ushi +Ġplur ality +W I +Ġdisadvant aged +Ġbatt alion +ob iles +45 1 +Ġcl ing +Ġunden iable +ĠL ounge +Ġha unt +p he +Ġquant ify +Ġdiff ered +Ġ[* ] +ĠV iz +c um +sl ave +Ġvide og +Ġqu ar +Ġbund les +ĠAl onso +t ackle +Ġneur onal +Ġlandsl ide +conf irmed +ĠDep th +Ġrenew ables +B ear +ĠMaced onia +Ġjer seys +Ġb unk +ĠSp awn +ĠControl s +ĠBuch anan +Ġrobot ics +Ġemphas izing +ĠTut orial +h yp +ist on +Ġmonument al +æ ° +ĠCar ry +Ġt bsp +en ance +H ill +art hed +Ġro tten +De an +Ġtw isting +Ġgood will +Ġimm ersion +L iving +Ġbr ushes +ĠC GI +ĠAt k +tr aditional +Ġph antom +ĠSt amina +Ġexpans ions +ĠMar in +Ġembark ed +ĠE g +int estinal +ĠPE OPLE +ĠBo oth +ĠApp alach +Ġreleg ated +V T +M IT +Ġmust er +Ġwithdraw ing +Ġmicrosc ope +ĠG athering +ĠC rescent +ĠArgent ine +ĠDec re +ĠDomin ic +Ġbud s +ant age +ĠI on +Ġwid ened +ONS ORED +ĠGl oves +iann opoulos +raz en +fe el +Ġrepay ment +Ġhind sight +ĠRE ALLY +ĠPist ol +ĠBra h +Ġwat ts +Ġsurv ives +Ġfl urry +iss y +Al ert +ĠUrug uay +Ph oenix +S low +ĠG rave +ĠF ir +Ġmanage able +Ġtar iff +ĠU DP +ĠPist ons +ĠNiger ian +Ġstrike outs +Ġcos metics +whel ming +f ab +c ape +pro xy +Ġre think +Ġover coming +sim ple +Ġw oo +Ġdistract ing +ĠSt anton +ĠTuls a +ĠD ock +65 9 +Ġdisc ord +ĠEm acs +ĠV es +ĠR OB +Ġreass uring +Ġcons ortium +Muslim s +3 21 +Ġprompt s +se i +ĠH itch +imp osed +ĠF ool +Ġindisc rim +wr ong +bu querque +D avis +! ] +Ġtim eless +ĠNE ED +Ġpestic ide +Ġrally ing +ĠCal der +Ġå ¤ +Ġx p +ĠUn le +ĠEx port +lu aj +B uff +) [ +Ġsq or +S audi +Ġis tg +Ġindul ge +pro c +Ġdisg usted +Ġcomp ounded +Ġn em +Ġschool ing +ĠC ure +process ing +S ol +Ġpro verb +it ized +ĠAlv arez +Ġscar f +Ġrect angular +re ve +Ġh ormonal +ĠSt ress +itiz en +Ġ4 25 +girl s +ĠNo ir +ĠR app +Ġmar ches +ch urch +ĠUs es +Ġ40 5 +ĠBer m +Ġord inances +ĠJud gment +Charg es +ĠZ in +Ġdust y +Ġstraw berries +Ġper ce +ĠTh ur +ĠDebor ah +net flix +ĠLam bert +Ġam used +ĠGu ang +Y OU +R GB +ĠC CTV +Ġf iat +r ang +Ġf ederation +ĠM ant +ĠB ust +ĠM are +respect ive +ĠM igration +ĠB IT +59 0 +Ġpatriot ism +Ġout lining +reg ion +ĠJos é +Ġbl asting +ĠEz ra +B s +Ġundermin es +ĠSm ooth +Ġcl ashed +rad io +Ġtransition ing +ĠBucc aneers +ĠOw l +Ġplug s +Ġh iatus +ĠPin ball +Ġm ig +ĠNut r +ĠWolf e +Ġinteg ers +Ġor bits +ĠEd win +ĠDirect X +b ite +Ġbl azing +v r +Ed ge +ĠP ID +ex it +ĠCom ed +ĠPath finder +ĠGu id +ĠSign s +ĠZ er +ĠAg enda +Ġreimburse ment +M esh +i Phone +ĠMar cos +ĠS ites +h ate +en burg +Ġs ockets +p end +Bat man +v ir +ĠSH OW +Ġprovision al +con n +ĠDeath s +AT IVE +Pro file +sy m +J A +Ġnin ja +inst alled +id ates +eb ra +ĠOm aha +Ġse izing +ĠBe asts +Ġsal ts +M ission +Gener ally +ĠTr ilogy +he on +leg ates +Ġd ime +Ġf aire +par able +G raph +Ġtotal ing +Ġdiagram s +ĠYan uk +ple t +ĠMe h +Ġmyth ical +ĠStep hens +aut ical +ochem istry +Ġkil ograms +Ġel bows +anc ock +ĠB CE +ĠPr ague +Ġimpro v +ĠDev in +Ġ" \ +par alle +Ġsuprem acists +ĠB illion +Ġreg imen +inn acle +Ġrequ isite +ang an +ĠBur lington +ain ment +ĠObject ive +oms ky +G V +Ġun ilateral +Ġt c +Ġh ires +ment al +Ġinvol untary +Ġtrans pl +ĠASC II + ¨ +Ev ents +Ġdoub ted +ĠKa plan +ĠCour age +ig on +ĠMan aging +ĠT art +Ġfalse hood +ĠV iolet +Ġair s +Ġfertil izer +Brit ain +Ġaqu atic +ou f +W ords +ĠHart ford +Ġeven ings +ĠV engeance +qu ite +G all +ĠP ret +Ġp df +ĠL M +ĠSo chi +ĠInter cept +9 20 +Ġprofit ability +ĠId le +ĠMac Donald +ĠEst ablishment +um sy +Ġgather ings +ĠN aj +Charl ie +Ġas cent +ĠProt ector +Ġal gebra +Ġbi os +for ums +EL S +Introdu ced +Ġ3 35 +Ġastron omy +Cont ribut +ĠPol ic +Pl atform +Ġcontain ment +w rap +Ġcoron ary +ĠJ elly +man ager +Ġheart breaking +c air +ĠChe ro +c gi +Med ical +ĠAccount ability +! !" +oph ile +Ġpsych otic +ĠRest rict +Ġequ itable +iss ues +Ġ19 05 +ĠN ek +c ised +ĠTr acking +Ġo zone +Ġcook er +ros is +Ġre open +Ġinf inity +ĠPharm aceutical +ens ional +Att empt +ĠR ory +Mar co +Ġawa its +H OW +t reated +Ġbol st +Ġreve red +Ġp ods +opp ers +00 10 +Ġampl itude +ric an +SP ONSORED +Ġtrou sers +Ġhal ves +ĠK aine +ĠCut ler +ĠA UTH +Ġsplend id +Ġprevent ive +ĠDud ley +if acts +umin ati +ĠY in +Ġad mon +ĠV ag +Ġin verted +Ġhast ily +ĠH ague +L yn +Ġled ger +Ġastron omical +get ting +Ġcirc a +ĠC ic +ĠTenn is +Lim ited +Ġd ru +ĠBY U +Ġtrave llers +Ġp ane +ĠInt ro +Ġpatient ly +Ġa iding +Ġlo os +ĠT ough +Ġ29 3 +Ġconsum es +Source File +Ġ"" " +Ġbond ing +Ġtil ted +Ġmenstru al +ĠCel estial +UL AR +Plug in +Ġrisk ing +N az +ĠRiy adh +Ġacc redited +Ġsk irm +é Ľ +Ġexam iner +Ġmess ing +Ġnear ing +ĠC hern +ĠBeck ham +Ġsw apped +Ġgo ose +K ay +Ġlo fty +ĠWal let +Ġ[ ' +Ġap ocalypse +Ġb amboo +ĠSP ACE +ĠEl ena +Ġ30 6 +ac ons +Ġtight ened +Ġadolesc ence +Ġrain y +Ġvandal ism +ĠNew town +Ġcon ject +c akes +Ġche ated +Ġmoder ators +par ams +E FF +Ġdece it +ĠST L +ĠTanz ania +ĠR I +Ġ19 23 +ĠEx ile +the l +Ġthe olog +Ġquir ky +ĠIr vine +Ġneed y +or is +U m +K a +Ġmail box +3 22 +Ġb os +ĠPet ra +K ING +Ġenlarg ed +O ften +Ġbad ass +Ġ3 43 +ĠPl aces +ĠC AD +Ġpr istine +Ġinterven ing +d irection +Ġl az +ĠD SM +Ġproject ing +ĠF unk +ag og +pay ment +n ov +Ġch atter +AR B +Ġexam inations +ĠHouse hold +ĠG us +F ord +4 14 +B oss +Ġmy stic +Ġle aps +ĠB av +ul z +b udget +Foot ball +Ġsubsid ized +Ġfirst hand +Ġcoinc ide +oc ular +Con n +ĠColl abor +Ġfool s +am ura +ah ar +r ists +Ġsw ollen +Ġexp ended +ĠP au +s up +Ġsp ar +Ġkey note +s uff +Ġunequ al +Ġprogress ing +str ings +ĠGamer gate +Dis ney +ĠEle ven +om nia +Ġscript ed +Ġear ners +bro ther +ĠEn abled +æ ³ +Ġlar vae +ĠL OC +m ess +Wil son +ĠTem plate +success fully +Ġparam ount +Ġcamoufl age +Ġbind s +ĠQu iet +ĠSh utterstock +r ush +Ġmasc ot +fort une +ĠCol t +ĠBe yon +hab i +Ġha irc +Ġ26 7 +ĠDe us +Ġtw itch +Ġconcent rating +Ġn ipples +c ible +Ġg ir +N Z +M ath +n ih +Requ ired +Ġp onder +ĠS AN +Ġwedd ings +Ġl oneliness +N ES +ĠMah jong +69 5 +add le +ĠGar ner +ĠC OUR +Br idge +Ġsp ree +ĠCald well +Ġbri bery +Ġ���� ���� +plug ins +Ġr acket +Ġchamp agne +vers ible +V ote +Ġmod ifiers +May or +6 80 +Ġassemb lies +ĠS ultan +ĠN ing +ĠLad ies +Ġsulf ur +Ġor bs +Ġ---- - +____ ___ +ĠJournal ism +Ġes ports +Ġl ush +Ġh ue +Ġspect ral +H onest +ãĥ ı +Ġbus hes +Ġrein forcement +Ġre opened +ĠWhe els +ĠM org +rie ving +Ġaux iliary +Ġj Query +ĠB AT +tes que +Ġver tex +p ure +f rey +ãĤ º +d os +Ġty ph +Ġc ull +Ġe q +Ġdec on +Ġtoss ing +Ġdispar ate +ĠBr igham +print f +led ged +Ġsu nd +Ġco zy +Ġhepat itis +per forming +Ġav al +ĠG G +f uture +Ġpet ertodd +ĠKos ovo +Ġmagn ets +Al ready +ĠEd ison +ĠCe res +ĠRA ID +Ġbrill iance +57 6 +Ġder ives +Ġhypert ension +ĠÎ Ķ +Ġlamb da +Ġfl air +Ġmission aries +Ġrap es +ĠSt arter +ĠMon ths +Ġdef y +Ġseism ic +ĠR aphael +Ġeuro zone +65 6 +z sche +Ġscr atched +Ġb ows +ĠLenn on +ĠGa ia +Ġdri pping +f acts +A le +Ġfrog s +ĠBre ast +ogene ity +ĠProsecut or +Ġampl ified +ĠHod g +ĠF n +Th ousands +ĠNI H +ĠMonitor ing +FT WARE +ĠPri ebus +ĠG rowing +hun ter +Ġdiagn ose +ĠM ald +ĠL R +Ġcrown ed +Ġburst ing +Ġdiss olution +j avascript +Ġuseful ness +ĠExec ution +: ( +ĠIv ory +a ah +Ġpersecut ed +viol ence +ist as +ĠCr ate +Ġimpuls es +ĠSp ani +ed es +Hand le +ĠZ erg +think able +Last ly +Ġspont aneously +Ġinconven ient +Ġdismiss ing +Ġpl otted +Ġeight y +Ġ7 37 +r ish +ĠThor nton +ath am +Ġsit com +V en +Rec ipe +t el +l und +Ġcle ars +ĠSas uke +Ġ25 8 +Ġopt ing +Ġen raged +est hetic +ĠA e +uch s +Pre p +Fl ow +Ġrun off +ĠE ating +ĠG iles +ĠAct ing +res ources +ib aba +Ġr pm +Ġske wed +ĠBl anc +ĠS akuya +Ġhot ter +Ġ19 24 +op ian +ck o +Ġcr umbling +Ġcapt ains +ĠAppropri ations +le aders +dro pping +an uts +Ġrevers ing +ĠP ose +ĠS ek +Sc ot +ĠIde a +c ise +ĠSloven ia +Ġ3 17 +Do ctor +Ġcro cod +ald i +Se a +ĠFar rell +Ġmerc enaries +ĠR NC +ĠGu ess +Ġp acing +M achine +Streamer Bot +ĠChar ity +Ġ29 8 +Ġcann ons +ĠTob y +TPP StreamerBot +ĠPass ion +cf g +Th om +Ġbad ges +ĠBern stein +. âĢĵ +ĠP OP +ĠCon j +Ġinitial ization +Ġbiod iversity +D ub +Ġfeud al +Ġdisclaim er +Ġc row +Ġign ition +ar f +S HA +Ġk Hz +h azard +ĠArt ists +oe uv +67 9 +ĠRud y +N ine +ĠRam adan +å ½ +itt o +Ġadren aline +C ert +Ġsmell ed +Ġimp unity +Ġag endas +ĠRe born +ĠCon cent +ĠSe ems +Ġo mega +ĠDust in +Ġback er +ĠSau ce +ĠBoy le +W IN +Ġsp ins +Ġpa uses +u pt +Ġshred ded +Ġstra pped +ĠCor ruption +Ġscr atches +Ġn i +Ġatt ire +ĠS AF +Factory Reloaded +ĠI PS +Ġ( % +Ġsem inar +f ocus +c ivil +Ġ18 60 +int osh +Ġcontin ual +Ġabbre vi +ĠS ok +oc obo +X M +Ġfr antic +Ġunavoid able +Ġar tery +Ġannot ations +b ath +Cl imate +Ġd ors +ĠSl ide +co ord +ĠRel oad +ĠL DL +ĠLove craft +Ġunim agin +Ġresemb led +Ġbarr acks +n p +Ġsurrog ate +Ġcategor ized +ãĤ © +Ġvacc inated +Ġdrain age +Ġind ist +ĠWhats App +Ġ18 70 +oler ance +inv oke +am orph +Ġrecon nect +Ġem anc +Ġblind ness +Ġ12 80 +intern et +c ollar +Ġalt ru +Ġab yss +ĠT RI +65 7 +Ġinf used +HE AD +Ġforest ry +ĠWood y +ĠC i +w i +s am +78 4 +hol iday +Ġmog ul +ĠF ees +ĠD EN +In ternal +ur bed +f usc +at om +ĠIll usion +Ġpoll ed +Ġfl ap +Ġco ax +L GBT +An aly +ĠSect ions +ĠCalif orn +em n +Ġh ither +ĠN IGHT +Ġn ailed +ĠPip eline +39 1 +o of +ĠPr imal +vere nd +Ġsl ashing +Ġret ri +avi our +Ġdepart ing +g il +IS C +Ġmid way +Ġultras ound +Ġbeh aving +ĠT ara +class es +V irtual +ĠColon ial +Ġstri pping +Ġorchestr ated +ĠGra ves +45 2 +ĠIron ically +ĠWrit ers +Ġl ends +ĠMan z +Ġra ven +Ġoxid ative +Ġ26 6 +EL F +act ually +asc ar +D raft +Ġfavour able +Ġhumili ating +Ġf idelity +ĠH of +ĠX uan +49 6 +Ġlay ered +at is +79 0 +Ġpay check +it on +K ar +ĠVM ware +ĠFar mer +Ġserv ic +gl omer +Ġsl ump +ĠFab ric +ĠD OC +est ing +Ġreass ure +Ġph yl +v olt +it ory +R ules +Ġoxid ation +Ġpri zed +Ġmist ress +ĠDj ango +WAR N +å ij +Ġenc ode +ĠFeed back +Ġstupid ity +I an +ĠYugoslav ia +× ¨ +ac l +UT E +19 77 +Ġqual ifies +Ġpuls es +pret ty +Ġfro ze +Ġs s +Iter ator +Ġur gently +Ġm ailed +ĠCh am +Ġsust aining +Ġbas il +Ġpupp ies +il ant +ĠP LEASE +l ap +ace ous +F ear +ĠMaster y +aut omatic +ĠT AG +Ġant im +ag les +47 3 +fram es +Ġwh ispers +ĠWho ever +Ġbra very +ĠUK IP +ract ions +"" " +Ġt ame +Ġpart ed +every thing +CON T +Ġind ebted +Ġadd r +re k +IR ED +Ġem inent +cl inton +Ġo usted +Ġreview er +Ġmelt down +Ġre arr +ĠY ao +the real +aby te +Ġst umbling +Ġbat ches +Ġ25 9 +Ġcontrace ptive +Ġprost itute +ens is +De cl +ĠSt rikes +M ilitary +ĠO ath +v acc +pp ings +05 2 +Ġpart Name +amp ing +Rep orts +K I +CH R +Ġsubt ly +sw ers +Bl ake +us ual +Ġcontest ants +Ġcart ridges +ĠGRE AT +Ġbl ush +ĠâĢ º +47 2 +Ġreason ed +ãĥ ¤ +paralle led +Ġd yn +ag ate +Ġnight ly +å Ĩ +55 6 +Ġsem antic +ĠAdv oc +Ġ !! +Ġdisag rees +ĠB W +V eh +Ġharm ing +Ġembr aces +Ġstri ves +Ġin land +ĠK ard +Ġhe ats +ĠGin ny +ut an +ern aut +yl ene +ĠE lev +J D +Ġh ars +ĠStar r +Ġsk ysc +Ġcollabor ators +Us ually +Ġrev olutions +ĠSTAT S +Ġdism antle +Ġconfident ly +Ġkin etic +Al i +Ġpercent ile +Ġextract ing +ill ian +est ead +Ġphysic ists +ĠMarsh al +Ġfell owship +Ġd ashed +ĠU R +ĠSi oux +ĠComp act +am ide +P ython +ĠLe igh +ĠPharm ac +ist rates +her ical +Ġf ue +ĠE min +Ġ( { +ĠNeighbor hood +Ġdisrupt ing +ĠD up +Ġg land +ĠSe v +ĠMar ian +arg on +ĠD und +Ġ< !-- +Ġstr and +Ġstadium s +z os +Ġpsych osis +ĠR ack +Ġbrilliant ly +ï¸ ı +Ġsubmer ged +ĠInst it +ĠCh ow +Ġc ages +ĠH ats +ĠU rs +Ġdil uted +us at +ien ne +ĠMembers hip +ĠBur k +Ġ ie +Ġarche type +D rug +ult on +ĠSp ock +ĠMcK ay +ĠDep end +F eatured +S oc +19 78 +ĠB ere +Ġrelent lessly +Ġcripp ling +Ġar thritis +çĶ Ł +ĠTrop ical +ĠBul g +ĠCher yl +Ġadm irable +Ġsub title +Over ride +Ġorig inating +ĠC CP +Ġsw ore +ĠSo le +ĠDis orders +3 29 +Ġprocess ion +Ġref urb +Ġimm ersed +requ ently +Ġskept ics +Ġcer amic +m itter +en stein +b elt +ĠT IT +b idden +Ġf ir +m ist +> ] +Ġwe ave +ĠParad ox +Ġentr usted +ĠBarcl ays +Ġnovel ist +og ie +80 6 +Ġnin ety +Ġdisag reements +@@@@ @@@@ +ĠAus chwitz +c ars +ĠL ET +t ub +arant ine +P OS +Ġback story +Ġcheer ful +ĠR ag +ek a +bi ased +Ġinexper ienced +ak ra +ĠW itt +t an +Ġrap ist +Ġplate au +ch al +ĠInqu is +exp ression +Ġc ipher +Ġsh aving +add en +re ly +( \ +ism a +ĠReg ulatory +CH AR +ily n +N VIDIA +G U +Ġmur m +la us +Christ opher +Ġcontract ual +ĠPro xy +ĠJa ime +ĠMethod ist +Ġstew ards +st a +per ia +Ġphys iology +Ġbump ed +Ġf ructose +Austral ian +ĠMet allic +ĠMas querade +ar b +Ġprom ul +Ġdown fall +Ġbut cher +Ġb our +ĠIN FORMATION +ĠB is +pect s +ad ena +Ġcontempl ating +ar oo +cent ered +ĠPe aks +Us ed +Ġmod em +Ġg enders +Ġ8 000 +37 1 +Ġm aternity +ĠR az +Ġrock ing +Ġhandgun s +ĠD ACA +Aut om +ĠN ile +Ġtum ult +ĠBenef it +ĠAppro ach +works hop +ĠLe aving +G er +inst ead +Ġvibr ations +Ġrep ositories +49 7 +ĠA unt +ĠJ ub +ĠExp edition +Al pha +Ġs ans +Ġoverd ue +Ġoverc rowd +Ġlegisl atures +Ġp aternal +ĠLeon ardo +Ġexp ressive +Ġdistract ions +Ġsil enced +tr ust +Ġb iking +Ġ5 60 +Ġpropri et +Ġimp osition +Ġcon glomer +Ġ= ================================================================ +ĠTe aching +ĠY ose +int ensive +T own +Ġtroll ing +ĠGr ac +ĠAS US +Y o +Ġspecial s +ĠNep h +ĠGod zilla +Dat abase +ĠHe gel +Ġ27 2 +19 76 +ĠGl oria +Ġdis emb +ĠInvestig ations +ĠB ane +ag ements +St range +Ġtre asury +ĠPl ays +Ġundes irable +Ġwid ening +Ġverb ally +Ġinf ancy +Ġcut ter +f ml +Ġ21 00 +prot otype +f ine +Ġdec riminal +Ġdysfunction al +Ġbes ie +ĠErn st +z eb +Ġnort heastern +Ġa ust +por ate +ĠMar lins +Ġsegreg ated +ew orld +ĠMa her +Ġtra verse +Ġmon astery +ur gy +G ear +s and +Com pl +ĠE MP +Ġpl ent +ĠMer cer +Ġ27 6 +TA BLE +Config uration +H undreds +Ġpr ic +Ġcollabor ating +ĠPar amount +ĠCumm ings +Ġ( < +Ġrecord er +Ġfl ats +Ġ4 16 +wh ose +Font Size +ĠOr bit +Y R +Ġwr ists +Ġb akery +) } +ĠB ounty +ĠLanc aster +Ġend ings +acc ording +ĠSal am +e asy +75 5 +ĠBur r +ĠBarn ett +onom ous +Un ion +Ġpreced ence +ĠScholars hip +ĠU X +Ġroll out +Ġbo on +al m +ĠCan ter +æ µ +Ġround ing +Ġcl ad +Ġv ap +ĠF eatured +is ations +Ġ5 40 +pol ice +Ġunsett ling +Ġdr ifting +ĠLum ia +ĠObama Care +ĠF avor +Hy per +ĠRoth schild +ĠMil iband +an aly +ĠJul iet +H u +Ġrec alling +a head +69 6 +Ġunf avorable +Ġd ances +O x +Ġleg ality +Ġ40 3 +rom ancer +Ġinqu ire +ĠM oves +\ "> +ĠVari ant +ĠMess iah +ĠL CS +ĠBah á +75 6 +Ġeyeb row +Ġ ¥ +ĠMc F +ĠFort y +M as +Ġpan icked +Ġtransform ations +q q +Ġrev olves +ring e +ĠA i +ax e +Ġon ward +ĠC FR +ĠB are +log in +Ġliqu ids +Ġde comp +second ary +il an +ĠCon vert +ami ya +Ġprosecut ing +Ġâī ¡ +ĠYork ers +ĠByr ne +sl ow +aw ei +J ean +Ġ26 9 +ĠSky dragon +Ġ é +ĠNicarag ua +ĠHuck abee +ĠHigh ly +Ġamph ib +ĠPast or +ĠL ets +Ġbl urred +Ġvisc eral +ĠC BO +Ġcollabor ated +z ig +Leg al +Ġapart heid +Ġbr id +Ġpres et +ĠD ET +ĠAM A +× Ķ +arch ing +auc uses +build er +Ġpo etic +Ġem ulator +ĠMole cular +Ġhon oring +ise um +Ġtract or +ĠCl uster +ĠCal m +ared evil +Ġsidew alks +Ġviol in +Ġgeneral ized +ĠAle c +Ġemb argo +Ġfast ball +ĠHT TPS +ĠL ack +ĠCh ill +ri ver +C hel +ĠSw arm +ĠLev ine +ro ying +L aunch +Ġkick er +Ġadd itive +ĠDe als +W idget +cont aining +Ġescal ate +ĠOP EN +Ġtwe aked +Ġst ash +Ġsp arks +ĠEs sex +ĠE cc +Ġconv ict +Ġblog ging +I ER +ĠH L +Ġmurd erers +75 9 +ĠH ib +Ġde pl +ĠJ ord +S ac +Ġdis sect +ĠHow e +os her +Ġcustom izable +ĠFran z +Ġat ro +Ä ĩ +Ġ000 4 +Ġout post +R oss +Ġglyph osate +ĠHast ings +ĠBE FORE +Ġsh ove +o pped +ĠSc ala +Ġam ulet +an ian +Ġexacerb ated +Ġe ater +47 1 +UM E +Ġpul p +izont al +ĠZ am +ĠAT I +imm une +aby tes +Ġunnecess arily +ĠC AT +ĠAx is +Ġvisual ize +à ī +ĠRad ical +f m +Doc uments +ĠFor rest +Ġcontext ual +ĠSy mbol +Ġtent ative +ĠDO ES +ĠGood s +Ġintermitt ent +} : +medi ated +Ġridic ule +Ġathe ism +Ġpath ogens +ĠM um +Ġre introdu +Ġ30 7 +i HUD +Ġflash light +Ġsw earing +Ġp engu +B u +Ġrot ated +ĠCr ane +Ġ() ); +Ġfashion able +Ġendors ing +46 3 +) [ +Ġingest ion +Ġcook s +Ġ9 50 +ot omy +ĠIm am +Ġk a +Ġte aser +ĠGhost s +ĠãĤ µ +19 69 +Ï ĥ +ub by +Ġconver ter +zan ne +end e +ĠPre par +ĠNic kel +ĠChim era +h im +ĠTyr ann +ĠSabb ath +ĠNich ols +Ġra pt +ih ar +Ġshe lling +Ġillum inate +Ġdent ist +ut or +ĠInteg ration +Ġwh ims +ĠLiter ary +Be aut +Ġp archment +ag ara +Br and +Ġder og +âĢ¦ ) +ĠNor se +Ġunw itting +Ġc uc +Ġborder line +Ġupset ting +Ġrec ourse +Ġd raped +ĠRad ar +Ġcold er +ĠPep si +im inary +], [ +65 8 +V i +ĠF rem +ĠP es +Ġveter inary +ĠT ED +ĠEp idem +n ova +k id +Ġdev out +o ct +j ad +M oh +ĠP AY +Ġge ometric +Ġ3 23 +Ġcircum ference +ich ick +19 75 +ĠY uri +ĠSh all +ĠH over +un in +S pr +Ġg raft +ĠHapp iness +Ġdisadvant ages +att acks +Ġhub s +ĠStar Craft +é ĸ +Ġgall eries +ĠKor ra +Ġgrocer ies +ĠGors uch +Ġrap ists +Ġfun gi +ĠTyph oon +V ector +ĠEm press +b attle +4 68 +Ġparas ite +ĠBom ber +S G +ex ist +ĠP f +Ġun se +Ġsurge ons +B irth +ĠUn sure +ĠPrint ed +ĠBehavior al +ĠA ster +Pak istan +Ġun ethical +Ġs v +ĠIo T +Ġlay outs +P ain +Ġconst ants +ĠL W +ĠB ake +Ġtow els +Ġdeterior ation +ĠBol ivia +Ġblind ed +ĠW arden +ĠMist ress +Ġon stage +Ġcl ans +ĠB EST +19 60 +Ġant ique +Ġrhet orical +ĠPer cy +ĠRw anda +, . +B ruce +Ġtra umat +ĠParliament ary +Ġfoot note +id ia +ĠLear ned +se eking +gen ic +Ġdim ensional +H ide +èĢ ħ +Ġintrig ue +in se +Ġle ases +Ġapp rentices +w ashing +Ġ19 26 +V ILLE +Ġsw oop +s cl +Ġbed rooms +on ics +ĠCr unch +comp atible +Ġincap ac +ĠYemen i +ash tra +z hou +d anger +Ġmanifest ations +ĠDem ons +AA F +Secret ary +ACT ED +L OD +Ġam y +ra per +eth nic +4 17 +Ġpos itives +Ġ27 3 +ĠRefuge es +Ġus b +ĠV ald +odd y +ĠMahm oud +As ia +Ġskull s +ĠEx odus +ĠComp et +ĠL IC +ĠM ansion +ĠA me +Ġconsolid ate +storm s +ont ent +99 6 +Ġcl en +Ġm ummy +fl at +75 8 +ĠV OL +oter ic +n en +ĠMin ute +S ov +Ġfin er +R h +ly cer +Ġreinforce ments +ĠJohann es +ĠGall agher +Ġgym n +S uddenly +Ġext ortion +k r +i ator +T a +Ġhippocamp us +N PR +ĠComput ing +Ġsquare ly +Ġmod elling +ĠFor ums +ĠL isp +ĠKrish na +Ġ3 24 +Ġr ushes +Ġens ued +Ġcre eping +on te +n ai +il ater +ĠHorn ets +Ġob livious +IN ST +55 9 +Ġjeopard y +Ġdistingu ishing +j ured +Ġbeg s +sim ilar +ph ot +5 30 +ĠPark way +Ġs inks +ĠHearth stone +ib ur +ĠBat on +Av oid +Ġd ancer +Ġmag istrate +ary n +Ġdisturb ances +ĠRom ero +Ġpar aph +Ġmis chief +âĸ ĵ +ĠSh aria +Ġur inary +r oute +iv as +f itted +Ġeject ed +ĠAl buquerque +Ġ4 70 +Ġirrit ated +ĠZ ip +ĠB iol +à į +Ġden ounce +Ġbin aries +ĠVer se +Ġopp os +ĠKend rick +ĠG PL +Ġsp ew +ĠEl ijah +ĠE as +Ġdr ifted +so far +Ġannoy ance +ĠB ET +47 4 +ĠSt rongh +it ates +ĠCogn itive +oph one +ĠIdent ification +ocr ine +connect ion +Ġbox er +ĠAS D +ĠAre as +Y ang +t ch +ull ah +Ġdece ive +Comb at +ep isode +cre te +W itness +Ġcondol ences +ht ar +Ġhe als +Ġbuck ets +ĠLA W +B lu +Ġsl ab +ĠOR DER +oc l +att on +ĠSteven son +ĠG inger +ĠFriend ly +ĠVander bilt +sp irit +ig l +ĠReg arding +ĠPR OG +Ġse aling +start ing +Ġcard inal +ĠV ec +ĠBe ir +Ġmillisec onds +we ak +per se +Ġster ile +ĠCont emporary +ĠPh ant +ĠCl o +Ġout p +Ġex iled +Ġ27 7 +Ġself ie +Ġman ic +Ġn ano +ter ms +Alex ander +Ġres olves +Ġmillenn ia +Ġexpl odes +Ġconst ellation +Ġadul tery +m otion +D OC +Ġbroad casters +Ġkinderg arten +ĠMay weather +ĠE co +ich o +Ġ28 7 +l aun +Ġm ute +Ġdisc reet +Ġpres chool +Ġpre empt +De lete +ĠFre ed +P i +H K +Ġblock er +ĠC umber +Ġw rought +d ating +Ġins urer +Ġquot as +Ġpre ached +Ġev iction +ĠReg ina +ĠP ens +Ġsevent een +ĠN ass +D ick +Ġfold s +Ġd otted +ĠA ad +Un iversal +Ġp izz +ĠG uru +Ġso ils +Ġno vice +ĠNe ander +Ġst ool +Ġdeton ated +ĠPik achu +ĠMass ive +IV ER +ĠAb del +Ġsubdu ed +Ġtall est +Ġprec arious +Ġa y +r ification +ĠOb j +c ale +Ġun question +cul osis +ad as +igr ated +D ays +Ġque ens +ĠGaz ette +ĠCol our +ĠBow man +ĠJ J +ï ve +Ġdomin ates +Stud ent +Ġm u +Ġback log +ĠElect ro +Tr uth +48 3 +Ġcond ensed +r ules +ĠCons piracy +Ġacron ym +hand led +ĠMat te +j ri +ĠImp ossible +l ude +cre ation +Ġwar med +ĠSl ave +Ġmis led +Ġfer ment +ĠK ah +ink i +ke leton +cy l +ĠKar in +Hun ter +Reg ister +ĠSur rey +Ġst ares +ĠW idth +ĠN ay +ĠSk i +Ġblack list +uck et +Ġexp ulsion +im et +Ġret weet +vant age +Fe ature +Ġtro opers +Ġhom ers +9 69 +Ġconting ency +ĠW TC +ĠBrew er +fore ign +W are +S olar +Ġund ue +RE C +ulner able +path ic +ĠBo ise +Ġ3 22 +Ġarous ed +ĠY ing +ä¸ į +uel ess +Ġp as +Ġmor p +Ġfl oral +Ex press +ud ging +k B +ĠGr anted +Ø ¯ +ĠMich a +ĠGoth ic +ĠSPEC IAL +ĠRic ardo +F ran +Ġadminister ing +6 20 +por a +Ġ ® +Ġcomprom ises +Ġb itten +Ac cept +Th irty +Ð ² +Ġmater ially +ĠTer r +ig matic +ch ains +Ġdo ve +stad t +Mar vel +FA ULT +Ġwind shield +Ġ3 36 +ad ier +Ġsw apping +Ġflaw less +ĠPred ator +ĠMiche le +Ġprop ulsion +ĠPsych ic +Ġassign ing +Ġfabric ation +Ġbar ley +l ust +Ġtow ering +Ġalter cation +ĠBent ley +Sp here +Ġtun a +ĠClass es +Fre edom +un er +L ady +v oice +Ġcool est +or r +Ġpal p +$ { +Ġhyster ia +ĠMet atron +p ants +Ġspawn ing +Exper ts +ĠInvest ors +ĠAn archy +Ġshr unk +ĠVict im +Ġ28 9 +Ġec stasy +ĠB inding +58 5 +ĠMel ody +57 8 +ot ally +ĠE tsy +lig a +Ġapplaud ed +Ġswe ating +Ġredist ributed +Ġpop corn +Ġsem inal +f ur +ĠNeuro science +R and +ĠO st +ĠMadd en +ĠIncre asing +ĠDaw kins +ĠSub way +Ġar sen +cons erv +B UR +Ġsp iked +ĠLy ft +ĠImper ium +ĠDrop box +Ġfav oured +Ġencomp asses +gh ost +Ġins pires +Ġbur geoning +ĠY oshi +ĠVert ical +ĠAud itor +Ġint ending +Ġfilib uster +Bl oom +f ac +ĠCav s +ign ing +Ġcowork ers +ĠBarb arian +rem ember +FL AG +Ġaudit ory +ason ry +Col lege +Ġmut ed +gem ony +ob in +ĠPsych o +9 68 +Ġlav ish +Ġhierarch ical +ĠDr one +ou k +Ġcripp led +ĠMax im +Sl ot +Ġqu iz +ĠV id +if ling +Ġarchae ologists +Ġabandon ment +d ial +le on +ĠF as +T ed +Ġr aspberry +Ġmaneu vers +Ġbehavi ours +Ġins ure +Ġrem od +Sw itch +h oe +Ġsp aced +Ġafford ability +ĠF ern +not ation +ĠBal anced +Ġoccup ies +en vironment +Ġneck lace +Ġsed an +F U +ĠBrav o +Ġab users +ĠAn ita +met adata +ĠG ithub +ait o +ĠF aster +ĠWass erman +ĠF lesh +Ġth orn +r arily +ĠMer ry +w ine +Ġpopul ace +ĠL ann +Ġrepair ing +Ġpsy che +Ġmod ulation +aw aru +âĢĭ âĢĭ +ari j +Ġdecor ations +Ġapolog ise +ĠG arg +app ly +Ġgive away +ĠFl an +ĠWy att +U ber +Ġauthor ised +ĠMor al +HAHA HAHA +activ ate +Ġtorped o +ĠF AR +Ġam assed +ĠA ram +ark in +ĠVict ims +st ab +Ġo m +ĠE CO +Ġopio ids +Ġpurpose ly +ĠV est +Ġer g +at an +ĠSur gery +Ġcorrect ing +ĠOrt iz +ĠBe et +Ġrev oke +Ġfre eway +ĠH iggins +F ail +ĠFar ms +ĠAT P +h ound +Ġp oking +ĠCommun ists +mon ster +iment ary +Ġunlock ing +Ġunf it +we ed +en ario +at ical +ĠEnlight enment +ĠN G +ĠComp ensation +de en +ĠWid ow +ĠCind y +ĠAfter wards +Ġ6 000 +ikh ail +ag ically +Ġrat ified +Ġcasual ty +H OME +p sey +f ee +Ġspark ling +Ġd é +Ġconcert ed +C atal +Ġcomp lying +ĠA res +ĠD ent +Sh ut +Ġsk im +ad minist +Ġhost ilities +ĠG ins +Ġ6 08 +Ġm uddy +ĠMc Int +ĠDec ay +5 25 +Ġconspic uous +ĠEx posure +Ġresc ind +Ġwear able +Ġ3 28 +our met +ah s +ĠRob ots +Ġe clips +inst ance +ĠRE PORT +ĠApp l +0 30 +ĠSk ies +01 00 +Ġfall acy +S ocket +ĠRece iver +Ġsol ves +ĠButter fly +ĠSho pping +ĠFI RE +65 4 +Med ic +Ġsing ers +ĠNeed less +'' '' +isher s +ĠD ive +58 8 +Ġselect ively +Ġcl umsy +88 9 +Ġpurch aser +ear ned +ard y +Ġbenef iting +eng lish +Ġyield ing +ĠP our +Ġspin ach +Ġdel ve +ĠC rom +6 10 +Ġexport ing +ĠMA KE +Ġ26 3 +Ġg rop +Ġenv oy +ĠInqu iry +ĠLu igi +d ry +ĠT uring +Thumbnail Image +ĠVar iety +Ġfac et +Ġfl uffy +Ġexcerpt s +Ġsh orth +ĠOl sen +CL UD +Ġrel iant +ĠUN C +T our +Ġbat hing +Comp any +Ġglobal ization +P red +ĠMalf oy +Ġh oc +j am +craft ed +ĠBond s +ĠKiss inger +Eng land +Ġorder ly +cat entry +Ġ26 1 +Ġexch anging +ĠInt ent +ĠAmend ments +D OM +Ġst out +³³³³³³³³ ³³³³³³³³ +ĠAir bus +Ġ27 8 +hy de +P oll +Item ThumbnailImage +Ġlooph oles +ĠPill ar +Ġexpl or +St retch +A part +Ġun married +Lim it +ĠTransform ers +Ġintellect ually +unct ure +18 00 +Ġd arn +B razil +Ġleft over +ber us +f red +Mine craft +3 26 +ĠForm s +Ġproof s +ĠDes igned +Ġindex es +ĠSupp ose +EM S +ĠL oving +ĠBon nie +im ating +OT US +Ġconduct or +Ġbehav ed +ĠF ren +Ġsy nerg +Ġmillenn ium +Ġcater ing +ĠL auder +W r +ĠY iannopoulos +ĠAT F +Ġensl aved +Ġawaken ed +D VD +ĠED ITION +ĠConc ert +ĠChall enger +ĠH aku +umer ic +Ġdep recated +ĠSH AR +4 12 +Ġdy stop +Ġtremb ling +Ġdread ed +ĠSp ac +p adding +Re pl +ĠG arrison +M ini +Ġun paralleled +am ar +URR ENT +w reck +c ertain +t al +ĠC LS +app ings +Ġsens ed +Ġf encing +ĠPas o +ĠDes k +Ġsc off +Ġcontem plate +ĠL iga +l iquid +75 7 +Ġapp rentice +ĠUCH IJ +5 70 +ĠTh ousand +ĠIll um +Ġchampion ed +ãĤ Į +Ġelect ors +Ġ3 98 +ĠH ancock +round ed +ĠJ OHN +Ġuns atisf +Ġqual ifier +ĠGad get +EN E +Ġdead liest +ĠPl ants +Ġ ions +Ġacc ents +Ġtwe aking +Ġsh aved +F REE +ĠCh aser +Again st +9 60 +Ġmeth amphetamine +Ġnormal ized +Ġ$ \ +ĠPre cision +ĠGu am +Ġch oked +ĠX II +ĠCast ing +Tor rent +Ġscal p +ĠJagu ar +w it +Ġsem ic +ix ie +ĠG ould +Ġconf ines +N usra +ĠL on +ĠJ ugg +y cle +ĠCod ec +E gypt +Ġrest rain +ĠAl iens +Ġch oking +ĠD unk +ĠBell a +ab c +Ġsl ang +Ġneuro trans +s av +Ġempower ment +â ĨĴ +Ġclim bers +ĠM im +ĠF ra +ros se +Cap ital +ĠCth ulhu +Inter face +Ġprof icient +ĠIN TO +Ġ3 18 +ront al +5 80 +ĠDes pair +K enn +Ġscrim mage +ĠCo at +as ions +Ġwall paper +ĠJ ol +Ġresurg ence +Ġant iv +ĠB alls +² ¾ +Ġbuff ers +Ġsub system +ĠSt ellar +ĠL ung +A IDS +Ġerad icate +Ġblat antly +Ġbehav es +ĠN un +Ġant ics +ex port +DE V +w b +Ġph p +ĠInteg rity +Ġexplore r +Ġrev olving +auth ored +g ans +Ġbas k +Ġas ynchronous +å į +TH ING +69 8 +G ene +ĠR acer +ĠN ico +iss ued +Ġser mon +p ossibly +Ġsize of +Ġentrepreneur ial +ox in +ĠMin erva +Ġpl atoon +n os +ri ks +A UT +ĠAval anche +ĠDes c +ij 士 +ĠP oc +Ġconf erred +Î » +Ġpat ched +F BI +66 2 +Ġfract ures +Ġdetect s +Ġded icate +Ġconstitu ent +Ġcos mos +W T +Ġswe ats +Ġspr ung +b ara +s olid +Ġuns us +Ġbul ky +ĠPhilipp e +ĠFen rir +Ġtherap ists +ore al +^^ ^^ +Ġtotal ed +Ġboo ze +ĠR PC +Prosecut ors +Ġdis eng +ĠSh ared +Ġmotor cycles +Ġinvent ions +Ġlett uce +ĠMer ge +ĠJ C +Ġspiritual ity +ĠWAR NING +Ġunl ucky +ĠT ess +Ġtong ues +ĠD UI +T umblr +Ġle ans +Ġinv aders +Ġcan opy +ĠHur ricanes +ĠB ret +ĠAP PLIC +id ine +ick le +Reg arding +Ġve ggies +Ġe jac +ju ven +F ish +D EM +ĠD ino +Th row +ĠCheck ing +be ard +( & +Ġj ails +Ġh r +trans fer +iv ating +Ġfle ets +ĠIm ag +ĠMc Donnell +Ġsnipp et +Is a +ĠCh att +ĠSt ain +ĠSet FontSize +ĠO y +ĠMathemat ics +49 4 +Ġelectro ly +ĠG ott +ĠBr as +B OOK +ĠF inger +d ump +Ġmut ants +Ġrent als +Ġinter tw +Ġc reek +ail a +Bro ther +ĠDisc ord +pe e +raw ler +Ġcar p +Ġ27 9 +ãĤ· ãĥ£ +rel ations +Ġcontr asts +Col umn +Ġrec onnaissance +Ġun know +Ġl ooting +Ġregul ates +Ġopt imum +ĠChero kee +ĠA ry +Lat est +Ġroad side +Ġd anced +ĠUnic orn +A cknowled +Ġuncont roll +ĠM US +at io +ch ance +ha ven +VAL UE +Ġfavour ites +Ġceremon ial +b inary +pe ed +wood s +EM P +Ġv ascular +Ġcontempl ated +Ġbar ren +ĠL IST +Y ellow +ospons ors +Ġwhisk y +ĠM amm +ĠDeV os +min imum +H ung +44 2 +P ic +ĠSnap dragon +77 6 +Ġcar ving +Ġund ecided +Ġadvantage ous +Ġpal ms +ĠA Q +Ġst arch +L oop +Ġpadd le +Ġfl aming +ĠHor izons +An imation +bo ost +Ġprob abilities +ĠM ish +Ġex odus +ĠEditor ial +Ġfung us +Ġdissent ing +ĠDel icious +rog ram +ĠD yn +d isk +t om +Ġfab rics +ĠC ove +ĠB ans +Ġsoft en +ĠCON S +Ġin eligible +Ġestim ating +ĠLex ington +pract ice +of i +Ġshe dding +ĠN ope +Ġbreat hed +ĠCorinth ians +y ne +ek i +B ull +Ġatt aching +reens hots +Ġanaly se +ĠK appa +Ġuns ustainable +Ġinter pol +ank y +he mer +Ġprot agonists +Ġform atted +ĠBry ce +ĠAch illes +ĠAb edin +sh ock +Ġb um +b os +qu a +ĠW arn +q t +ĠDi abetes +8 64 +ĠIn visible +Ġvan ish +Ġtrans mitting +Ġmur ky +ĠFe i +Ġawa ited +ĠJur assic +umm ies +Ġmen acing +g all +C ath +B uilt +ild o +ĠV otes +Ġon t +Ġmun itions +ĠFre em +ÃŃ n +Ġdec ency +lo pp +ie ved +ĠG ord +Ġun thinkable +ĠNews week +Ġ3 21 +He at +Ġpresent er +ji ang +Ġpl ank +ĠAval on +Ġben z +ĠR out +Ġslam ming +ĠD ai +ou ter +ĠCook ie +ĠAlic ia +ge y +Ġvan ity +Ġow l +á µ +t ested +ĠAw akens +Ġcan v +Ġblind ly +ĠRid ley +ĠEm ails +Requ ires +ĠSer bian +ograp hed +if rame +eter ia +Ġaltern ating +qu iet +Ġsoc iology +ĠUn lock +ĠCommun ism +Ġo ps +Ġatt ribution +Ġab duction +ĠAb ram +Ġsidel ined +ĠB OOK +Ġref ining +ĠFe eling +ĠOs lo +ĠPru itt +r ack +ang ible +Ġcaut iously +ĠM ARK +eed s +M ouse +ĠStep h +ĠP air +S ab +99 7 +ĠBa al +B ec +Ġcomm a +ĠP all +ĠG ael +Ġmisunder stand +ĠP esh +Order able +Ġdis mal +ĠSh iny +% " +Ġreal istically +Ġpat io +ĠG w +ĠVirt ue +Ġexhaust ing +wh atever +oph ys +y ip +4 18 +Ad just +ĠWa iting +ess on +ĠMaz da +ĠDo zens +Ġstream lined +Ġincompet ence +ĠM eth +Ġeth os +ON ES +Ġincent iv +Ġgr itty +ĠBut cher +Head er +Ġexp onential +à Ł +Ġcorrel ate +Ġcons ensual +s ounding +R ing +Orig in +Ġcon clusive +fe et +ac ly +ĠF ernandez +Buy able +Ġd ucks +aunt lets +Ġel ong +Ġ28 6 +Ġsim ul +G as +ĠK irst +Ġprot r +ĠRob o +ĠAo E +op ol +Ġpsych ologically +sp in +ilater ally +ĠCon rad +W ave +44 1 +ĠAd vertisement +ĠHarm on +ĠOri ental +is Special +Ġpresum ptive +Ġw il +ĠK ier +ne a +Ġp pm +Ġhar bour +ĠW ired +comp any +Ġcor oner +atur days +ĠP roud +ĠN EXT +ĠFl ake +val ued +ce iver +Ġfra ught +Ġc asing +Ġrun away +Ġg in +ĠLaure nt +ĠHar lem +ĠCur iosity +qu ished +Ġneuro science +ĠH ulu +Ġborrow er +Ġpetition er +ĠCo oldown +W ARD +Ġinv oking +conf idence +For ward +Ġst s +pop ulation +Delivery Date +Fil m +ĠC ov +quick Ship +quickShip Available +prim ary +isSpecial Orderable +inventory Quantity +channel Availability +BO X +ĠMulti player +ĠJen ner +77 8 +ĠM d +Ġ~ /. +M N +Ġchild ish +Ġantioxid ant +ĠChrom ebook +Ġ27 4 +Ġscreen play +Ġadvent urous +ĠRelations hip +respons ive +ming ton +Ġcorner stone +ĠF ey +F IR +Ġrook ies +ĠF eaturing +Ġorig inate +Ġelectro des +ant es +Ġscript ures +Ġgl ued +Ġdiscont ent +Ġaff licted +lay out +B rave +Ġm osa +ĠQuant ity +ĠH ik +w inner +H ours +Ġent ail +ĠCell s +olog ue +Ġv il +Ġpre acher +Ġdecor ative +d ifferent +Ġprejud ices +ĠSm oking +ĠNotting ham +so Type +Ġrhyth ms +ĠAl ph +bl ast +Ste el +ĠDaniel le +Ġstr ife +Ġrem atch +so DeliveryDate +ĠF ork +t rip +ol ulu +hes es +C G +ĠPOLIT ICO +ost a +ĠDr ift +é¾įå ¥ +é¾įå¥ ij士 +Ġvet ting +ĠJin ping +ĠRec ession +Min or +ĠF raud +enf ranch +Ġconven ed +ĠNA ACP +ĠMill ions +ĠFarm ing +ĠW oo +ĠFl are +rit o +imm igrant +Ġvac ancy +ĠHE AD +ĠV aj +eg al +ĠV igil +Stud y +Ġru ining +Ġr acks +Ġhe ater +ĠRand olph +ĠBr ush +ĠT ir +Ø ¨ +Ġc ov +% ] +Ġrecount s +ĠO PT +ĠM elt +Ġtr uce +Ġcas inos +Ġcrus ade +Ġcarn age +Ġstri pe +ĠK yl +Text ures +Ġ6 98 +Ġpro clamation +Ġgood ies +Ġ........ .. +pro claimed +P olit +Ġtop ical +Ġspecial ize +ĠA min +g m +Ġanch ored +Ġbear ings +s ample +ĠHigh land +ĠAut ism +Ġmerc enary +Ġinterview er +L ER +ĠSom ers +Ġembry o +ĠAss y +Ġ28 1 +ĠEd iting +ĠCh osen +6 60 +Ġp ci +ĠThunder bolt +BI LL +Ġchuck led +jri wal +h of +Ġearth ly +() { +ind ependence +Ġdisp ers +ĠV endor +ĠG areth +Ġp als +P enn +ĠSub mit +ic um +Th u +Ġcl andestine +Ġcann ibal +ĠCl erk +E Stream +gal itarian +âĻ ¥ +g ew +Ġhor rend +ĠL ov +ĠRe action +ocr in +Class ic +Ġecho ing +Ġdiscl osing +ĠIns ight +og un +ĠInc arn +upload s +pp erc +guy en +Ġ19 01 +ĠB ars +68 7 +Ġb ribes +ĠFres no +ur at +ĠRe ese +Ġintr usive +Ġgri pping +ĠBlue print +ĠR asm +un ia +man aged +ĠHeb do +Ġ3 45 +Ġdec oding +Ġpo ets +Ġj aws +ĠF IGHT +am eless +ĠMead ows +ĠHar baugh +Inter view +ĠH osp +ĠB RA +Ġdelet ion +m ob +W alker +ĠMoon light +ĠJ ed +ĠSoph ia +Ġus ur +Ġfortun ately +ĠPut ting +ĠF old +Ġsan itation +Ġpart isans +IS ON +B ow +ĠCON C +ĠRed uced +ĠS utton +Ġtouch screen +Ġembry os +âĢ¢âĢ¢ âĢ¢âĢ¢ +ĠK rug +com bat +ĠPet roleum +Ġam d +ĠCos mos +Ġpresc ribing +Ġconform ity +ours es +Ġplent iful +Ġdis illusion +ĠEc ology +itt al +Ġf anc +Ġassass inated +regn ancy +Ġperenn ial +ĠBul lets +Ġst ale +Ġc ached +ĠJud ith +ĠDise ases +All en +Ġl as +Ġsh ards +ĠSu arez +ĠFriend ship +inter face +ĠSupp orters +add ons +46 2 +ĠIm ran +ĠW im +Ġnew found +ĠM b +An imal +Ġd arling +and e +Ġrh y +ĠTw isted +pos al +yn ski +Var ious +× ľ +ĠK iw +uy omi +Ġwell being +ĠL au +an os +Ġunm ist +Ġmac OS +Ġrest room +ĠOl iv +ĠAir ways +Ġtimet able +9 80 +Ġrad ios +v oy +ias co +Ġcloud y +ĠDraw ing +Any thing +Sy ria +ĠH ert +st aking +Ġun checked +Ġb razen +ĠN RS +69 7 +onom ic +est ablish +Ġl eng +Ġdi agonal +ĠF ior +L air +ĠSt ard +Ġdef icient +jo ining +be am +Ġomn ip +Ġbl ender +Ġsun rise +Mo ore +ĠF ault +ĠCost ume +ĠM ub +Fl ags +an se +Ġpay out +ĠGovern ors +ĠD illon +ĠBan ana +N ar +Ġtra iled +Ġimperial ist +um ann +ats uki +4 35 +ĠRoad s +Ġsl ur +ĠIde ally +Ġt renches +C trl +Ġmir rored +ĠZ el +ĠC rest +Comp at +ĠRoll s +sc rib +ĠTra ils +omet ers +w inter +Ġimm ortality +il ated +Ġcontrad icts +un iversal +ill ions +ĠM ama +opt im +AT URE +Ġge o +et ter +ĠCar lo +4 24 +Ġcanon ical +ĠStrongh old +n ear +Ġperf ume +Ġorche stra +od iac +Ġup he +Ġreign ing +vers ive +Ġc aucuses +ĠD EM +Ġinsult ed +Ġ---- -- +ĠCr ush +Ġroot ing +ĠWra ith +Ġwh ore +Ġto fu +C md +ĠB ree +Ġ$ _ +Ġr ive +ĠAd vertising +Ġw att +ĠH O +Ġpersu asive +ĠParam eters +Ġobserv ational +ĠN CT +ĠMo j +ĠSal on +Ġtr unc +Ġexqu isite +ĠMar a +Ġpo op +ĠAN N +Ex c +ĠWonder ful +ĠT aco +Ġhome owner +ĠSmith sonian +orpor ated +mm mm +Ġlo af +ĠYam ato +ĠInd o +Ġcl inging +á s +Ġimm utable +h ub +Or ange +Ġfingert ips +ĠWood en +ĠK idd +ĠJ PM +ĠDam n +C ow +c odes +48 2 +Ġiniti ating +ĠEl k +ĠCut ting +Ġabsent ee +ĠV ance +ĠLil ith +G UI +Ġobsc ured +Ġdwar ves +ĠCh op +ĠB oko +Val ues +Ġmult imedia +Ġbrew ed +Reg ular +CRIP TION +ĠMort al +Ġa pex +Ġtravel er +Ġbo ils +Ġspray ing +Rep resent +ĠStars hip +4 28 +Ġdisappro val +Ġshadow y +Ġlament ed +ĠRe place +ĠFran ç +67 7 +d or +Ġunst oppable +Ġcoh orts +gy n +ĠClass ics +ĠAm ph +Ġsl uggish +ĠAdd iction +ĠPad res +Ġins cription +Ġin human +min us +ĠJere miah +at ars +Ter ror +ĠT os +ĠSh arma +ast a +c atch +Ġpl umbing +ĠTim bers +Sh ar +H al +ĠO sc +Ġcou pling +hum ans +Ġsp onge +Ġid ols +ĠSp a +ĠAdv ocate +ĠBe ats +lu a +Ġtick ing +Ġload er +ĠG ron +8 10 +Ġstim ulated +Ġside bar +ĠManufact urer +ore And +19 73 +Ġpra ises +ĠFl ores +dis able +ĠElect rical +ra ise +E th +Ġmigr ated +Ġlect urer +K ids +ĠCa vern +Ġk ettle +Ġgly c +ĠMand ela +ĠF ully +å§ « +FIN EST +Ġsquee zing +ĠRy der +amp oo +oreAnd Online +Inst oreAndOnline +Buyable InstoreAndOnline +Ġcommem orate +ĠRamp age +Aust in +ĠSh roud +ĠRu ins +9 15 +ĠK H +Ġwater front +ĠE SC +b aby +ĠC out +ĠEm blem +Ġequival ents +49 2 +Un ique +ĠNiet zsche +brow ser +Ġim itation +ĠWere wolf +ĠKir in +ac as +' ," +Ġà ¾ +Review ed +Ġc unt +Ġvo ic +ĠLen ovo +Ġbond ed +48 1 +Ġinhib itors +Ġendeav ors +ĠHav ana +ĠSt out +ĠJ olly +A ctor +*/ ( +Ġoccur rences +ĠT ens +Incre ased +ĠACT ION +Ġ ãĢĮ +ĠRank ings +ĠB reat +Ġ30 9 +D ou +Ġimpact ing +ĠDuc hess +pre fix +Q B +Ġsummon ing +Ġbest owed +ĠKe pler +ĠPOW ER +c ube +ĠK its +ĠG rip +Ġop ium +Ġrep utable +t oc +ich ael +ĠR ipple +Ġcaf é +ĠZ oom +ĠBur ma +Ġwa ive +Ġst alls +Ġdem eanor +inc erity +Ġfluor ide +ĠSH OULD +Par is +Ġlong ing +Ġpl at +Ġgross ly +Ġbull s +Ġshowc asing +ex pected +ĠG addafi +engine ering +Re peat +ĠK ut +Ġconce ivable +Ġtrim med +osc ope +ĠCand idate +ĠT ears +rol og +Lew is +S UP +Ġroad map +Ġsal iva +Ġtrump et +Jim my +Ġmirac ulous +Ġcolon ization +Ġam put +ĠGN OME +ate ch +D ifferent +ĠE LE +ĠGovern ments +ĠA head +ãħĭ ãħĭ +word press +L IB +ĠIn clude +ĠDor othy +0 45 +ĠColomb ian +Ġle ased +88 4 +Ġde grading +ĠDa isy +i ations +Ġbapt ized +Ġsurn ame +co x +Ġblink ed +ãĥ ¢ +Ġpoll en +Ġder mat +Ġre gex +ĠNich olson +ĠE ater +ç ľ +rad or +Ġnarrow er +Ġhur ricanes +Ġhalluc inations +r idden +ISS ION +ĠFire fly +Ġattain ment +Ġnom inate +Ġav ocado +ĠM eredith +Ġt s +Ġreve rence +Ġe uph +Ġcr ates +ĠT EXT +Ġ4 43 +Ġ3 19 +J SON +iqu ette +Ġshort stop +ic key +Ġpro pelled +Ġap i +ĠTh ieves +77 9 +Ġovers aw +Ġcol i +ĠNic ola +Ġover cl +ik awa +ĠC yr +Ġ38 4 +78 9 +ĠAll ows +10 27 +Det roit +TR Y +set up +ĠSocial ism +Sov iet +s usp +ĠAP R +ĠShut down +Ġal uminium +zb ek +ĠL over +GGGG GGGG +Ġdemocr acies +Ġ19 08 +ĠMer rill +ĠFranco is +gd ala +Ġtraff ickers +ĠT il +ĠGo at +Ġsp ed +ĠRes erv +Ġpro d +55 2 +Ġc ac +ĠUn iv +ĠSch we +Ġsw irling +ĠWild erness +ĠEgg s +Ġsadd ened +Ġarch aic +H yd +Ġexcess ively +B RE +Ġaer ospace +ĠVo ices +Cra ig +Ġign ited +In itially +ĠMc A +Ġhand set +Ġreform ing +Ġfrust rations +ĠDead pool +ĠBel ichick +ract or +ĠRagnar ok +ĠD rupal +ĠApp roximately +19 20 +ĠHub ble +arm or +ĠSar as +ĠJon as +Ġnostalg ic +Ġfeas ibility +Sah aran +Ġorb iting +Ġ9 70 +R u +Ġsh in +ĠInvestig ators +Ġinconsist encies +ĠP AN +B G +Ġgraz ing +Ġdetect ors +ĠStart up +ĠFun ny +ĠNa omi +Consider ing +Ġh og +ut f +ce mic +Ġfort ified +ĠFun ctions +Ġcod ec +nut rition +H at +" ! +micro soft +55 8 +ĠTh in +ĠA CE +Al ias +ĠO PS +p apers +P K +ãĢ İ +Ġimpro bable +N orthern +equ al +Ġlook out +Ġty res +ĠMod ified +ĠK op +Abs olutely +Ġbuild up +sil ver +Ġaud i +Ġgro tesque +ĠSab er +ĠPres byter +ON Y +Ġglac iers +ĠSho als +ĠK ass +ĠH RC +ĠNic ol +ĠL unch +ĠF oss +âĸ Ĵ +AD RA +ĠOne Plus +o ing +ground s +Ġincident al +Ġdatas ets +68 9 +ĠClarks on +Ġassemb ling +ĠCorrect ions +Ġdrink ers +Ġqual ifiers +Ġle ash +Ġunf ounded +ĠH undred +Ġkick off +T i +Ġrecon cil +ĠGr ants +ĠCompl iance +ĠDexter ity +Ġ19 06 +w arn +D allas +Max imum +n ard +av ia +be aut +ens itivity +tr ace +Ġpione ers +ĠF ract +ãĢ ı +Ġpre cept +Ġgloss y +ĠI EEE +Ac ross +Ġ6 80 +S leep +che on +Ġsatir ical +ĠMin otaur +ĠCla ude +Ġr é +ape go +Ġcar rot +ĠSem in +ino a +Ġz o +Ind ependent +Ġdiagn oses +ĠC ue +M AR +Ġrend ition +ĠK ik +Ġpath ology +Ġselect s +Link edIn +Ġass ay +ĠD res +Ġtext ual +post ed +IT AL +ĠM aul +N eal +Ġinter connected +Ġerr atic +ĠVir us +Ġ5 30 +Ġenvironmental ists +ĠP helps +Ġeng agements +ĠIN ST +Ġeconom ical +nox ious +Ġg earing +izz y +Ġfavor ably +ĠMcG ill +T erm +Ġh anged +Ġball park +ĠRe yes +Ġbe ware +ĠP sal +ĠMass acre +q i +Ġin accessible +acly sm +Ġfr ay +ill ac +Ġbitter ly +ĠCert ification +Mich igan +Ġir respective +al ore +Em pty +Ġendorse ments +Ġund et +f g +equ ipped +Ġmerc iless +ĠC ust +Ġimm ature +Ġvou cher +ĠBlack well +Ñ ı +h awk +dis ciplinary +ile e +ĠMak oto +ĠD ude +ãĥĩ ãĤ£ +Y ears +Ġin ver +Ġsh aman +ĠY ong +ip el +ell en +ĠCath y +br ids +Ġs arc +65 1 +N ear +Ġground work +Ġam az +Ġ4 15 +ĠHunting ton +hew s +ĠB ung +Ġarbit rarily +ĠW it +ĠAl berto +Ġdis qualified +best os +46 1 +Ġp c +Ġ28 4 +ro bat +Rob in +Ġh ugs +ĠTrans ition +ĠOcc asionally +Ġ3 26 +ĠWh ilst +ĠLe y +Ġspaces hip +cs v +Ġun successfully +ĠA u +le ck +ĠWing ed +ĠGrizz lies +. � +Ġne arer +ĠSorce ress +ĠInd igo +El se +8 40 +let es +Co ach +Ġup bringing +ĠK es +Ġseparat ist +Ġrac ists +Ġch ained +Ġabst inence +lear ning +Ġrein stated +Ġsymm etry +Ġremind ers +ĠChe vy +Ġm ont +Ġexempl ary +ĠT OR +Z X +Ġqual itative +ĠSt amp +ĠSav annah +ĠRoss i +Ġp aed +Ġdispens aries +ĠWall s +ĠCh ronic +Ġcompliment ary +ĠBeir ut +Ġ+ --- +igs list +Ġcrypt ographic +mas ters +ĠCap itals +Ġmax imal +Ġent ropy +Point s +Ġcombat ants +l ip +ĠGl ob +ĠB MC +ph ase +th ank +HT TP +Ġcomm uter +Ġ\( \ +.. / +ĠReg ener +ĠDO I +ĠActiv ision +Ġsl it +os al +RE M +Ġch ants +Y u +Ke ys +Bre xit +ĠFor ced +Ari zona +Ġsquad ron +IS O +ĠMal one +Ġ3 38 +Ġcontrast ing +Ġt idal +Ġlib el +Ġimpl anted +Ġupro ar +ĠC ater +Ġpropos itions +M anchester +ĠEuro s +it amin +G il +ĠEl ven +ĠSe ek +ĠB ai +Ġredevelop ment +ĠTown s +ĠL ub +! ", +al on +K rist +Ġmeas urable +Ġimagin able +Ġapost les +Y N +7 60 +Ġster oid +Ġspecific ity +ĠL ocated +ĠBeck er +ĠE du +ĠDiet ary +uts ch +ĠMar ilyn +Ġbl ister +ĠM EP +ĠK oz +ĠC MS +y ahoo +ĠCar ney +Ġbo asting +ĠC aleb +By te +read s +ad en +Pro blem +ĠWood ward +S we +S up +ĠK GB +Set up +Ġtac it +Ġret ribution +Ġd ues +ĠM ü +. ? +ä¸ Ń +p ots +Ġcame o +ĠP AL +educ ation +A my +like ly +g ling +Ġconstitution ally +ĠHam m +ĠSpe ak +Ġwid gets +br ate +Ġcra ppy +ĠI ter +Ġanticip ating +ĠB out +P ixel +ĠY ep +ĠLaur ie +Ġh ut +Ġbullet in +ĠSal vation +Ġch ats +ear able +Honest ly +AL TH +onse qu +c ult +isco very +ovy ch +Ġse lves +ĠSat oshi +S ounds +Ġconver gence +ĠRosen berg +19 74 +Ġnas al +Ġfull est +Ġfer ocious +x us +ist e +AM S +Ġlobb ied +Ġso othing +ĠGun n +t oday +0 24 +Ġinspir ational +ĠN BN +p b +g ewater +or ah +all owed +ĠCol iseum +Ġspecial izing +Ġinsane ly +ĠT ape +del ay +Ġt arn +ĠP ound +Ġmel anch +Ġdeploy ments +il and +Ġless en +Ġfur ry +ĠUE FA +Ġblood shed +ĠMe ier +ither ing +Ġhe irs +ĠJ aw +ax ter +ĠPublic ations +Ġal ters +int ention +ĠWinc hester +d etermination +ĠLif etime +th in +Mon ster +7 80 +Ġapprox imation +Ġsuper markets +ĠSecond s +or os +h uge +Ġb ribe +ĠLIM ITED +un ed +Ġmis interpret +ĠIn jury +Ġ3 67 +Ġthreshold s +ĠCarn ival +Ġgastro intestinal +Ġguid eline +Ġde ceived +f eatures +Ġpurported ly +ĠRon nie +ĠNew t +Ġsp acious +as us +Ġsuperhero es +ĠCyn thia +le gged +k amp +ch io +Ġth umbnail +ĠShir ley +ill ation +Ġshe ds +ĠZ y +E PA +Ġdam s +Ġy awn +n ah +ĠPe ggy +ĠE rie +ĠJu ventus +ĠF ountain +r x +don ald +al bum +ĠComp rehensive +Ġc aching +ĠU z +ulner ability +ĠPrinc iple +ĠJ ian +ing ers +cast s +ĠOs iris +ch art +t ile +ĠTiff any +ĠPatt on +ĠWh ip +Ġovers ized +J e +ĠCind erella +ĠB orders +ĠDa esh +M ah +Ġdog ma +Ġcommun ists +v u +Coun cil +Ġfresh water +Ġw ounding +Ġdeb acle +Ġyoung ster +Ġthread ed +ĠB ots +ĠSav ings +ãģ Ĥ +ol ing +oh o +Ġillum ination +M RI +Ġlo osen +tr ump +ag ency +ur ion +Ġmoment arily +ĠCh un +ĠBud apest +ĠAl ley +D isk +Ġaston ished +ĠCon quer +ĠAccount ing +h aving +ĠWe in +ĠAl right +Ġrev olver +Ġdel usion +Ġrelic s +Ġad herent +qu ant +Ġhand made +or io +Ġcomb ating +c oded +Ġquad ru +re th +N ik +ĠTrib al +ĠMyster ious +Ġin hal +ĠWin ning +ĠClass ification +ch anged +Ġun ab +Ġsc orn +icip ated +w l +ond uctor +Ġrein forcing +ĠChild hood +an ova +Ġadventure r +Ġdoctor al +ĠStrateg ies +Ġengulf ed +ĠEnc ounter +Ġl ashes +Crit ical +ric ular +ĠU TF +oci ation +check ing +ĠConsult ing +Run time +per iod +ĠAs gard +Ġdist illed +ĠPas adena +ĠD ying +ĠCOUN TY +Ġgran ite +Ġsm ack +Ġparach ute +ĠS UR +Virgin ia +ĠF urious +78 7 +ĠO kin +Ġcam el +ĠM bps +19 72 +ĠCh ao +ĠC yan +j oice +ef er +ĠW rap +ĠDeb ate +S eg +Ġfore arm +ĠIgn ore +Ġtim estamp +Ġprob ing +ĠNo on +ĠGra il +f en +Ġdorm ant +ĠFirst ly +ĠE ighth +ĠH UN +ĠDes ire +or as +Girl s +ĠDes mond +z ar +am ines +O AD +exec ute +Ġbo obs +ĠAT L +_ ( +Chel sea +Ġmasturb ation +ĠCo C +Ġdestroy er +ĠCh omsky +Ġsc atter +ĠAss ets +79 6 +ĠC argo +Ġrecept ive +ĠSc ope +Ġmarket ers +Ġlaun chers +Ġax le +ĠSE A +se q +ĠM off +f inding +ĠGib bs +Georg ia +extreme ly +N J +Ġlab orers +st als +Ġmed iation +ĠH edge +at own +Ġi od +des pite +v ill +J ane +ex istence +Ġcoinc ided +ĠUt ilities +ĠChe ap +Ġlog istical +Ġcul mination +ĠNic otine +p ak +F older +Ġrod ents +st uff +Ġlaw fully +Ġreper to +io ch +j j +Dial ogue +HH HH +lic tion +Look s +Ġ29 7 +Ġtur rets +ĠAb andon +Ġinc ess +ĠTraff ord +Ġcur led +Ġprefer ring +Ġprivat ization +Ġir resist +ĠP anda +ĠSh ake +ĠMc Gr +ãĥ Ħ +und ers +Ġdiscrim inated +Ġbart ender +I LE +Atl antic +Ġprop ensity +ĠW iz +ĠG im +con ference +Ġrein forces +G h +w agon +Ġe erie +F al +Ġhug ged +rac ist +R IC +F u +Ġf iller +ĠSt ub +Ġeng raved +ĠWrest le +Ġimagin ative +ĠPe er +ĠFact ors +an us +ĠDrac ula +mon itor +Ġrou ters +ib ia +ĠBoo lean +end ale +ĠSl aughter +ĠSh ack +R FC +ĠSpiel berg +S ax +ĠPH OTO +ĠCl over +ĠR ae +Dep ending +ĠMem or +ar am +Ġpier ced +Ġcur tains +v ale +ĠInqu isition +ĠP oke +Ġforecast ing +Ġcompl ains +S ense +ĠHer mes +isc overed +Ġb ible +ĠMor ph +Ġg erm +78 5 +D ON +Ġcon gen +Ġcr ane +ĠD PR +Ġrespect fully +R oom +ĠN aw +ĠDal ai +re ason +ĠAng us +Educ ation +ĠTitan ic +Ë ľ +Ġo val +un ited +Ġthird s +Ġmoist ur +ĠC PC +M iami +Ġtent acles +ĠPol aris +ex c +ex clusive +ĠPra irie +Ġcol ossal +ĠBl end +sur prisingly +ÃŃ s +Ġindo ctr +Ġbas al +ĠMP EG +und o +Spl it +Develop ment +Ġlan tern +19 71 +Ġprov ocation +Ġang uish +ĠB ind +ĠLe ia +duc ers +ipp y +conserv ancy +Ġinitial ize +ĠTw ice +ĠSu k +Ġpred ic +Ġdi ploma +Ġsoc iop +Ing redients +Ġhamm ered +ĠIr ma +Q aida +Ġglim ps +ĠB ian +Ġst acking +Ġf end +gov track +Ġun n +dem ocratic +ig ree +Ġ5 80 +Ġ29 4 +Ġstraw berry +ID ER +Ġcher ished +ĠH ots +Ġinfer red +Ġ8 08 +ĠS ocrates +O regon +ĠR oses +ĠFO IA +Ġins ensitive +Ġ40 8 +Recomm end +ĠSh ine +Ġpain staking +UG E +ĠHell er +ĠEnter prises +I OR +ad j +N RS +L G +Ġalien ated +Ġacknowled gement +ĠA UD +ĠRen eg +Ġvou chers +Ġ9 60 +Ġm oot +ĠDim ensions +Ġc abbage +B right +g at +ĠK lu +Ġlat ent +Ġz e +ĠM eng +Ġdis perse +Ġpand emonium +H Q +Ġvirt uous +ĠLoc ations +ee per +prov ided +Ġse ams +ĠW T +iz o +PR OV +Ġtit anium +Ġrecol lection +Ġcr an +Ġ7 80 +ĠN F +49 1 +64 2 +p acking +59 8 +text ure +Sp ider +fre edom +cipl ed +ĠTAM ADRA +âĻ ¦ +aut hent +ĠW ANT +r ified +Ġr ites +Ġuter us +k iss +Ġâī ¤ +Ġsk illet +Ġdis enfranch +ĠGa al +Comp an +Ġage ing +gu ide +B alt +Ġiter ator +Ġdiscretion ary +t ips +Ġprim ates +ĠTechn ique +ĠPay ments +az el +ĠR OCK +stant ial +0 60 +Ġd mg +ĠJack ets +ĠPlay off +Ġnurs ery +ĠSy mb +art on +Ġannex ation +Color ado +Ġco ils +ĠSh oes +âĦ¢ : +ĠRo z +COM PLE +ĠEve rest +ĠTri umph +J oy +G rid +à ¼ +process or +ĠPros per +ĠSever us +ĠSelect ed +r g +ĠTay yip +St ra +Ġski ing +Ġ? ) +Ġpe g +Tes la +Ġtime frame +Ġmaster mind +ĠN B +scient ific +ĠSh it +gener ic +IN TER +N UM +Ġst roll +ĠEn ix +ĠM MR +ĠE MS +m ovie +Ĥ ª +Ġminim izing +idd ling +Ġilleg itimate +Ġprot otyp +Ġpremature ly +Ġmanual s +obb ies +ĠCass idy +D EC +des ktop +Ġaer os +Ġscreen ings +Ġdeb ilitating +ĠGr ind +nature conservancy +Ġf ades +ter mination +assets adobe +F actor +Ġdefinitive ly +P oké +ap ult +ĠLaf ayette +C orn +ĠCor al +Ġstagn ant +T ue +Ġdissatisf action +G ender +Ġkid neys +ĠG ow +ĠDef eat +ĠAsh ton +Ġcart els +Ġfore closure +ĠExpl ore +stre ngth +ot in +Ġveterin arian +Ġf umble +Ġpar ap +ĠSt rait +r ils +Ġpr ick +ĠBerm uda +ĠAm munition +skin ned +Ġab ound +ĠB raz +Ġshar per +ĠAsc ension +Ġ9 78 +Ġpreview s +Ġcommun ion +ĠX Y +Ġph ony +Ġnewcom er +Ġ3 32 +." ," +Ġredist ribution +Prot ect +ĠSo f +K al +Ġlip stick +w orst +Ġtang led +Ġretrospect ive +int eger +Ġvolunte ering +Ġ19 07 +Ġ -------------------- +ic hen +Ġunve iling +Ġsen seless +Ġfisher ies +\ - +Ġh inges +Ġcalcul us +My th +Ġund efeated +Ġoptim izations +Ġdep ress +Ġbill board +ĠY ad +ĠPy ramid +Is n +I de +Ġleg ion +ĠK ramer +ent anyl +Ġpenet rating +ĠHaw th +ĠPR ODUCT +ĠGer ard +ĠP act +ĠIn cluding +ĠEl ias +ĠEl aine +vis ual +Ġhum ming +Ġcond esc +ĠF asc +ä¸ Ĭ +Ġe galitarian +Ġdev s +ĠD ahl +O ps +D H +ĠB ounce +id ated +ald o +Ġrepublic an +Ġh amb +ĠS ett +ograph ies +CH APTER +Ġtrans sexual +Ġsky rocket +ans wer +Ġmark up +Ø ª +Ġhero ine +Comp are +ĠT av +Be ast +Ġsuccess ors +Ġna ïve +ĠBuck ley +st ress +me at +Ġdownload able +Ġindex ed +Ġsc aff +ĠL ump +ĠHom o +Stud io +In sp +Ġr acked +far ious +ĠPet ty +Ex ternal +Ġ19 09 +W ars +com mit +put ers +Ġun ob +ĠEr r +ĠE G +ĠAl am +ĠSiber ia +ĠAtmosp heric +IS TER +ĠSatan ic +trans lation +ĠL oud +tra umatic +l ique +Ġreson ate +ĠWel ch +Ġspark ing +ĠT OM +t one +Ġout l +Ġhandc uffed +ĠSer ie +8 01 +Ġland marks +ĠRee ves +Ġsoft ened +Ġdazz ling +ĠW anted +month s +Mag ikarp +Ġunt reated +ĠBed ford +M i +ĠDynam o +O re +79 5 +Ġwrong ful +Ġl ured +Ġcort isol +Ġve x +d rawn +ile t +Download ha +ĠF action +Ġlab yrinth +Ġhij acked +w aters +er ick +Ġsuper iors +ĠRow ling +ĠGu inness +Ġt d +99 2 +Ġune arthed +Ġcentr if +Ġsham eless +P od +ĠF ib +Ġ icing +Ġpredict or +Ġ29 2 +fore station +con struct +C and +@ # +Ġag itated +Ġre pr +OV A +Ġkn itting +ĠLim a +Ġf odder +68 4 +ĠPerson a +k l +7 01 +Ġbreak up +á ¸ +Ġapp alled +Ġantidepress ants +ĠSus sex +Har ris +ĠTher mal +ee ee +U pload +Ġg ulf +Ġdoor step +ĠSh ank +L U +ĠM EN +ĠP ond +s orry +Ġmis fortune +n ance +Ġb ona +M ut +Ġde graded +ĠL OG +ĠN ess +an imal +Ġa version +und own +Ġsupplement ed +ĠC ups +Ġ50 4 +Ġdep rive +ĠSpark le +Å Ĥ +ĠMed itation +auth ors +ĠSab an +ĠN aked +air d +ĠMand arin +ĠScript ures +ĠPerson nel +ĠMahar ashtra +Ġ19 03 +ĠP ai +ĠMir age +omb at +Access ory +Ġfrag mented +T ogether +Ġbelie vable +ĠGl adiator +al igned +ĠSl ug +M AT +Ġconvert ible +ĠBour bon +amer on +ĠRe hab +nt ax +Ġpowd ered +pill ar +Ġsm oker +ĠMans on +ĠB F +5 11 +ĠGood ell +ĠD AR +m ud +g art +Ġob edient +ĠTrans mission +ĠDon ation +8 80 +Ġbother ing +Material s +ãĤ ± +dest roy +Ġfore going +Ġanarch ism +ĠK ry +ice ps +Ġl ittered +ĠSch iff +Ġanecd otal +un its +Ġf ian +ĠSt im +ĠS OME +ĠInv aders +Ġbehaviour al +ĠVent ures +Ġsub lime +Ġfru ition +ĠPen alty +Ġcorros ion +¶ ħ +Ġlik ened +Ġbesie ged +ween ey +ĠCre ep +Ġlinem en +mult i +ic ably +ud der +Ġvital ity +Ġshort fall +ĠP ants +ap ist +H idden +ĠDro ps +med ical +Ġpron unciation +ĠN RL +Ġinsight ful +J V +ĠBe ard +ĠCh ou +Ġchar ms +Ġb ins +Ġamb assadors +ĠS aturdays +Ġinhib itor +ĠFr anch +6 01 +', ' +ĠCon or +art ney +ĠX peria +g rave +be es +ĠProtest ants +Ġso aking +ĠM andal +Ġph ased +Ġ6 60 +Ġsc ams +Ġbuzz ing +ĠItal ians +ĠLoren zo +ĠJ A +Ġhes itated +Ġcl iffs +ĠG OT +ingu ishable +Ġk o +Ġinter ruption +Z ip +Lear ning +Ġundersc ores +ĠBl ink +K u +57 9 +ĠAut ob +I RE +Ġwater ing +Ġpast ry +8 20 +Ġvision ary +ĠTempl ar +awa ited +Ġpist on +Ġant id +current ly +Ġp ard +Ġw aging +Ġnob ility +ĠY us +Ġinject ing +f aith +ĠP ASS +å º +Ġret ake +ĠPR OC +Ġcat hedral +b ash +Ġwrest lers +Ġpartner ing +Ġn oses +Ġ3 58 +Trans form +am en +Ġb outs +ĠId eal +ĠConstant in +Ġse p +ĠMon arch +att en +ĠPe oples +mod ified +Ġmor atorium +Ġpen chant +Ġoffensive ly +Ġprox ies +ok ane +ĠTaiwan ese +ĠP oo +ĠH OME +us ional +Ġver bs +ĠO man +vis ory +Ġpersu asion +Ġmult it +Ġsc issors +G ay +ow ay +oph ysical +l us +gn u +Ġap ocalyptic +Ġabsurd ity +Ġplay book +Ġautobi ography +I UM +Ġsne aking +ĠSim ulation +pp s +ell ery +Plan et +Ġright fully +Ġn iece +ĠN EC +ĠIP O +ĠDis closure +lean or +ous y +ST ER +Ġ28 2 +Cru z +Ch all +64 3 +ĠSurv ive +ĠF atal +ĠAm id +ap o +We apons +D EN +7 70 +ĠGreen wald +Ġlin en +al os +Ġpollut ants +ĠPCI e +k at +Ġp aw +ĠK raft +C hem +ĠTermin ator +Ġre incarn +Ġ] [ +ĠSe eds +Ġsilhou ette +ĠSt ores +Ġgro oming +ĠD irection +ĠIs abel +ĠBr idges +ðŁ ij +E ED +ĠM orsi +Ġval ves +ĠRank ed +ĠPh arma +ĠOrgan izations +Ġpenet rated +ĠRod ham +ĠProt oss +Ġove rest +Ġex asper +ĠT J +Ġ 000000 +Ġtrick le +Ġbour bon +WH O +Ġw retched +Ġmicrosc opic +Ġcheck list +Ġad orned +R oyal +Ad minist +ĠRet irement +ĠHig hest +We ather +ile ge +Ġincre ments +ĠC osponsors +Ġmas se +ĠS inn +r f +Ġh ordes +as sembly +75 4 +ĠNat asha +ĠTY PE +ĠGEN ERAL +Ġarr anging +Ġ40 7 +l ator +Ġg lean +Ġdisc redited +Ġclin icians +UN E +Ġachie ves +ĠEm erson +com plex += [ +Ġprincip ally +Ġfra il +p icked +Ġthan king +Ġre cl +ĠL AST +Ġsupp ressing +il ic +Ġantidepress ant +ĠLis bon +Ġth or +Ġsp a +Ġking doms +ĠPear ce +em o +Ġpl ung +Ġdiv est +Ġ ******************************** +b is +osp els +ad r +Sp irit +hall a +P ink +end ez +Ġresurrect ed +esc ape +ĠRosen stein +Ġge ological +Ġnecess ities +Ġcarn iv +ĠE lys +ĠBar ney +Ġ29 6 +dig y +ST ON +D OWN +Ġmil estones +Ġk er +Ġdismant ling +Ġre prim +Ġcross ings +19 45 +Ġpatri archy +Ġblasp hemy +Ġ3 59 +met ry +ĠOb esity +ĠDiff erences +bl ocking +ãĥķ ãĤ¡ +ich ita +ĠSab ha +ph alt +ĠCol o +ual a +effic ients +ĠMed ina +con sole +55 7 +ĠHann ibal +ĠHab it +ĠF ever +Ġthen ce +Ġsyn agogue +Ġessential s +Ġw ink +ĠTr ader +ID A +ĠSp oiler +ĠIceland ic +ĠHay ward +Ġpe ac +Ġmal ice +Ġflash back +Ġth w +Ġlay offs +L iquid +Ġtro oper +Ġh inge +ĠRead ers +Ph ill +ĠB auer +Cre ated +Ġaud its +ac compan +Ġunsus pecting +ier a +6666 6666 +Ġbro ch +Ġapprehend ed +ĠM alk +cer ning +ĠCod ex +O VER +M arsh +ĠD eng +ĠExp ression +Ġdisrespect ful +Ġasc ending +t ests +ĠPlaint iff +ster y +ĠAl ibaba +din and +ĠDem psey +Applic ations +mor al +Ġthrough put +Ġquar rel +Ġm ills +Ġhe mor +ĠC ASE +terror ist +st im +ifest yle +ro zen +CE PT +Ar k +u ci +lect ic +Ġirrit ating +she ets +A y +Ġrede emed +Ġhorn y +ĠTe ach +ĠS ear +dem ocracy +4 65 +ĠRest ore +Ġstand by +ĠP is +iff in +Ġsleep y +Ġextr ater +Ġcompl iments +Fram eworks +Ġinstall s +Ġb anging +sur face +found land +Ġmetaph ysical +Ġ28 3 +oul s +dev ices +Ar gs +ĠSac rifice +ĠMcC orm +es on +Cons ervative +ĠM ikhail +see ing +is ively +ĠRo oms +ĠGener ic +Ġenthusi astically +Ġgri pped +Ġcomed ic +ĠElectric ity +Ġgu errilla +Ġdec oration +ĠPerspect ive +Ġconsult ations +Ġun amb +Ġplag iar +Ġmagic ian +Ġe rection +ĠTour ism +or ied +ro xy +11 00 +T am +Ī è +Î ³ +× ª +ĠPred ators +Nit rome +Ġtelesc opes +project s +Ġun protected +Ġst ocked +ĠEnt reprene +nex pected +Ġwast ewater +V ill +Ġint imately +Ġi Cloud +ĠConst able +Ġspo of +Ġne farious +Ġfin s +Ġcens or +ĠMod es +ĠEs per +ar bon +Ġinter sections +Ġlaud ed +Ġphys i +Ġgener ously +ĠThe Nitrome +ĠTheNitrome Fan +Ġar isen +ĠÙ Ī +Ġg lands +ĠPav ilion +ĠGu pta +Ġuniform ly +Ġr amps +ri et +ĠWH EN +ĠVan essa +Ġrout ed +Ġlim p +ĠC PI +p ter +int uitive +Ġv aping +Ġexperiment ed +ĠOlymp us +ĠAm on +Ġsight ing +Ġinfiltr ate +ĠGentle man +Ġsign ings +ĠMe ow +ĠNav igation +che cks +4 33 +Ġel apsed +ĠBulg arian +esp ie +ĠS OM +d uring +Ġsp ills +anc a +ĠPly mouth +M AL +Ġdomest ically +ĠWater gate +ĠF AM +k illed +ed ited +ĠYour self +Ġsynchron ization +ĠPract ices +ST EP +Ġgen omes +ĠQ R +not ice +Ġloc ating +z in +Ġ3 29 +al cohol +Ġk itten +V o +Ġr inse +Ġgrapp le +ĠSc rew +ĠD ul +A IR +Ġle asing +ĠCaf é +Ġro ses +ĠRes pect +Ġmis lead +Ġperfect ed +Ġnud ity +Ġnon partisan +ĠCons umption +Report ing +Ġnu ances +Ġdeduct ible +ĠSh ots +Ġ3 77 +Ġæ ľ +ano oga +Ben ef +ĠB am +ĠS amp +if ix +Ġgal van +ĠMed als +rad ius +Ġno bles +Ġe aves +igr ate +K T +ĠHar bour +u ers +Ġrisk ed +re q +Ġneuro t +get table +ain a +Rom ney +Ġunder pin +Ġlo ft +ĠSub committee +ĠMong ol +b iz +Ġmanif ests +ass isted +ĠG aga +Ġsy nergy +Ġreligious ly +ĠPre f +ĠG erry +T AG +ĠCho i +4 66 +beh ind +ĠO u +Gold Magikarp +Ġhemor rh +R iver +Ġtend on +Ġinj ure +ĠF iona +Ġp ag +Ġag itation +|| || +ur an +ĠE SA +Ġest eem +Ġdod ging +Ġ4 12 +r ss +Ġce ases +ex cluding +Ġint akes +Ġinsert s +Ġemb old +ĠO ral +up uncture +4 11 +ĠUn ified +ĠDe le +Ġfurn ace +ĠCoy otes +ĠBr ach +L abor +Ġhand shake +Ġbru ises +Gr ade +éĹ ĺ +ĠGram my +ile en +St ates +ĠScandinav ian +ĠKard ash +8 66 +Ġeffort lessly +ĠDI RECT +ĠTH EN +ĠMe i +ert ation +19 68 +Ġgro in +w itch +Requ irements +98 5 +Ġroof s +Ġest ates +ĠH F +Ġha ha +Ġdense ly +ĠO CT +Ġpl astics +Ġincident ally +ĠTr acks +ĠTax es +Ġch anted +Ġforce ful +ĠBie ber +ĠK ahn +K ent +ĠC ot +lic ts +F ed +Ġhide ous +ĠVer d +ĠSynd icate +ĠIl legal +J et +ĠD AV +re asonable +c rew +Ġfundamental ist +Ġtruth ful +ĠJ ing +Ġl il +Ġdown ed +Ġen chanted +ĠPolic ies +ĠMcM aster +ĠH are +ides how +Ġpar ams +en cers +gorith m +Ġallow ances +Ġturb ulent +Ġcomplex ities +ĠK T +Ġ3 37 +ĠGen etic +F UN +D oug +t ick +Ġg igs +ument hal +Ġpatriarch al +Ġcal c +, ... +Ġc out +ĠGu an +Ġpath ological +ĠR ivals +Ġunder rated +Ġflu orescent +ĠJ iu +arna ev +ĠQu an +Ġ4 29 +Ġ ਠ+M ario +Con struct +ĠC itation +ĠR acial +ĠR SA +ĠF idel +Ġ3 95 +Person ally +C ause +à » +rad ical +in en +Ġvehement ly +ĠPap a +Ġintern ship +Ġfl akes +ĠRe ck +Luck ily +B ra +20 20 +rav ings +R N +W onder +Ser iously +Ġre usable +Ġpoll uted +ĠP eng +le igh +ind le +Ġcircuit ry +ĠMad onna +ĠB ART +Res idents +att ribute +Phil adelphia +Cl ub +Ġplan ner +Ġfr antically +Ġfaith fully +ĠTerrit ories +ĠL AT +ĠAnders en +an u +ĠP ARK +ĠS ora +i age +ĠPlay offs +ĠG CC +4 27 +Ġab norm +ĠL ever +Ġdisob edience +As ync +ĠShe a +V ert +Ġsk irts +ĠSaw yer +x p +Ġwors ening +Ġsc apego +ĠAng le +oth al +Ġtro ve +ĠSt y +ĠN guyen +mar ine +ide on +Dep ths +Bl og +ĠIll uminati +Ġtract s +Ġorgan ise +Ġo str +F s +Ġlever aging +ĠD aredevil +as ar +Ġl ang +Ġex termin +urs ions +ĠRom o +ãĤ¤ ãĥĪ +Ġcont ended +Ġencounter ing +ĠTable t +ĠAltern ate +sk ill +Ġswe ets +Ġco hesive +cap acity +Ġrep ud +Ġl izard +ro o +Ġpilgr ims +ĠR uff +ĠInstr ument +ĠLog o +uit ous +E H +Ġsales man +Ġank les +L ed +ĠPat ty +ud os +Own er +Ġdiscrep ancies +k j +M U +Ġuncond itional +Dragon Magazine +i ard +O ak +ĠConvers ation +be er +ĠOs aka +D elta +us ky +Ġsecret ion +Ġpl aza +Ġm ing +Ġde pletion +ĠM ous +ĠI TS +ĠH imal +ĠFle ming +Ġcyt ok +ĠH ick +Ġbat ters +ĠInt ellectual +6 75 +é r +IS ION +ĠQu entin +ĠCh apters +ih adi +Ġco aster +WAY S +ĠL izard +ĠY or +and ering +S kin +ha ust +ab by +Ġportray ing +Ġwield ed +d ash +Ġprop onent +Ġr ipple +Ġgrap hene +Ġfly er +Ġrec urrent +Ġdev ils +Ġwater fall +æĺ ¯ +go o +Text Color +Ġtam pering +IV ES +TR UMP +ĠAb el +ĠS AL +ĠHend ricks +ĠLu cius +b ots +Ġ40 96 +IST ORY +Gu est +ĠN X +in ant +Ben z +ĠLoad ed +ĠCle ver +t reatment +Ġta vern +Ġ3 39 +ĠT NT +ific antly +Tem perature +F el +Ġunder world +ĠJud ges +Ġ< + +Ġst ump +Ġoccup ancy +Ġab er +ĠF inder +) ", +ĠN unes +res et +in et +ect omy +Ġwell ness +ĠP eb +quart ered +and an +Ġneg atives +ĠTh iel +ĠCl ip +ĠL TD +Ġbl ight +Ġreperto ire +K yle +Ġqu er +ĠC es +Ġha pl +98 9 +ĠTh ames +isc opal +Des k +ivari ate +ĠEx cellence +found ation +Ġâ ĩ +X i +Ġmyster iously +esty les +Ġper ish +ĠEng els +ĠDE AD +09 0 +}} } +ĠUn real +Ġrest less +ID ES +orth odox +ĠInter mediate +Ġdin ners +ĠTr out +ĠSe ym +ĠHall s +og ged +Ġtraged ies +Ġdid nt +67 6 +Ġail ments +Ġobserv able +ĠV ide +ad apt +ĠD usk +Ġprofessional ism +ĠPres cott +ĠInd ies +p ox +ĠMe hran +W ide +Ġend emic +ĠPar an +B ird +Ġped als +ĠI U +ĠAdam ant +ĠH urt +Ġcorrel ates +urd en +Ġspons oring +cl imate +ĠUnivers ities +ĠK not +enn es +ĠDam ian +ĠAx el +S port +Ġbar b +ĠS no +sh own +ste en +ud ence +Ġnon violent +Ġhom ophobia +Ġbiom ass +ĠDet ail +Ġsrf N +ĠT une +accompan ied +I ENCE +Al bert +ĠMong o +z x +ĠCer berus +or bit +c ens +Ġsl ay +SH ARE +H Y +Ġb rawl +ĠPro be +Ġnonex istent +ĠClare nce +ĠBlack burn +Ġport als +ĠR ita +ĠRem ain +ĠLe vant +Ġtrick ed +ĠF erry +aver ing +ĠStraw berry +ĠAn swers +Ġhorrend ous +ĠA man +Supp lement +ĠT oad +Ġpe eled +Ġman oeuv +ĠU zbek +mond s +ĠH ector +Ġ40 2 +pe es +fix es +Ġd j +Ġres umes +Ġaccount ant +Ġadvers ity +Ġham pered +ĠL arson +Ġd oping +part s +H ur +Ġbe arded +Ġy r +ĠPlug in +å¥ ³ +Ġ/ ** +rol ley +Ġwaters hed +ĠSub mission +if lower +AS C +Ġcho ir +Ġsculpt ures +m A +incre asing +ai i +Ġsne akers +Ġconfront s +ĠEle phant +ĠEl ixir +Ġrec al +ĠT TL +w idget +ĠW ax +ĠGr ayson +Ġha irst +Ġhumili ated +ĠWAR N +app iness +ĠT TC +F uel +Ġpol io +Ġcomplex es +Ġbab e +ĠX IV +P F +). [ +P arts +Ġ4 35 +M eg +ĠY ards +ĠAL P +Ġy ells +Ġprin ces +Ġbull ies +ĠCapital ism +ex empt +FA Q +ĠSp onge +ĠAl a +Ġpleas antly +Ġbu f +Ġden ote +Ġunp ublished +Ġkne eling +asc a +Ġl apse +al ien +99 4 +Ġrefere es +ĠLaw yers +S anta +Ġpuzz ling +ĠProm etheus +ĠPh araoh +ĠDel ay +Ġfacilit ates +ĠC ES +Ġjew els +Ġbook let +ond ing +Ġpolar ization +ĠMor an +ĠSal ad +ĠS OS +ĠAdv ice +PH OTOS +IC AN +iat ures +ex press +ĠWonder land +ĠC ODE +ĠCL ASS +9 75 +Ġg rep +ĠD iesel +ĠGl ac +! ?" +Ġr m +o ine +disc rimination +ĠN urse +m allow +Ġv ortex +ĠCons ortium +Ġlarge Download +stra ight +augh lin +G rad +Ġpublic ized +ĠW aves +ĠRed d +Ġfest ivities +ĠM ane +ar ov +Ġfleet ing +ĠDr unk +ug en +C ele +Ġchromos omes +ĠD OT +-+-+ -+-+ +Ġbus iest +ĠBe aver +Sy rian +ĠK yr +k as +ĠCross Ref +19 50 +76 01 +Ġrepe aling +ĠWin ners +ĠMac ro +ĠD OD +bl ance +S ort +64 1 +Ġmet re +ĠD irk +Ġgo ggles +Ġdraw backs +Ġcomplain ant +Ġauthor izing +Ġantit rust +oper ated +Ġm ah +Ġexagger ation +Am azing +ĠSer aph +Ġha ze +w ow +Ġextingu ished +Ġcan yon +ĠB osh +Ġv ents +Ġsc rape +Cor rect +4 26 +Ġav g +Dem and +ĠâĪ ¼ +Ġmicrobi ota +"} ]," +ĠSt ev +B io +ĠPlan es +Ġsuggest ive +Ġdec ipher +ĠRefuge e +ĠKe jriwal +ĠGreen peace +Ġdecl ass +ĠSound ers +Ġth o +Ġdec rypt +Ġbr ushing +ĠJane iro +ip op +S i +8 77 +ĠGeoff rey +Ġc pu +ĠHaz el +Ġview points +Ġcris py +ĠNot ification +Ġsold er +ĠMod est +ĠHem isphere +Ġcass ette +in cludes +Ġident ifiers +ĠC ALL +in cent +T odd +ĠSwe ep +Ġ3 34 +b oss +Ġsm ir +gin x +Ġtown ship +Ġg rieving +ĠMos que +Net flix +AS ED +ĠMillenn ials +oc om +19 67 +Ġbold ly +s leep +Ġes che +arij uana +Ġsw irl +ĠPen al +Ġneglig ent +ĠStephen son +K ER +ĠZ oro +ris is +Ġlocal ization +ĠSeym our +ĠAng lic +red itation +prot ection +ĠPa ige +Ġo mit +ĠR ousse +ĠT ub +Ġinv itations +t ty +Ġm oss +ph ysical +C redits +Ġan archy +Ġchild care +Ġl ull +ĠM ek +ĠL anguages +lat est +ĠSan ford +Ġus ability +Ġdiff use +ĠD ATA +Ġsp rites +ĠVeget a +ĠProm otion +ãĥ¼ ãĤ¯ +rict ing +z ee +Tur kish +ĠTD s +pro ven +57 1 +Ġsmug glers +707 10 +Ġreform ed +ĠLo is +Ġun fl +ĠWITH OUT +ĠReturn ing +ann ie +ĠTom as +Fr anc +ĠProf it +ĠSER V +ĠR umble +ik uman +es an +Ġt esters +Ġgad get +Ġbrace let +ĠF SA +comp onent +Ġparamed ics +Ġj an +ĠRem em +ĠSk inner +Ġl ov +ĠQu ake +rom a +Ġfl ask +Pr inc +Ġover power +Ġlod ging +ĠK KK +ret te +Ġabsor bs +w rote +Ġ ," +K ings +ĠH ail +ĠFall ing +xt ap +ĠHel ena +ire ns +L arry +Ġpamph let +ĠC PR +G ro +ĠHirosh ima +Ġhol istic +". [ +Ġdet achment +Ġas pire +Ġcompl icit +ĠGreen wood +Ġresp awn +ĠSt upid +ĠFin ished +f al +b ass +Ġab hor +Ġmock ery +ĠFe ast +VID EO +Ġcon sec +ĠHung ry +P ull +ĠH ust +it ance +? ãĢį +) -- +ĠPar allel +con v +4 69 +ha ar +w ant +P aper +m ins +ĠTor o +ĠTR UMP +ĠR ai +D W +ĠW icked +ĠL ep +Ġfun ky +Ġdetrim ent +ios is +ache v +Ġde grade +im ilation +Ġret ard +Ġfrag mentation +Ġcow boy +ĠY PG +ĠH AL +Parent s +ĠS ieg +ĠStra uss +ĠRub ber +× IJ +Fr ag +Ġp t +Ġoption ally +ĠZ IP +ĠTrans cript +ĠD well +88 2 +M erc +ĠM OT +ãĥ¯ ãĥ³ +Ġhun ts +Ġexec utes +In cludes +Ġacid ic +ĠRespons ibility +ĠD umb +we i +And erson +ĠJas per +ight on +abs olutely +Ad ult +Ġpl under +Mor ning +ĠT ours +ĠD ane +Î º +ĠT EST +ĠG ina +Ġcan ine +aw an +Ġsocial ists +ĠS oda +Ġimp etus +ĠSupplement ary +oli ath +ĠKinn ikuman +mitted ly +second s +Ġorganis ers +Ġdocument aries +Vari able +GRE EN +Ġres orts +Ġbr agging +Ġ3 68 +Art ist +w k +bl ers +Un common +ĠRet rieved +Ġhect ares +Ġtox in +r ank +Ġfaith s +ĠG raphic +Ġve c +ĠL IA +Af rican +Ġard ent +end iary +L ake +ĠD OS +cient ious +ĠOk awaru +ĠAll y +ĠTim eline +D ash +ĠI c +contin ue +Ġt idy +Ġinstinct ively +ĠP ossibly +ĠOut door +ĠWould n +Ġl ich +ĠBr ay +ĠA X +Ġà ī +Ġ+ # +\ ' +Direct ory +ab iding +Ġf eral +ic ative +but t +Ġper verse +S alt +Ġwar ped +Ġnin eteen +Ġcabin ets +Ġsrf Attach +ĠSl oan +Ġpower ing +reg ation +F light +se vere +Ġst ren +Ġc og +ap ache +Ġâ Ŀ +Ġcaf eteria +p aces +ĠGrim oire +uton ium +Ġr aining +Ġcir cling +Ġlineback ers +c redit +Ġrep atri +ĠCam den +lic ense +Ġly ric +Ġdescript or +Ġval leys +Ġre q +Ġback stage +ĠPro hibition +ĠK et +Op ening +S ym +æĸ ¹ +Ġserv ings +Ġoverse en +Ġaster oids +ĠMod s +ĠSpr inger +ĠCont ainer +è » +ĠM ens +Ġmult im +Ġfire fighter +pe c +Ġchlor ine +Ð ¼ +end i +Ġsp aring +Ġpolyg amy +ĠR N +ĠP ell +Ġt igers +Ġflash y +ĠMad ame +S word +Ġpref rontal +Ġpre requisite +uc a +Ġw ifi +Ġmiscon ception +Ġharsh ly +ĠStream ing +ot om +ĠGiul iani +foot ed +Ġtub ing +ind ividual +z ek +n uclear +m ol +Ġright ful +49 3 +Ġspecial ization +Ġpassion ately +ĠVel ocity +ĠAv ailability +T enn +Ġl atch +ĠSome body +Ġhel ium +cl aw +Ġdi pping +XX X +Ġinter personal +7 10 +Ġsub ter +Ġbi ologists +ĠLight ing +Ġopt ic +Ġden im +end on +ĠC orm +Ġ3 41 +ĠC oup +Ġfear less +Ġal ot +ĠCliff ord +ĠRun time +ĠProv ision +up dated +lene ck +Ġneur on +Ġgrad ing +ĠC t +sequ ence +in ia +con cept +Ġro aring +ri val +ĠCaucas ian +Ġmon og +key es +Ġappell ate +Ġlia ison +EStream Frame +ĠPl um +! . +Ġsp herical +Ġper ished +Ġbl ot +Ġben ches +Ġ4 11 +Ġpione ered +Ġhur led +Jenn ifer +ĠYose mite +Ch air +Ġreef s +Ġelect or +ĠAnt hem +65 2 +Ġun install +Ġimp ede +Ġbl inking +Ġgot o +Dec re +A ren +Ġstabil ization +ĠDis abled +ĠYanuk ovych +Ġoutlaw ed +ĠVent ura +ten ess +Ġplant ation +Ġy acht +ĠHu awei +Ġsol vent +Ġgr acious +Ġcur iously +Ġcapac itor +Ġc x +ĠRef lex +Ph ys +ĠC f +pt in +cons ervative +Ġinv ocation +c our +F N +ĠNew ly +H our +As ian +ĠLe ading +ĠAer ospace +An ne +Ġpre natal +Ġdeterior ating +H CR +ĠNorm andy +ol ini +ĠAm bro +9 10 +Ġset backs +ĠT RE +Ġs ig +ĠSc ourge +59 7 +79 8 +Game play +Ġm sec +M X +Ġprice y +ĠL LP +aker u +Ġover arching +ĠB ale +Ġworld ly +Cl ark +Ġscen ic +Ġdisl iked +ĠCont rolled +T ickets +ĠE W +ab ies +ĠPl enty +Non etheless +Ġart isan +Trans fer +ĠF amous +Ġinf ield +ble y +Ġunres olved +ĠML A +ãĤ Ĥ +Cor rection +Ġdemocr at +ĠMore no +ro cal +il ings +Ġsail or +Ġr ife +h ung +Ġtrop es +Ġsn atched +ĠL IN +ĠB ib +ES A +ĠPre v +ĠCam el +run time +Ġob noxious +4 37 +Ġsum mers +Ġunexpl ained +ĠWal ters +cal iber +Ġg ull +ĠEnd urance +ä½ ľ +Ġ3 47 +Ir ish +Ġaer obic +Ġcr amped +ĠHon olulu +à © +us erc +ec ast +AC Y +ĠQu ery +ãĤ¹ ãĥĪ +Bet a +Ġsuscept ibility +ĠSh iv +ĠLim baugh +Ġà ĸ +ĠN XT +ĠM uss +ĠBrit ons +ES CO +EG IN +Ġ% % +Ġsec ession +ĠPat ron +ĠLu a +n aires +ĠJPM organ +us b +ocy te +Ġcouncill ors +ĠLi ang +f arm +Ġnerv ously +Ġattract iveness +ĠK ov +j ump +Pl ot +Ġst ains +ĠStat ue +ĠApost les +he ter +ĠSUP PORT +Ġoverwhel m +Y ES +Ġ29 1 +d ensity +Ġtra pping +M it +Ġf ide +ĠPam ela +atl antic +Dam n +Ġp ts +OP A +Ġserv icing +Ġoverfl owing +ul o +ĠE rit +t icket +light ing +ĠH mm +ãĥ¼ ãĥ« +im oto +Ġchuck le +4 23 +ãģ ķ +sh ape +Ġque ues +Ġanch ors +ãĤ¼ ãĤ¦ãĤ¹ +F er +Ġaw oke +Ġ6 66 +h ands +Ġdiver gence +Ġ50 5 +T ips +Ġdep ot +Ġske w +ĠDel iver +op ot +Ġdiv ul +ĠE B +uns igned +ĠUn i +X box +Ġfor ks +Ġ7 02 +å ¯ +Ġpromot ers +ĠV apor +Ġlev ied +sl ot +Ġpig ment +Ġcyl inders +C RE +Ġsn atch +Ġperpet ually +Ġl icking +ĠFe et +ĠKra ken +ĠHold en +ĠCLS ID +m r +Ġproject or +Ġden otes +Ġchap el +ĠTor rent +b ler +R oute +ĠDef endant +ĠPublisher s +ĠM ales +ĠInn ov +ĠAg ility +rit er +ty mology +st ores +L ind +Ġf olly +ĠZur ich +B le +Ġnurt ure +Ġcoast line +uch in +D omin +Ġfri vol +ĠCons olid +res ults +M J +Ġphyl ogen +Ġha uled +ĠW iley +ĠJess ie +ĠPrep are +ĠE ps +Ġtreasure r +I AS +Ġcolon ists +Ġin und +ĠWW F +ĠCon verted +6 000 +out side +ĠApp earance +ĠRel ic +ĠM ister +s aw +Ġresult ant +Ġadject ive +ĠLaure l +ĠHind i +b da +Pe ace +Ġreb irth +Ġmembr anes +Ġforward ing +Ġcoll ided +ĠCar olyn +K ansas +5 99 +ĠSolid GoldMagikarp +Be ck +Ġstress ing +ĠGo o +ĠCooper ative +Ġf s +ĠAr chie +L iter +ĠK lopp +J erry +Ġfoot wear +War ren +Ġsc ree +h are +Under standing +P ed +Ġanth ology +ĠAnn ounce +M ega +Ġflu ent +Ġbond age +ĠDisc ount +il ial +C art +ĠNight mares +Sh am +ĠB oll +uss ie +H ttp +Atl anta +Ġun recogn +ĠB id +Ġunder grad +Ġforg iving +ĠGl over +AAAA AAAA +4 45 +V G +pa io +kill ers +Ġrespons ibly +Ġmobil ize +Ġeffect ed +ĠL umin +Ġk ale +Ġinfring ing +ann ounced +Ġf itt +b atch +ĠT ackle +ĠL ime +ĠAP P +uke mia +Ġrub y +Ġex oner +ĠCas ual +0 70 +Ġpel vic +Ġautom ate +ĠK ear +ĠCoast al +Ġcre ed +Ġbored om +ĠSt un +ri ott +Ĥ İ +Ġregener ate +Ġcomed ians +ĠOP ER +Sp ons +id ium +on is +L ocated +05 7 +Ġsusp ense +ĠD ating +C ass +Ġneoc ons +ĠShin zo +Ġaw oken +ch rist +ĠMess ages +att led +ĠSpr ay +ĠSp ice +C W +Ġshield ing +ĠG aul +Am id +Ġparam ilitary +Ġmult if +ĠTan ner +il k +Ġgodd amn +g ements +Ġbe friend +m obi +Ġ3 88 +fold er +acc a +Ġins in +g ap +N ev +fif th +Ġpsychiat ry +b anks +TH IS +Ġhar b +ac qu +Ġfac ade +ĠPower Point +80 3 +Ġbl uff +Sh ares +Ġfavor ing +El izabeth +Ãį Ãį +Ġr anger +77 2 +ĠAr che +h ak +ĠGen etics +ĠF EMA +Ġev olves +Ġest e +ĠP ets +ĠM é +ĠInterest ing +ĠCanter bury +ch apter +ĠStar fleet +Sp anish +Ġdraw back +ĠNor wich +9 70 +n orth +ag anda +Ġtransform ative +ram ids +bi ology +ad ay +Ġpropag ation +ĠGam ma +ĠDen ise +ĠCalcul ator +ent imes +ĠB ett +Ġapp endix +ĠHD D +AK ING +Ġst igmat +Ġhol ster +Ġord inarily +Ch ance +ĠCont rary +Ġad hesive +Ġgather s +6 12 +re au +ony ms +ew ays +Ġindu ces +Ġinterchange able +se m +Wh it +Ġtr ance +Ġincorpor ation +ĠExt ras +Fin ancial +Ġawkward ly +ĠStur geon +ĠH Y +Norm ally +ĠEnd ing +ĠAss ist +enc rypted +Ġsub jug +Ġn os +Ġfan atic +C ub +C U +?" . +Ġirre versible +å Ĥ +03 1 +ĠH AR +sp read +ul ia += $ +Sc ope +L ots +Ġlif estyles +ol on +Ġf eds +Ġcongrat ulate +web kit +Ġindist inguishable +ĠSw ing +Ġcommand ments +qu ila +ab ella +m ethyl +ann abin +Ġo vere +Ġlob ster +ĠQU EST +ĠCONT IN +bern atorial +:::: :::: +ĠTra ve +ĠSam oa +AN I +75 2 +Ð ´ +userc ontent +ĠMod erate +y eah +ĠK itt +Ġwe e +Ġstuff ing +ĠInter vention +ĠD ign +Ġware houses +ĠF iji +Ġpel lets +Ġtake away +ĠT ABLE +ĠClass ical +col lection +Ġland fall +ĠMus cle +Ġsett les +ĠAD V +Ġ3 44 +L aura +Ġf ared +ĠPart ial +4 36 +oss ibility +ĠD aly +ĠT arant +ĠFu ji +am l +c ence +55 1 +ĠProced ures +ĠO CD +ĠU D +t in +Q UI +ach o +4 38 +Ġgl itches +Ġenchant ment +Ġcalcul ates +IR O +ĠH ua +alys es +ĠL ift +um o +Ġle apt +Ġhypothes ized +ĠGust av +it ans +VERS ION +æ ł +Rog er +Ġr and +ĠAd apter +Ġ3 31 +ĠPet ition +k ies +M ars +Ġunder cut +ze es +ĠLy ons +ĠDH CP +Miss ing +Ġretire es +Ġins idious +el i +> ) +. ãĢį +Ġfinal ists +ĠA ure +Ġacc user +Ġwas tes +ĠY s +ĠL ori +Ġconstitu encies +Ġsupp er +Ġmay hem +or ange +Ġmis placed +Ġmanager ial +Ġex ce +ĠCL I +Ġprim al +ĠL ent +Cry stal +h over +ĠN TS +end um +Ġd w +ĠAl c +n ostic +Ġpres erves +ĠTs arnaev +Ġtri pled +rel ative +Arc ade +k illing +ĠW EEK +ĠH anna +D ust +Com pleted +ģ « +Ġappro ves +ĠSur f +ĠLuther an +ven ants +Ġrobber ies +we ights +soft ware +at ana +ug al +Ġgrav y +ĠC ance +OLOG Y +ly ak +Ton ight +Ġunve il +Ġ19 04 +ĠMin ion +ent ious +st ice +pack ages +ĠG EAR +Ġg ol +ĠHutch inson +ĠProf ession +ĠG UN +ĠDiff erence +ĠTsuk uyomi +ĠLes bian +6 70 +Ġfug itive +ĠPlan etary +-------------------------------- ------------------------ +Ġacc rued +Ġch icks +Ġsto pp +Ġblock ers +C od +Ġcomment ers +ĠSomew here +ĠPhot ographer +the me +Ġmay oral +w u +Ġanten nas +Ġrev amped +ĠSubject s +it é +im ura +Ġentr ances +liter ally +Ġten ets +ĠO MG +ĠMP H +ĠDon key +ĠOff ense +Ġ" + +Sn ap +ĠAF B +Ġan imate +ĠS od +His panic +Ġinconsist ency +D b +F Y +Ex port +Ġa pe +Ġpear l +ib el +ĠPAC s +Ġ{ \ +Ġact u +ĠHS BC +camp us +Ġpay off +Ġde ities +ĠN ato +ou ple +Ġcens ored +ĠCl ojure +Ġconf ounding +en i +Ġreck on +op he +Ġspot ting +Ġsign ifies +Ġprop el +Ġfest ive +S uggest +Ġpled ging +ĠB erman +Ġrebell ious +Ġovershadow ed +Ġinfiltr ated +j obs +67 2 +Ġscal able +Ġdomin ion +ĠNew foundland +ĠMead ow +Ġpart itions +AM I +Ġsupplement ary +str ument +Ġhair y +Ġperpet uate +Ġnuts hell +ĠPot ato +ĠHob bit +Ġcur ses +Flo at +Ġquiet er +Ġfuel ing +Ġcaps ules +ĠL ust +ĠH aunted +Exec utive +Ġchild birth +G re +Ġrad iant +å İ +Ġm alls +Ġin ept +ĠWarrant y +Ġspect ator +E h +t hens +Ġculmin ating +æ © +ary a +ãĤ ® +ilit arian +ĠOR IG +ĠSp ending +pt ives +ĠS iren +ĠRec ording +ay ne +Ġv im +Ġspr ang +T ang +ĠM FT +mor ning +ĠWe ed +m peg +cess ion +ĠCh ung +7 30 +w arning +56 2 +handed ly +P oor +P olitics +: # +Ġp ian +Ġfec es +ĠDocument ation +Ġban ished +Ġ3 99 +ĠAR C +Ġhe inous +J ake +ĠAm ir +way ne +v re +os henko +Ġnotebook s +Ġfound ational +Ġmarvel ous +ixt ape +Ġwithdraw als +Ġh orde +ĠD habi +is able +ĠK D +Ġcontag ious +ĠD ip +ĠAr rows +Ġpronoun s +Ġmorph ine +ĠB US +68 2 +Ġk osher +fin ished +ĠInstr uments +Ġf used +yd en +ĠSal mon +F ab +aff ected +K EN +C ENT +Dom ain +Ġpoke mon +ĠDr inking +G rowing +ĠInvestig ative +ĠA ether +em i +Ġtabl oid +Ġrep ro +ĠNot withstanding +ĠBers erker +Ġdram as +Ġclich é +Ġb ung +ĠU RI +ĠD os +0 44 +Ġpast ors +Ġl s +Ġac rylic +aun ts +Ed ward +Ġmajor ities +B ang +Ġfield ing +ĠRepl acement +ĠAl chemy +pp ard +ĠRome o +ĠSan ct +ĠLav rov +ib ble +Inst ruct +Ġimp ractical +ĠPlay boy +ce phal +Ġsw aps +Ġk an +ĠThe o +Ġillust rating +Ġdismant led +ĠTrans gender +ĠG uth +UG H +Ġtriumph ant +Ġencomp ass +Ġbook mark +udd in +j er +Ġpred icate +ES H +Ġwhen ce +ĠAB E +Ġnon profits +Se qu +Ġdi abetic +Ġp end +Ġheart felt +sh i +Ġinter acts +ĠTele com +Ġbombard ment +dep ending +ĠLow ry +ĠAd mission +ĠBl ooming +ust ration +ene gger +B rew +Ġmol ten +ĠNer d +P IN +âĸ Ģ +ave ment +Ġtou red +Ġco efficients +ĠTray von +ans son +Ġsand y +t old +fl ows +Ġpop ulous +ĠT inder +ĠBl iss +R achel +Min imum +Ġcontest ant +ĠRed uce +ĠMor se +ĠGrass ley +ĠClick er +Ġexp r +Ġs incerity +Ġmar qu +Ġelic it +ĠPro position +ĠDemon ic +Ġtac os +G reek +Ġpost war +Ġin sofar +ĠP ork +Ġ35 2 +doctor al +walk ing +Ġmid term +ĠSam my +sight ed +ĠTR ANS +ic i +AL D +ĠUS L +ĠF ISA +ĠAm pl +ĠAlex andra +ine lli +Tr ain +Ġsign ify +ĠVers us +Ġob fusc +Ġk h +Ġagg ro +ĠRen ault +Ġ3 48 +5 18 +ox icity +0 22 +ĠTw ist +Ġgoof y +D ynamic +Ġbrief ings +m ight +8 99 +Ġderog atory +T ro +Ġfor ging +ĠKor an +ĠMar ried +ĠBuc s +Ġpal ate +ĠCon version +m able +4 13 +Ġ( _ +Ġs iph +ĠN EO +col lege +Ġmarg inally +Ġfl irt +ĠTra ps +ĠP ace +é »Ĵ +Ġgoalt ender +Ġforb ids +Ġcler ks +ĠT ant +ĠRobb ins +ĠPrint ing +Ġpremie red +Ġmagn ification +ĠT G +ĠR ouse +ĠM ock +odynam ics +Ġpre clude +ism o +ĠPul itzer +Ġaval anche +ĠK odi +rib une +ĠL ena +Elect ric +Ġref inery +Ġend owed +Ġcounsel ors +Ġd olphin +ĠM ith +Ġarm oured +hib ited +Beg in +ĠP W +O il +ĠV or +ĠShar if +ĠFraz ier +est ate +Ġj ams +Pro xy +Ġband its +ĠPresbyter ian +ĠPrem iere +t iny +ĠCru el +Test ing +Ġhom er +ĠV ERS +ĠPro l +ĠDep osit +ĠCoff in +Ġsemin ars +Ġs ql +ĠDef endants +Altern atively +ĠR ats +ç « +ethy st +' > +Ġiss uer +58 9 +Ġch aired +ĠAccess ories +man ent +Ġmar row +ĠPrim ordial +C N +Ġlimit less +ĠCarn age +Ġund rafted +q v +IN ESS +on ew +Ġco hesion +98 7 +Ġne cks +Ġfootball er +ĠG ER +Ġdetect able +ĠSupport ing +ĠCS V +oc ally +k Hz +Ġund e +Ġsh one +Ġbud ding +tra k +Stand ing +ĠStar craft +ĠKem p +Ben ch +Ġthw arted +ĠGround s +ath i +L isa +Dial og +ĠS X +V ision +Ġingen ious +Ù IJ +Ġfost ering +ĠZ a +ĠIn gram +Ġ" @ +N aturally +6 16 +0 35 +ĠF AC +H mm +55 4 +Ġacceler ator +ĠV end +Ġsun screen +Ġtuber culosis +rav iolet +ĠFunction al +ĠEr rors +ed ar +19 66 +ĠSpect re +ĠRec ipes +88 5 +ĠM ankind +L iverpool +Ġ| -- +Ġsubst itutes +ĠX T +w ired +Ġinc o +ĠAf gh +E va +ic c +S ong +K night +Ġdilig ently +ĠBroad cast +A id +Ġaf ar +ĠH MS +aton in +ĠGr ateful +Ġfire place +ĠOm ni +e uro +ĠF RE +ĠSh ib +ĠDig est +t oggle +Ġheads ets +Ġdiff usion +ĠSqu irrel +ĠF N +Ġdark ened +out her +Ġsleep s +ĠX er +gun s +Ġset ups +Ġpars ed +Ġmamm oth +ĠCur ious +g ob +ĠFitz patrick +ĠEm il +im ov +........ ..... +ĠB enny +Second ly +Ġheart y +Ġcons on +st ained +Ġgal actic +cl ave +Ġplummet ed +Ġp ests +Ġsw at +Ġrefer rals +ĠLion el +h oly +Ġunder dog +ĠSl ater +ĠProv ide +ĠAm ar +ress or +å Į +ong a +Ġtim id +Ġp iety +ĠD ek +Ġsur ging +az o +Ġ6 10 +Ġdes ks +ĠSp okane +ĠAn field +Ġwars hips +ĠCob ra +Ġar ming +clus ively +ĠBad ge +ag ascar +ĠPR ESS +ĠMcK enzie +ĠFer dinand +burn ing +Af ee +Ġtyr ann +ĠI w +ĠBo one +100 7 +ĠRe pt +Ċ Âł +Ġcar avan +ĠD ill +ĠBundes liga +Ch uck +Ġheal er +ãĥ¼ãĥ Ĩ +ĠH obby +Ġneg ate +Ġcrit iques +section al +mop olitan +Ġd x +Ġouts ourcing +ĠC ipher +t ap +Sh arp +Ġup beat +Ġhang ar +Ġcru ising +ĠNi agara +Ġ3 42 +ill us +ĠS v +Ġsubt itles +Ġsqu ared +Ġbook store +Ġrevolution aries +ĠCarl ton +ab al +Ut ah +Ġdesp ise +ĠU M +cons ider +aid o +Ġc arts +ĠT urtles +Tr aining +Ġhonor ary + ¢ +Ġtri angles +4 22 +Ġreprint ed +Ġgrace ful +ĠMong olia +Ġdisrupt ions +ĠB oh +Ġ3 49 +Ġdr ains +Ġcons ulate +Ġb ends +Ġm afia +ur on +ĠF ulton +m isc +Ġren al +Ġin action +ck ing +Ġphot ons +Ġbru ised +ĠC odes +og i +Ġn ests +ĠLove ly +ĠLib re +ĠD aryl +Ġ# ## +S ys +. ," +Ġfree zes +est ablishment +and owski +Ġcum bers +ĠSt arg +ĠBom bs +Ġleg ions +Ġhand writing +Ġgr un +ĠC ah +sequ ent +Ġm oth +ĠMS M +Ins ert +F if +Ġmot el +Ġdex ter +ĠB ild +hearted ly +Ġpro pe +ĠText ure +ĠJ unction +ynt hesis +oc ard +ĠVer a +ĠBar th +Ġμ g +Ġl ashed +Ġ35 1 +ĠZ amb +ĠSt aples +ĠCort ex +ĠCork er +Ġcontinu um +ĠWR ITE +unt a +rid or +Ġde ems +0 33 +ĠG OLD +p as +Ġrep ressive +ãĥĨ ãĤ£ +Ġbaff led +Sc ar +Ġc rave +Ġ ______ +Ġentrepreneurs hip +ĠDirector ate +Ġ' [ +Ġv ines +Ġasc ended +ĠGR OUP +ĠGood bye +Ġdo gged +ãĥ´ ãĤ¡ +Man ufact +Ġunimagin able +ri ots +ier rez +Ġrel ativity +ĠCraft ing +ra ught +ud en +c ookie +Ġassass ins +Ġdissatisf ied +ac ci +Ġcondu it +Sp read +ĠR ican +n ice +izz le +Ġsc ares +ĠWH Y +ph ans +5 35 +Ġprot racted +ĠKrist en +5 36 +ĠSc rib +ĠNe h +Ġtwent ies +Ġpredic ament +Ġhandc uffs +Ġfruit ful +ĠU L +ĠLud wig +Ġatt est +ĠBre aker +Ġbi ologically +ĠDeal er +Ġrenov ations +f w +ess en +Al ice +ĠHen ri +Ġun ilaterally +ĠS idd +h ai +ĠSt retch +S ales +Ġcumbers ome +ĠJ avier +Ġtrend y +Ġrot ting +ĠChall enges +Ġscra ps +Ġfac ets +ĠVer onica +ĠVer ge +ĠS ana +Al ien +ĠR ih +Ġrad ial +ect ar +Ġ6 30 +cl i +Mar ie +Ġwild fire +ĠCat o +h ander +Ġwait ress +Ġch ops +ĠS ECTION +Ġblunt ly +ĠCat alog +n ian +stud y +Ġpat rolling +ĠT enth +nex us +ĠN ON +op sy +Ġsc athing +s ie +Ġdeterior ated +V B +Naz is +Ġdep ictions +Ġauthent icated +ĠCon ce +k rit +Ġpromul g +ĠL ONG +U FC +ĠVis itors +ĠRec all +Ġrehab ilit +ĠSL I +Ġglac ier +ĠB ite +Ġ50 3 +Ġvom it +Ġfer mented +ĠKh alid +Ġgrad ed +ĠMag icka +ĠIch igo +power ful +ic ators +75 3 +Ġsh rew +Ġ35 6 +Ġlegal izing +Ġall otted +ĠArch demon +ith ing +igg urat +V OL +Le od +Ġo ily +Ġindu cing +Ġamy gdala +Ġadm ins +ĠAcqu isition +C AN +Ġsche matic +Ġmo an +ĠCamer oon +Ġt ink +Ġmer ry +Ġbutter flies +ĠGo ff +Ġworks pace +ĠCor ona +Ġj avascript +ĠD olphin +ĠCant or +4 64 +to e +AP S +ĠAg ing +Ġpadd ed +ĠZ heng +ĠHe ld +Ġest ranged +Ġ7 70 +. } +ĠDun ham +Ġsm okes +Ġcap itals +und ai +Sh in +ĠFound ing +Ġent itle +Ġcenter piece +D iscover +Ġthere to +al ert +ĠN ou +ĠAnaly st +l c +F H +FI ELD +ĠP OV +gr ay +Ġar cs +ĠH OT +Ġr s +Ġoblig atory +ĠArchitect s +ĠS ven +ĠF EC +0 200 +Christ mas +ĠAlban ia +rat om +58 7 +Ġhard ships +Ġaut os +ĠCharg es +Ġap es +Ġ3 76 +wal let +Ġintox ication +Ġgobl in +Ġ5 70 +++++++++ ++++++++ +ĠYel p +ĠMag netic +ĠBr iggs +R ail +Ġspawn s +ĠW iggins +Ġshowc ased +Ġres orted +ub en +Ġwh ipping +Ġim itate +Ġdigest ion +ĠUS PS +ĠG est +Ġye a +ĠT ight +ind al +ic as +` . +C AST +'' ; +ĠF et +opath ic +In valid +Ġregrett ed +Ġbro ccoli +ĠSc ores +e ve +Ġpost ings +Ġaccum ulating +Ġneed less +elf th +Ġmay ors +Ġsc rib +Ġanecd otes +Ġbot ched +ĠRib bon +ĠConstant ine +i uses +ess es +Ġdev ise +Comp ared +Ġp udding +Ġg arg +Ġev oke +79 7 +Ġdet ox +9 09 +ĠPie ces +ĠMcC artney +Ġmet ast +ĠK rypt +P OR +Ġt ending +ĠMerch ants +Pro of +ĠV arg +ĠPort able +ãĥ¼ãĥĨ ãĤ£ +B rain +25 00 +Ġfol iage +Ø ¹ +Ġment ors +ĠA ires +Ġminimal ist +Ġing ested +ĠTro jan +ĠQ ian +inv olved +0 27 +Ġer oded +RA FT +Ġbl urry +M ob +Ġbuff et +ĠFn atic +ae a +KN OWN +ĠIn it +s afety +en um +ACT ION +ĠCrus her +ĠD ates +Ġ ................ +c alling +ak ov +Ġvent ured +Ġ5 55 +au ga +H art +ĠA ero +M AC +Ġthin ly +Ġar ra +ST ATE +ild e +ĠJac qu +ĠFem ales +Ġthe orem +Ġ3 46 +Ġsmart est +ĠPU BLIC +ĠK ron +ĠB its +ĠV essel +ĠTele phone +Ġdec ap +Ġadj unct +ĠS EN +mer ga +Ġred acted +Ġpre historic +Ġexplan atory +ĠRun s +ĠUtt ar +ĠM anny +ĠAUTH OR +ĠUnle ashed +ĠBow ling +be ans +79 3 +Ġunivers es +Ġsens it +ĠK ung +re peat +ctr l +Ġp aced +Ġfull er +Cl ock +Ġrec omb +ĠF aul +ĠB unker +Ġpool ed +Ġan a +ĠM outh +LL OW +hum ane +Ġbull do +ĠMicha els +f am +Ġwreck ed +Ġport rays +ĠWh ale +ĠH es +Ġguess es +ĠBrow se +ĠL APD +Ġconsequ ential +ĠInn ocent +ĠD RAG +Ġtrans gress +ĠO aks +Ġtri via +ĠRes on +ĠA DS +-- + +ĠT oll +Ġgrasp ing +ĠTHE M +ĠT ags +ĠCon clusion +Ġpract icable +Ġho op +Ġunintention ally +Ġign ite +ĠM ov +ur ized +le hem +Ter min +Ġcolour ful +ĠLin ear +ĠEll ie +G y +Ġman power +Ġj s +Ġem oji +ĠSHAR ES +_ . +0000 7 +Ġsophistic ation +Ġunders core +Ġpract ise +Ġbl ob +op ens +Uk raine +Ke eping +Y C +J R +ult imate +Cl aim +Ġautom obiles +99 3 +ste el +Ġpart ing +ĠL ank +... ? +Ġ38 5 +Ġremem brance +Ġe ased +Ġcov ari +ĠS ind +Effect ive +Ġdisse mination +ĠMo ose +ĠCl apper +br ates +App ly +Ġinv is +Ġwors ened +âĢĶ - +Ġlegisl ator +ĠL ol +ĠRow e +Ġdealers hip +um ar +id ences +Ġinvestig ates +Ġc ascade +Ġbid der +ĠB EN +Iron ically +Ġpres iding +Ġd ing +Ġcontrad icted +Ġshut s +ĠF IX +Ġ3 66 +Dist rict +Ġsin ful +ĠChar isma +o ops +Ġtot ality +Ġrest itution +ĠOpt imus +ĠD ah +Ġcl ueless +urn ed +Ġnut rit +Ġland owners +Ġfl ushed +Ġbroad en +m ie +Ġprint ln +Ġn ig +ĠCorp us +J en +Ġprot o +ĠWik imedia +ĠPal o +C OR +Ġstory lines +Ġevangel icals +ĠDar rell +Ġrot or +ĠH W +sk illed +ery l +Ġbe gg +ĠBl umenthal +Ġwe aving +Ġdown wards +ĠJack et +ĠANG EL +Te chnology +Ġes oteric +alde hyde +Ġfur iously +Ġforeign er +We ak +CH O +ĠH ound +Exper ience +ĠPlay station +ĠM IA +ĠU ng +cl oth +ag all +Ġcal ming +iz ens +St ruct +ĠW itches +ĠCeleb ration +Ġ........ ...... +pt roller +ĠTC U +Ġb unny +ãĥ į +ut orial +Ġup scale +ĠSt a +ĠCol ossus +Ġchlor ide +ĠZ ac +ĠRe asons +ĠBrook ings +ĠWH ITE +][ / +ĠL ose +9 05 +Ġunders ide +ern els +Ġv ape +do zen +upp et +ĠST OP +mat ical +ĠStat ements +hed dar +P AC +Custom er +Ġmem os +ĠP J +end ars +ĠLim its +l augh +Ġstabil ized +ĠALE C +Y A +Up grade +al am +Ġtechn o +Ġan ew +fore seen +Ġcolleg iate +ĠPy ro +ĠD ism +Ġfront line +Ġammon ia +I U +Qu ite +John ny +ass in +G OP +ĠSt yles +ĠSovere ign +acter ial +5 49 +ĠR IP +ĠL ists +Ġ3 64 +ĠRece p +s ocket +ĠByr d +ĠCand le +An cient +Ġappell ant +en forcement +ace a +ans ki +Ġold s +88 6 +Ġsl urs +Ġem pires +Ġbuck le +Ġalien ation +ĠAber deen +Ġunic orn +Ġoverr iding +ĠL X +pp a +Ġdesp ised +ĠB ugs +ĠB ST +S outhern +5 33 +Ġhall mark +ĠPost er +Ġstem med +Ġprincip als +ĠT ECH +ĠSand wich +It aly +Ġche esy +ĠSet TextColor +ĠProt ective +ĠC ohn +J O +apt op +Re ason +Lead er +ĠUnder stand +ĠFr idays +ĠContin uous +Ġcl ipping +ĠR ye +Ġber th +tim er +ann is +re act +Ġbuff alo +ĠPar as +Ġ6 55 +Ġpres ided +ĠSun rise +Ġve ts +Ġcl oves +ĠMcC ull +Stre ngth +G AN +Ġill iter +ĠPric ing +l é +Ġresist or +Ġbr un +ĠSuff olk +Ñ ĭ +ĠL iver +Re leased +Ġwhat s +8 60 +ĠMe asures +Ġden ouncing +ĠRy zen +Ġsou ven +Ġcareg ivers +ch ini +ĠScar lett +Ġt rough +Cong ratulations +Ġtax is +ĠTrad ition +j it +Ġtable top +Ġhither to +Ġdis information +off ensive +h ra +ĠDISTR ICT +Ġcompl icate +chen ko +ĠRecon struction +Ġpalp able +Ġa usp +Ġ4 28 +Ġshowc ases +ĠPublic ation +know ledge +inn on +4 19 +Ġretri eval +and ers +Ġref ute +Ġinqu ired +g ur +Ġneg ativity +Ġcons erve +Ġafter life +Ġpres upp +ĠGill espie +Ġm t +ĠD N +T ap +Ġper pend +ĠS my +does n +Ġsp illing +Ġhyp ers +K ate +® , +ke pt +ĠP owered +Ġj a +ĠK lux +ard e +ab an +Ġ4 44 +Ġflatt ened +ĠImprove ments +urg a +ĠK und +Ġins cribed +Ġfac ult +Ġunpre pared +ĠCons umers +Ġsatisf ies +Ġpul monary +Ġinf iltration +Ġex ternally +Ġcongrat ulations +ag han +Ġair liner +Ġfl ung +Ġfly ers +G D +Ġsnipp ets +Ġrec ursive +Ġmaster ing +L ex +Ġovert ly +v g +Ġluck ily +Ġenc ro +ĠLanc et +ĠAbyss al +function al +Ġs ow +Ġsqu id +Ġnar ration +Ġn aughty +ĠHon our +ĠSpart ans +Ġsh atter +ĠTac oma +ĠCal ories +ĠR aces +Sub mit +Ġpurpose fully +w av +ĠY ok +F est +ĠG err +Met ro +Ġit iner +f amous +Ġ" { +in line +was her +Iss ue +ĠCL IENT +oz o +Vers ions +7 25 +ĠGl ock +Ġshield ed +ĠPC R +ENC Y +ĠWe ld +ĠSim pl +Ġredirect ed +ĠK ham +Ġ( > +Ġlab ou +Ġdi apers +ss l +Ġcell ar +organ isms +ore sc +ĠBer ks +did n +Sh ipping +C hest +Ġund one +Ġmillion aire +Ġc ords +ĠYoung er +appropri ately +Ġsequ els +u ve +ant icipated +Ġle wd +ĠSh irt +ĠDmit ry +V eter +Ġsl aying +ĠY ar +Ġcompl ication +I owa +ĠEric a +ĠBL M +g irlfriend +b odied +6 26 +19 63 +Ġintermedi ary +Ġcons olation +M ask +ĠSi em +ow an +Beg inning +Ġfix me +Ġculmin ated +Ġcon duc +ĠVolunte er +Ġpos itional +Ġgre ets +ĠDefin itions +Ġthink er +Ġingen uity +Ġfresh men +ĠMom ents +Ġ35 7 +ate urs +ĠFed Ex +s g +69 4 +Ġdwind ling +ĠBO X +sel age +Ġt mp +Ġst en +ĠS ut +Ġneighbourhood s +Ġclass mate +f ledged +Ġleft ists +Ġclim ates +ATH ER +ĠScy the +ul iffe +Ġs ag +Ġho pped +ĠF t +ĠE ck +ĠC K +ĠDo omsday +k ids +Ġgas ped +Ġmon iker +ĠL od +ĠC FL +t ions +r ums +fol ios +Ġm d +Ġunc anny +Ġtrans ports +ĠLab rador +Ġrail ways +Ġappl iance +ĠCTR L +æ Ģ +Pop ulation +ĠConfeder acy +Ġunb earable +Ġdors al +ĠIn form +op ted +ĠK ILL +Mar x +Ġhypoc ritical +q us +ĠN umerous +ĠGeorg ian +ĠAmbro se +ĠL och +Ġgu bernatorial +ĠX eon +ĠSupp orts +ens er +ee ly +ĠAven ger +19 65 +Ar my +Ġju xtap +Ġcho pping +ĠSpl ash +ĠS ustainable +ĠFin ch +Ġ18 61 +ict ive +at meal +ĠG ohan +Ġlights aber +ĠG PA +ug u +ĠRE PL +vari able +Ġher pes +Ġdesert s +ac iously +Ġsitu ational +week ly +ob l +Ġtext ile +ĠCorn wall +Ġcontrace ptives +ĠA ke +] - +ä¹ ĭ +: , +ĠW em +ĠB ihar +Ġ' . +Ġbe re +Ġanal ogue +ĠCook ies +Ġtake off +Whe el +Ġmaj estic +Ġcomm uting +0 23 +ĠCor pse +ass ment +min i +Ġgor illa +ĠAl as +ere e +Ġacquaint ances +ĠAd vantage +Ġspirit ually +Ġey ed +pm wiki +ĠE nder +Ġtrans lucent +Ġnight time +ĠIM AGES +5 45 +ĠK amp +ĠFre ak +Ġ ig +Port land +4 32 +ĠM ata +Ġmar ines +Ġh ors +ater asu +ĠAtt ribution +Ġ-------- - +Ġk ins +ĠBEL OW +++ + +Ġre eling +ol ed +Ġcl utter +ĠRel ative +Ġ4 27 +B US +Ġa vert +ĠChe ong +ĠA ble +ĠPry or +Develop er +Ġen cyclopedia +ĠUSA F +ĠG arry +Sp ain +Bl ocks +Ġexp osition +ĠGamer Gate +W OR +Ġstockp ile +Ġclot hed +ĠT one +ĠR ue +t umblr +Ġtreacher ous +Ġf rying +Ñ Į +ĠS ph +Ġrest raints +Ġemb odies +ĠG es +S afety +Ġnegoti ators +min ing +ĠAppalach ian +L OS +ĠJenn a +Ġpass ers +ç ĭ +sn ap +Ġshort en +creat or +Ġinn umerable +uther land +67 4 +ĠW OM +ĠAs cend +ĠArm ory +ĠTrans action +K ick +Ġsuit case +day Name +Ġwaste ful +mar riage +ĠMcC abe +ite ch +ĠO ss +Cl osure +ĠTreasure r +Ġindec ent +ĠD ull +Ġresid ences +19 59 +ĠS ettlement +Ham ilton +Ġself ies +ĠRank ing +ĠBark ley +ĠB ore +ĠW CS +ĠMar itime +ĠH uh +ĠForest ry +Ġcultiv ating +ĠBall ard +Ġg arrison +ĠSD L +9 30 +Ġnas cent +Ġirresist ible +Ġaw fully +\/ \/ +Ġequ ate +Ġanthrop ology +ĠSylv ia +Ġintest ine +Ġinnoc uous +cess ive +ag ra +ĠMet roid +G rant +8 55 +ģ ĸ +Ġ" _ +ãĥĥ ãĥī +Ġappra isal +ĠFred dy +04 6 +Ġ40 6 +Ġ18 30 +Ġd ocking +St atic +Ġp ont +ĠVolt age +ĠSt ead +ĠMort gage +ĠJon ah +Y L +CLASS IFIED +Ġas bestos +nik ov +Ġcoll agen +ĠOrb ital +P ocket +7 99 +Ġhy brids +inc hes +Ġinv oice +und y +Ġinequ alities +T rend +w ashed +B ALL +Ġluc id +ĠComment ary +Ġw itty +Br andon +Ġbru ising +Ġ6 20 +es cent +box ing +P OL +Ġ3 78 +R ect +Ġlic ences +ĠMcG ee +p ressed +D anny +Ġj ammed +ord inate +Ġle th +Ġdistingu ishes +ĠYam aha +IL S +ĠH ume +ĠC ategories +Rober ts +Ch art +Ġbeet le +ĠGra veyard +Ġ($ ) +o ÄŁ +Ġtw ilight +are lla +á ½ +Ġbooth s +ĠH HS +ĠFeld man +Ġexcav ation +Ġphilosoph ies +at ography +ĠGar age +te chnology +Ġunfor gettable +Ġver ifying +Ġsubord inates +E ls +Ġne b +G aming +EN A +ĠAchieve ment +it ters +ĠG abe +Ġd umps +for cer +Ġpo ignant +ĠM BA +ĠHe idi +ime i +Ġm ages +Ġliber ate +Ġcircum cised +ĠMer maid +ĠMat th +t ogether +ĠW ichita +Ġstore front +ĠAd in +V II +Four th +Ġexplore rs +W ER +Not able +Bro ok +m ens +F aith +-------- - +ĠJ ou +¬ ¼ +Ġpine apple +Ġam alg +el n +ark able +ĠãĤµ ãĥ¼ãĥĨãĤ£ +ĠãĤµãĥ¼ãĥĨãĤ£ ãĥ¯ãĥ³ +Ġov arian +ĠE choes +Ġhairc ut +Ġp av +Ġch illed +anas ia +Ġsty led +Ġd ab +ni per +Ġminister ial +ĠD UP +T an +Ġsul ph +ĠD eter +ĠBo hem +od an +Ġeduc ator +â ĵĺ +sp ir +Ch icken +ĠE leanor +Ġqu i +Ġheav iest +Ġgrasp ed +U RA +Ġcro oked +Jess ica +pro blem +Ġpred etermined +Ġman iac +Ġbreath s +ĠLauder dale +Ġh obbies +y z +Cr ime +Ġcharism a +d L +Ġle aping +Ġk ittens +Ang elo +ĠJ ACK +ĠSu zanne +Ġhal ting +ENT ION +Ġswall owing +ĠEarthqu ake +Ġeight eenth +ĠN IC +ĠIN F +ĠCons cious +Ġparticular s +circ le +7 40 +Ġbene volent +Ġ7 47 +Ġ4 90 +Ġr undown +ĠVal erie +ĠB UR +Ġcivil isation +ĠS chn +W B +ot ide +intern ational +Ġj ohn +Ġ19 02 +Ġpe anuts +Ġflav ored +k us +Ġro ared +Ġcut off +é £ +Ġorn ament +Ġarchitect ures +Ġ3 69 +ol or +ĠWild e +ĠC RC +ĠAdjust ed +Ġprov oking +land ish +Ġrational ity +Ġjust ifies +Ġdisp el +Ġa meric +ĠPol es +Ø © +Ġen vis +ĠD oodle +ä½ ¿ +igs aw +auld ron +Techn ical +T een +up hem +ĠX iang +Ġdetract ors +ĠZ i +ĠJournal ists +Ġconduc ive +ĠVolunte ers +Ġs d +Know ing +Ġtrans missions +ĠPL AN +ĠL IB +Ġall uded +Ġob e +Ġd ope +ĠGold stein +Ġwavelength s +ĠDest ination +nd a +ug i +Ġattent ive +ĠLe an +ral tar +Ġman g +mb uds +ak ings +b ender +Ġacc ol +Ġcraw led +N OW +Min nesota +Ġflour ished +ĠZ up +ĠSuper visor +ĠOliv ier +Ex cellent +Ġwid en +D one +Ġw ig +Ġmiscon ceptions +Cor p +W an +Ġvener able +ĠNot ably +ĠKling on +an imate +Bo ost +ĠS AY +miss ing +ibli ography +mel on +Ġpay day +Ø ³ +bo le +Ġve iled +ĠAl phabet +It alian +Ġever lasting +ĠR IS +ĠC ree +rom pt +Ġh ating +Ġgrin ning +Ġge ographically +OS H +Ġwe eping +ĠÂłĠÂłĠÂłĠÂł ĠÂłĠÂłĠÂłĠÂł +Ġimpe cc +Let ter +Ġblo ated +PL A +ĠFe in +Ġper sever +Th under +Ġa ur +ĠR L +Ġpit falls +âĸ º +Ġpredomin ant +Ġ5 25 +7 18 +AP E +7 14 +Ġfarm land +ĠQ iao +Ġv iolet +ĠBah amas +Ġinflic ting +ĠE fficiency +Ġhome brew +Ġundert ook +Ġcur ly +ĠHard ing +man ia +59 6 +Ġtem pered +Ġhar rowing +ĠP ledge +ĠFranken stein +è ª +M otion +Ġpredict ably +ĠExpl osion +oc using +er d +col o +FF ER +Ġback field +ĠV IDE +ue bl +N arr +ĠArg ument +Ġgen omic +Ġbout ique +Ġbatt ed +ĠB inary +Ġg amb +ĠRh ythm +67 3 +Ġa float +ĠOlymp ia +Y ING +Ġend if +is in +Ġwin ters +Ġsc attering +I v +D istance +Ġtr u +ĠCom fort +Ġne xus +Ġair flow +ĠByz antine +p ayers +con i +ĠB etsy +D eal +ĠN ug +ĠContin ent +red ibly +Ġoptim izing +al beit +Ġec static +ĠPro to +ç · +iv ot +âĸ Ħ +em p +rou nder +Ġcl out +ĠI ST +66 3 +ĠDoll ars +ĠD AC +Ġsubsc ribed +Ġrehears al +Ġam ps +ĠSh ang +es m +Ġspr inkle +Ġassail ant +ĠO o +ĠCoin base +T act +Ġret ina +Ġn uns +R ON +att o +Ġj ug +ĠSV G +Ġb ikini +ĠFI LE +ĠFound ers +ep ort +ĠK P +Ġrest ores +ĠTh ick +Ġash ore +Ġappro vals +R ender +M AG +G raham +ĠCort ana +ãĥ³ ãĤ¸ +ss h +or ians +ars ity +ĠInsp ired +u pper +Ġsign alling +Ġreb uke +Ġfl ares +Ġdownt ime +Stud ies +Ġstagn ation +ĠSequ ence +Ġgr unt +Ġass ures +ĠPL A +59 2 +Ġintra ven +d epend +Sus an +ĠManz iel +Man ia +Cont ract +Ġsl ams +Ġcult ured +Ġcred itor +L IST +ĠH UM +ĠChatt anooga +serv ed +Ġclo aked +ĠF TP +p owder +ĠSt ella +uct ive +Ġcheap ly +ĠMU CH +ĠGalile o +Ġsu ites +spe ech +Ġdeliber ations +ĠCh ips +« ĺ +Bal ance +ĠWyn ne +ĠAk ron +Ass et +Ġhon oured +Ġed ged +Like wise +anim ous +ĠW age +ĠEz ek +ad vertisement +ĠRT X +ĠM AD +Ġmigr ating +ĠS QU +Ġ4 75 +Ed ited +Ġshorth and +ĠBas ics +Ġcro tch +ĠEV EN +Ġv m +effic iency +Ġcal ves +ĠF rie +ĠBrill iant +Ġstri kers +Ġrepent ance +Ġarter ies +r l +B ed +h ap +Ġcrypt ography +ĠSab res +Ġ4 14 +vi ks +ih ara +aps es +T alking +Ġintertw ined +Ġdoc ks +Ġalle le +ĠArt ifact +ĠH IM +t orn +ç ķ +Ġop acity +ĠE ly +os uke +Ġn ipple +Ġhand written +ĠV K +ĠChamber lain +ĠLa os +ig raph +g row +Ġtr illions +Ġdescend ant +ĠSail or +as uring +Ġce ilings +ĠWare house +f lying +ĠGl ow +Ġn ont +Ġmiscar riage +Ġrig s +Ġmin istries +Ġelabor ated +Ġdel usional +ĠHum ane +Ġ3 79 +n ets +Ġblack out +add ers +Ġn p +ĠT ire +ro sc +Ġsub div +Ġlink age +Ġchron ological +ĠHER O +Ġres ettlement +ĠVin yl +Ġpast oral +ĠMob il +ĠBar bar +Co oldown +ĠF ritz +c riminal +re pe +Ġbell ig +ĠBre ed +Ġ4 18 +Ġsem blance +ij k +Ġcur tail +Ġclin ch +cont ained +ĠProm pt +ast on +Ġw i +Ġpursu its +5 15 +ĠGl oss +Ġfl ips +Ġcoup ons +Ġcl oning +ĠLike ly +Rem oved +ĠQu artz +r ices +ĠSpe ars +Ġp ious +Ġdep reciation +ĠD are +oun ces +am az +O nt +Ġp innacle +d ocker +0 26 +ĠW yr +ĠPro per +Ë Ī +n il +By tes +Ġseek er +t rial +Ġunf olds +ĠMar se +Ġextravag ant +ĠSurviv ors +RED ACTED +ĠSpeed way +ĠCra igslist +sub mit +ĠGener ations +Ġup holding +Ġblood stream +ĠMiss ions +ĠL awn +Ġlim bo +ene i +H uh +ĠWild cats +pre p +ĠMark us +ĠFor bidden +rit ic +IN O +Ġexhib iting +requ ent +ch uk +Ġhabit ual +ĠComp atibility +Dr ag +RIP T +uj ah +GR OUND +Ġdelinqu ent +Ġburn er +Ġcontempor aries +Ġgimm ick +load s +Ġno zzle +p odcast +ĠW ak +ĠStat en +ĠK uh +ãģ ĵ +inter rupted +Ġinv incible +ĠBurn ett +cig arette +ĠPeb ble +ĠTem porary +ĠMar ino +58 2 +Ġwast eland +ident ly +T x +Ġr ite +ĠPan asonic +ĠM iddles +ĠHort on +ae us +Ġc uring +Ġm ats +Ġadj ourn +Ġfears ome +pe z +bo ats +Ġpro pell +Ġconflic ted +ĠAng er +Ġinsurg ent +K arl +Ġco ales +Ġsouth western +Ġdis su +ĠO vert +******** **** +Ġbox ed +ĠBr une +aa a +Ġgard ening +ĠEng el +tr acks +Ġpur ified +Ġplace holder +ĠL ikes +Ġd an +G ab +Ġe ct +ĠF aw +ĠEl iot +Ġ' , +otrop ic +ĠRu in +hed on +Ġca ul +Ġa ft +ĠCad illac +gh a +ass ian +ud eb +ĠT ick +Ġadjust s +AR GET +5 37 +isc he +ant y +ĠFried rich +ĠBl izz +ĠA OL +Camp aign +Ġmamm al +ĠVe il +ĠK ev +ĠMaur it +ĠDam ien +N ation +E astern +Ġ{ : +Ġ= ================================ +Ġstereotyp ical +Ġatt ic +ĠCy borg +requ ire +Ġaward ing +ĠPap ua +bt n +b ent +B oo +Ġ( = +ĠX ander +ĠSomers et +Ġcatch y +Ġcert ify +STR UCT +Ġit al +Ġt ides +ĠBr ands +G ray +comp etitive +Ġcur ator +ĠD G +omin ium +ĠGM Os +ci ating +ĠCarm en +ow ard +Balt imore +Ġr gb +C u +Ġwip es +spe ll +IT NESS +Ġsummar izes +ĠRe vis +Ġwhistlebl owers +ĠBre ach +Ġcro chet +k os +ews ki +Ġrep et +Ġcrim son +ĠKar achi +read able +dim ension +ĠI gor +ild ed +ĠZ ed +ĠKe ane +ĠCos metic +DE P +Ġretreat ing +ĠU A +ens ical +Ġd usk +ĠDick ens +Ġaren as +ĠPass age +level s +Ġcur v +P ope +Ġch ores +ĠEl ise +ĠComp ass +b ub +Ġmamm alian +ĠSans krit +ĠAN C +ĠCr ack +Q ual +L aun +amp unk +Ġlearn ers +Ġglam orous +Ġfur the +erm ott +c and +Gener ic +Ġnarr ated +Ġdisorder ly +ĠTrans actions +ĠDet ention +ĠR oku +Ä į +Ġunder statement +ĠS aur +ĠRodrig o +ĠAS AP +S in +Ġre joice +Method s +Ġelectro de +Ġworsh ipped +Ġid i +ĠPhys icians +Ġpop up +Ġde ft +ĠRem oval +ĠBu enos +ver bs +Ġfun k +ush a +rict ion +ore a +ĠBang alore +ĠKen obi +zz i +Ġnorm ative +Ġgobl ins +Ġcaf es +ĠUN CLASSIFIED +ĠF ired +S IGN +Ġs clerosis +ĠV oter +ĠSon ny +ĠExt end +ĠEV s +Ar senal +Ġp si +Ġwid est +ĠT us +Ġlo oms +Ġjust ifying +ĠGr anger +è ¯ +Ref er +58 3 +Ġflour ishing +ab re +Ġr ave +ĠCont ra +Ġ18 98 +Add s +Ġf ul +ĠCo oke +some one += # +67 1 +Ġy ak +Ġar te +ĠMis cellaneous +ĠDet ection +ĠCl ancy +â ģ +ass ies +Ġval iant +ĠFemin ist +cor ruption +V el +P ear +Ġsucc inct +Ġquick est +k w +Ġsp itting +ĠL ibraries +åħ ī +ant z +D ad +ĠSpec ifications +rup ulous +and r +RES ULTS +Ġsnow ball +Ġpred is +ĠB axter +ĠNurs ing +ĠCh aff +s we +Ġout age +Ġnest ing +Ġnotor iety +tr igger +on ite +j on +Ġf ou +ook ed +ĠCelebr ity +re ality +Ġfat ig +Ġhug ging +Ġbother s +ĠPan zer +ĠCh andra +fig ured +Ġvol ts +ĠCloud s +Ġfee ble +ĠCur ve +ĠAs us +78 6 +abs or +ĠV ICE +ĠH ess +Ġmanufact ures +Ġgri zz +ĠPower ful +ac id +Ġsub sections +ĠKrug man +ĠAl ps +is u +Ġsequ est +ĠUlt ron +ĠT inker +ĠGo ose +Ġmism atch +Att orney +Ġmorph ology +ĠSix ers +ut tered +ĠE LECT +gr an +Rus sell +ĠG SL +Ġfort night +Ġ. ) +Ġapost le +pr one +el ist +Unt itled +ĠIm plementation +ist ors +Ġtank er +Ġpl ush +Ġattend ants +ĠT ik +ĠGreen wich +ĠY on +ĠSP L +cell s +unt led +S olution +ĠQu é +Ġvac ated +Ġupt ick +ĠMer idian +æ ĥ +ĠDr ill +9 25 +58 4 +Ġrenov ated +ĠKub rick +zy k +Ġl ousy +pp el +ohyd rate +ĠI zzy +lesi astical +CC C +ĠAj ax +Ġad apters +ĠPetra eus +Ġaffirm ation +ĠST OR +le ms +ad oes +ĠConstantin ople +Ġp onies +Ġl ighthouse +Ġadherent s +ĠBre es +omorph ic +Fight ing +Ġpl aster +ĠP VC +ĠOb st +Ġdear ly +ĠTo oth +icks on +Ġsh aming +P lex +A gg +ĠâĢ¦ " +Ġsub reddits +Ġpige on +ĠResident ial +ĠPass ing +Ġl um +ĠP ension +Ġpessim istic +Ġ4 32 +z inski +c ade +0 75 +Ġapolog ised +iy ah +Put ting +Ġgloom y +ĠLy me +=-=-=-=- =-=-=-=- +ĠT ome +ĠPsych iatric +ĠH IT +c ms +ap olog +Ġbreak er +Ġdeep en +Ġtheor ist +ĠHigh lands +Ġb aker +Ġst aples +Ġinterf ered +ĠAb ortion +jo ined +ch u +Ġform ulate +Ġvacc inations +Ġban ter +phe us +Ġoutfield er +ĠM eter +Ġ# #### +Ġ18 95 +Ġnarrow ing +ĠST ORY +f p +ĠC ST +ign ore +Ġproclaim ing +ĠR U +ĠB ALL +yn a +65 3 +Ġpos it +P RE +59 4 +ĠRegist rar +ĠPil grim +ic io +Ġpre tt +Ġlif eless +Ġ__ _ +Ne igh +ĠCh urches +orn o +Ġor cs +Ġkind red +ĠAud it +Ġmillenn ial +ĠPers ia +g ravity +ĠDis ability +ĠD ARK +W s +od on +Ġgrand daughter +ĠBro oke +ĠA DA +ER A +Ġpick ups +ĠWil kinson +ĠSh ards +ĠN K +Ġexp el +ĠKis lyak +Ġj argon +Ġpolar ized +ian e +Pub lisher +Ġreb utt +Ġapprehens ion +ĠK essler +Ġpr ism +F UL +19 64 +ĠL oll +ä ¿ +le thal +Å Ł +Ġg hetto +Ġb oulder +ĠSlow ly +ĠOsc ars +ĠInst ruction +ĠUl tr +ĠM oe +N ich +ĠP ATH +( * +ĠRE LEASE +un ing +rou se +en eg +Ġre imb +ĠDet ected +Do S +Ġster ling +Ġaggreg ation +ĠLone ly +ĠAtt end +hig her +Ġairst rike +ks on +SE LECT +Ġdef lation +ĠHer rera +C ole +rit ch +Ġadvis able +F ax +Ġwork around +Ġp id +mort em +ers en +Ġtyp o +Ġal um +78 2 +ĠJam al +script s +Ġcapt ives +ĠPres ence +ĠLie berman +angel o +Ġalcohol ism +ass i +Ġrec ite +Ġgap ing +Ġbask ets +ĠG ou +Brow ser +ne au +Ġcorrect ive +und a +sc oring +ĠX D +Ġfil ament +Ġdeep ening +ĠStain less +Int eger +Ġbu ggy +Ġten ancy +ĠMub arak +Ġt uple +ĠD roid +ĠS itting +Ġforfe it +ĠRasm ussen +ixt ies +es i +ĠKim mel +Ġmetic ulously +Ġap opt +ĠS eller +08 8 +ec ake +hem atically +T N +Ġmind less +Ġdig s +ĠAcc ord +ons ense +em ing +br ace +Ġe Book +ĠDist ribut +ĠInvest ments +w t +] ), +beh avior +56 3 +Ġbl inding +ĠPro testers +top ia +Ġreb orn +ĠKel vin +ĠDo ver +ĠD airy +ĠOut s +Ġ[ / +Ï Ģ +b p +ĠVan ity +ĠRec ap +ĠHOU SE +ĠF ACE +Ġ4 22 +69 2 +ĠAnt ioch +cook ed +Ġcoll ide +Ġa pr +Ġsle eper +ĠJar vis +Ġalternative ly +ĠLe aves +ĠM aw +Ġantiqu ity +ĠAdin ida +Ġab user +Poké mon +Ġass orted +ĠRev ision +ĠP iano +ĠG ideon +O cean +Ġsal on +Ġbust ling +ogn itive +ĠRah man +Ġwa iter +Ġpres ets +ĠO sh +ĠG HC +oper ator +Ġrept iles +Ġ4 13 +ĠG arr +ĠCh ak +Ġhas hes +Ġfail ings +Ġfolk lore +Ġab l +ĠC ena +ĠMac Arthur +ĠCOUR T +Ġperipher y +app ers +Ġreck oned +ĠInf lu +ĠC ET +Ġ3 72 +ĠDefin itive +ass ault +4 21 +Ġreservoir s +Ġd ives +ĠCo il +DA Q +Ġvivid ly +ĠR J +ĠBel lev +Ġec lectic +ĠShow down +ĠK M +ip ed +reet ings +ĠAs uka +L iberal +ĠÏ Ħ +Ġbystand ers +ĠGood win +uk ong +S it +ĠT rem +Ġcrim inally +ĠCirc us +ch rome +88 7 +Ġnan op +ĠOb i +ĠL OW +o gh +ĠAuth ors +ob yl +Ur ban +Ġt i +ĠWe ir +t rap +ag y +Ġparent heses +Ġout numbered +Ġcounter productive +ĠTob ias +ub is +P arser +ST AR +Ġsyn aptic +ĠG ears +Ġh iber +Ġdebunk ed +Ġex alted +aw atts +H OU +Ch urch +ĠPix ie +ĠU ri +ĠForm ation +ĠPred iction +C EO +Ġthro tt +ĠBrit ann +ĠMad agascar +ë ĭ +Ġbill boards +ĠRPG s +ĠBe es +complete ly +F IL +Ġdoes nt +ĠGreen berg +re ys +Ġsl ing +Ġempt ied +ĠPix ar +ĠDh arma +l uck +ingu ished +Ġend ot +Ġbab ys +05 9 +che st +r ats +Ġr idden +Ġbeet les +Ġillum inating +Ġfict itious +ĠProv incial +Ġ7 68 +Ġshe pherd +ĠR ender +Ġ18 96 +C rew +Ġmold ed +ĠXia omi +ĠSp iral +Ġdel im +Ġorgan ising +Ġho ops +ĠBe i +z hen +Ġfuck in +Ġdec ad +Ġun biased +am my +sw ing +Ġsmugg led +Ġk ios +ĠP ERSON +ĠInquis itor +Ġsnow y +Ġscrap ing +ĠBurg ess +P tr +ag ame +R W +Ġdro id +ĠL ys +ĠCass andra +Jac ob +Ġ35 4 +Ġpast ure +Ġfr anc +ĠScot ch +ĠEnd s +ĠI GF +def inition +Ġhyster ical +ĠBrown e +77 1 +Ġmobil ization +æ ķ +iqu eness +Th or +Ġspear headed +Ġembro iled +Ġconject ure +jud icial +Ch oice +Ġpaper back +P ir +Ġrec overs +ĠSur ge +ĠSh ogun +ĠPed iatrics +ãģ ł +Ġsweep s +ĠLabor atories +ĠP acks +al us +add in +Ġhead lights +g ra +Ev idence +COL OR +Ad min +Ĭ ± +Ġconco ct +s ufficient +Ġun marked +Ġrich ness +Ġdiss ertation +Ġseason ing +Ġg ib +ĠM ages +un ctions +ĠN id +che at +ĠTM Z +c itizens +ĠCatholic ism +n b +Ġdisemb ark +ĠPROG RAM +a ques +Ty ler +Or g +ĠSl ay +ĠN ero +ĠTown send +IN TON +te le +Ġmes mer +9 01 +Ġfire ball +ev idence +aff iliated +ĠFrench man +ĠAugust a +0 21 +Ġs led +Ġre used +ĠImmun ity +Ġwrest le +assemb led +Mar ia +Ġgun shots +ĠBarb ie +Ġcannabin oids +ĠTo ast +ĠK inder +IR D +Ġre juven +Ġg ore +Ġrupt ure +Ġbre aching +ĠCart oon +Ġ4 55 +ĠPale o +6 14 +Ġspe ars +ĠAm es +ab us +Mad ison +GR OUP +Ġab orted +y ah +Ġfel on +Ġcaus ation +Ġprep aid +Ġp itted +op lan +ĠShel ley +ĠRus so +ĠP agan +Ġwill fully +ĠCan aver +und rum +ĠSal ary +ĠAr paio +read er +ĠR ational +ĠOver se +ĠCa uses +Ġ* . +Ġw ob +Ke ith +ĠCons ent +man ac +77 3 +6 23 +Ġfate ful +et imes +Ġspir ited +ĠD ys +Ġhe gemony +Ġboy cot +ĠEn rique +em outh +Ġtim elines +ĠSah ara +ĠRel ax +ĠQuin cy +ĠLess ons +ĠE QU +SE A +N K +ĠCost co +Incre ase +Ġmotiv ating +ĠCh ong +am aru +ĠDiv ide +Ġped igree +ĠTasman ia +ĠPrel ude +L as +9 40 +57 4 +Ġch au +ĠSp iegel +un ic +-- > +ĠPhil ips +ĠKaf ka +Ġuphe aval +Ġsent imental +Ġsa x +ĠAk ira +ser ial +Mat rix +Ġelect ing +Ġcomment er +ĠNeb ula +ple ts +ĠNad u +ĠAd ren +Ġen shr +ĠR AND +fin ancial +ĠCly de +uther ford +Ġsign age +Ġde line +Ġphosph ate +rovers ial +f ascist +ĠV all +ĠBeth lehem +Ġfor s +Ġeng lish +S olid +N ature +Ġv a +ĠGu ests +Ġtant al +Ġauto immune +;;;;;;;; ;;;; +ĠTot ally +ĠO v +Ġdef ences +ĠCoc onut +Ġtranqu il +Ġpl oy +Ġflav ours +ĠFl ask +ãĤ¨ ãĥ« +ĠWest on +ĠVol vo +8 70 +Ġmicro phones +ver bal +R PG +Ġi ii +; } +0 28 +Ġhead lined +Ġprim ed +Ġho ard +ĠSh ad +ĠEN TER +Ġtri angular +Ġcap it +l ik +ĠAn cients +Ġl ash +Ġconv ol +Ġcolon el +en emy +G ra +Ġpub s +ut ters +Ġassign s +ĠPen et +ĠMon strous +ĠBow en +il ver +H aunted +ĠD ing +start ed +pl in +Ġcontamin ants +ĠDO E +ff en +ĠTechn ician +R y +Ġrob bers +Ġhot line +ĠGuard iola +ĠKau fman +row er +ĠDres den +ĠAl pine +E lf +Ġf mt +ĠS ard +urs es +g pu +Un ix +Ġunequiv ocally +ĠCitizens hip +qu ad +m ire +ĠS weeney +B attery +6 15 +Ġpanc akes +Ġo ats +M aps +ĠCont rast +mbuds man +ĠE PS +Ġsub committee +Ġsour cing +Ġs izing +ĠBuff er +ĠMand atory +Ġmoder ates +ĠPattern s +ĠCh ocobo +ĠZ an +ĠSTAT ES +ĠJud ging +ĠIn her +* : +Ġb il +ĠY en +Ġexh ilar +oll ower +z ers +Ġsn ug +max imum +Ġdesp icable +ĠP ACK +ĠAn nex +Ġsarcast ic +Ġlate x +Ġt amp +ĠS ao +b ah +ĠRe verend +ĠChin atown +ĠA UT +d ocumented +ĠGA BA +ĠCan aan +ĠÙ ħ +Ġgovern s +pre v +E sc +ĠEst imates +OS P +Ġendeav our +ĠCl osing +omet ime +every one +Ġwor sen +Ġsc anners +Ġdev iations +ĠRobot ics +ĠCom pton +Ġsorce rer +Ġend ogenous +Ġem ulation +ĠPier cing +ĠA ph +ĠS ocket +Ġb ould +ĠO U +ĠBorder lands +Ġ18 63 +G ordon +ĠW TO +Ġrestrict s +Ġmosa ic +Ġmel odies +ç Ħ +T ar +Ġdis son +ĠProv ides +Ġ ...... +b ek +F IX +Ġbro om +ans hip +Do ctors +Ġner ds +ĠReg ions +na issance +Ġmet e +Ġcre pt +pl ings +Ġgirlfriend s +kn it +ig ent +ow e +Ġus hered +ĠB az +M obil +4 34 +ĠPres ents +orig in +Ġins omnia +ĠA ux +4 39 +ĠCh ili +irs ch +G AME +Ġgest ation +alg ia +rom ising +$ , +c row +ĠIn spection +at omic +Rel ations +J OHN +rom an +ĠClock work +ĠBak r +m one +M ET +Ġthirst y +Ġb c +Ġfacult ies +R um +Ġnu ance +ĠD arius +ple ting +fter s +etch up +Reg istration +ĠK E +R ah +Ġpref erential +ĠL ash +ĠH H +Val id +ĠN AV +Ġstar ve +ĠG ong +z ynski +ĠAct ress +Ġw ik +Ġun accompanied +lv l +Br ide +AD S +ĠCommand o +ĠVaugh n +Wal let +Ġho pping +ĠV ie +Ġcave ats +Ġal as +if led +ab use +66 1 +Ġib n +Ġg ul +Ġrob bing +t il +IL A +Ġmit igating +Ġapt ly +Ġty rant +Ġmid day +ĠGil more +ĠDe cker +Ġ§ § +part ial +Ex actly +Ġphen otype +Ġ[+ ] +ĠP lex +ĠI ps +vers ions +Ġe book +Ġch ic +g ross +":" "},{" +ĠSur prisingly +M organ +Ġresid ues +ĠConf ederation +in feld +Ġl yr +mod erate +Ġperpend icular +V K +Ġsynchron ized +Ġrefres hed +Ġad ore +ĠTor ment +ol ina +Ġ26 00 +Item Tracker +Ġp ies +ĠF AT +ĠR HP +0 48 +ĠRES P +ĠB J +all ows +P and +Ġunw elcome +ĠV oc +ĠBast ard +ĠO W +ĠL AR +ĠHeal er +Environment al +ĠKen yan +ĠTr ance +ĠP ats +Ġali ases +ĠGar field +Ġcampaign er +Ġadvance ments +ĠOkin awa +ĠC oh +ows ky +Ġstar ved +Ġsize able +Ġ: -) +Ġm RNA +Ġsusp ensions +ist ar +Scot land +Pr in +-------------------------------- ---------------- +Ġ50 2 +Ġteasp oons +Ġ10 50 +Ġcoerc ive +ĠMason ic +edd ed +ĠPass enger +Ġl att +Ġbr aces +ĠSt eal +ĠNY T +ĠK ats +ĠCel est +ae z +T u +ĠCoul ter +ðŁ ĺ +Fl ickr +ĠWil mington +ith s +++ ; +Ġv ending +Ġneg ro +ĠPh i +ĠYellow stone +Call back +Ġsh ampoo +ĠSh ades +w at +Ġsuper human +Ġridic uled +Ġhol iest +om bo +Ġintern s +Ġh one +ĠPar agu +UR I +Ġd angling +ãĤ » +so v +ict ional +av ailability +Ġrev ocation +Ġd ow +in ic +ĠTHE IR +Ġis o +Ġout ings +ĠLeth al +Ġ) )) +Ġinacc ur +Ġout landish +Ġan us +let ico +id on +l ol +Ġun regulated +Ġsuccumb ed +Ġc uff +ĠWast eland +let al +Ġsub str +Ġcoff ers +Ġautom akers +ov i +ĠX ue +ĠDayton a +Ġjar ring +Ġf umes +Ġdisband ed +z ik +itt on +Ġstriking ly +Ġsp ores +Ad apter +.) : +ĠLynd on +ival ry +Ġor ally +Ġtumult uous +Ġdisple asure +Ġcon es +or rect +Ġappe ase +Ġder by +ĠTrip oli +ĠAl ess +Ġp oked +ĠGu ilty +v P +En ough +Ġorig inals +6 99 +Ġrabb i +Ġproverb ial +Ġpostp one +el ope +ĠMist y +Ġstaff ed +ĠUn employment +redit ary +Ġdilig ent +re comm +me asures +as in +8 25 +Ġpond s +Ġmm ol +ĠS AR +ĠC ARE +Ġ3 71 +Ġclen ched +ĠCors air +Ġcaric ature +z n +att ach +ĠSch ro +spe ak +p ainted +ĠS uc +ĠE NT +Ġcell ul +ĠP aid +di agn +WH ERE +Ġtext ed +B arn +Ġret racted +ĠRe ferred +S av +Ġup keep +Ġwork places +ĠTok ens +Ġampl ify +cl inical +Ġmult ic +mber g +Ġconvol uted +Reg ion +5 65 +ĠTop ic +Ġsn ail +Ġsal ine +Ġins urrection +ĠPet r +f orts +B AT +ĠNav ajo +Ġrud imentary +ĠLak sh +OND ON +Me asure +Ġtransform er +ĠGodd ard +Ġcoinc ides +ir in +R ex +ĠB ok +qu it +Ġshotgun s +Ġprolet arian +Ġsc orp +ĠAd a +5 14 +Ġsl ander +record ed +Ġemb ell +ris ome +Ġapolog izing +ĠMul cair +ĠGib raltar +Cl a +Ġall ot +ĠAtt ention +Ġ4 33 +le ave +Ġwh ine +ĠIss a +ĠFa ust +ĠBar ron +hen y +Ġvictim ized +J ews +Ġnurt uring +ett el +W inged +ĠSub tle +Ġflavor ful +ĠRep s +eng ed +call back +Ġdirection al +Ġcl asp +ĠDirect ions +plan et +icult ure +Hel per +ic ion +ac ia +Ġç ¥ŀ +Ġsur ges +Ġcan oe +ĠPrem iership +be en +Ġdef ied +ĠTro oper +Ġtrip od +Ġgas p +ĠE uph +ĠAd s +vern ight +high ly +R ole +Ġent angled +ĠZe it +6 18 +ĠRust y +Ġhaven s +ĠVaugh an +HA EL +ĠSER VICE +/ , +Ġstr icken +Ġdel usions +Ġb is +ĠH af +Ġgrat ification +Ġent icing +UN CH +Ad ams +ĠOL ED +ĠBeet le +Ġ18 99 +ĠSO FTWARE +ateg or +V L +ĠTot em +ĠG ators +AT URES +Ġimped ance +Reg istered +ĠC ary +ĠAer ial +on ne +en ium +Ġd red +ĠBe g +Ġconcurrent ly +Ġsuper power +ĠX an +j ew +imes ter +ĠDick inson +âĶ ģ +F la +Ġp ree +ĠRoll ins +© ¶æ +Ġden omination +ĠL ana +5 16 +Ġinc iting +sc ribed +j uries +ĠWond ers +app roximately +Ġsusp ending +Ġmountain ous +ĠL augh +oid al +N s +Det ect +) = +ĠL uthor +ĠSchwarz enegger +ĠMull er +ĠDev i +ec ycle +J ar +6 13 +ĠL ongh +B ah +ĠSP ORTS +n w +Ġref inement +Ġwater ways +Ġd iner +Bl ade +68 3 +F ac +Ġinitial s +Ġro g +Ġparan ormal +B UT +Ġ[ ( +ĠSw anson +ĠM esh +âĸ ¬ +Impro ve +ĠRad iation +ĠEst her +ĠE sk +ĠA ly +ik y +Ġir rad +ĠBuck ingham +Ġref ill +Ġ. _ +Re pe +CON CLUS +Ġdifferent iated +Ġchi rop +ĠAt kins +Pat tern +Ġexc ise +Ġcab al +N SA +ĠST A +ĠS IL +ĠPar aly +Ġr ye +ĠHow ell +ĠCount down +ness es +alys ed +Ġres ize +ãĤ ½ +Ġbudget ary +ĠStr as +w ang +Ġap iece +Ġprecinct s +Ġpe ach +Ġsky line +Ġ35 3 +pop ular +App earances +ĠMechan ics +ĠDev Online +S ullivan +Z en +Ġp u +op olis +5 44 +Ġde form +Ġcounter act +ĠL ange +Ġ4 17 +Con sole +77 4 +Ġnodd ing +Ġpopul ism +Ġhe p +Ġcoun selling +compl iance +U FF +Ġunden iably +Ġrail ing +ĠHor owitz +ĠSim one +ĠBung ie +Ġa k +ĠTal ks +x ff +fl ake +Cr ash +Ġsweat y +Ġban quet +ĠOFF IC +Ġinvent ive +Ġastron omer +ĠStam ford +ĠSc are +ĠGRE EN +olic ited +Ġr usher +Ġcent rist +ight ing +Ġsub class +Ġdis av +Ġdef und +ĠN anto +oci ate +m ast +Ġpac if +Ġm end +e ers +imm igration +ESS ION +Ġnumber ing +Ġlaugh able +ĠEnd ed +v iation +em ark +P itt +Ġmetic ulous +ĠL F +Ġcongrat ulated +ĠBir ch +Ġsway ed +Ġsemif inals +Ġhum ankind +m atter +ĠEqu ip +opa usal +S aid +ĠLay out +Ġvo icing +Ġth ug +Ġporn ographic +I PS +Ġmo aning +Ġgriev ance +Ġconf essions +esc al +TEXT URE +Aut hent +os aurus +P urchase +Ġreleg ation +al ter +ĠÂł Âł +Ġr iddled +Ġo gre +ĠLow ell +Occ up +E at +ĠHy der +ĠAdvis er +Com merce +H unt +ĠOr th +ĠComp etitive +ĠCL A +CD C +Ġsal ads +F le +Ġindustrial ized +` , +ĠO WN +Ġbec k +ĠPart icularly +oub t +Ġm M +ĠHuss ain +ĠChen nai +Ġ9 20 +Ġappoint ing +ĠCull en +,,,, ,,,, +Ġp ores +ver ified +Ġbi ochemical +em ate +Ġcoward ly +ĠHels inki +ĠEthiop ian +S OURCE +ER C +est ro +Ġbi otech +ĠS our +Ġbrew er +Bloom berg +Ġintens ify +Gl ass +an co +ĠF DR +gre SQL +ĠF ires +©¶æ ¥µ +ec o +100 1 +ĠHom eless +Ġinstant aneous +ĠH aste +ig el +D iamond +Ġp aving +Ġland fill +Ġd ads +h oun +: ] +Ġinc endiary +ĠLiving ston +ĠHil bert +ĠChe cks +st yles +in ators +ĠCl ive +ph rine +Ġchimpan zees +Ġp all +ĠJ M +ĠAad haar +ð Ŀ +Ġachie vable +dis abled +P ET +OOOO OOOO +M ot +Ġint angible +Ġbal let +ĠWe bs +ĠEst imated +Effect s +Ġb ailed +Josh ua +Ġturb ulence +Ġoccup ant +ĠDay light +Ġ36 1 +me et +Ġstat ically +Ġon look +Ġk i +il legal +Ġvel vet +Ġdehyd ration +Ġacqu ies +ĠRe z +ak ura +ĠU pton +at ro +Ġincomp rehensible +Ġback door +ĠRh ino +7 27 +Ġmath s +) + +Ġhe resy +Ġd f +ĠRoc he +ĠL ydia +Ġpanc reat +re ply +arre ll +Ġsolicit ation +Ġcirc adian +BI P +Ġfor ay +Ġcrypt ic +iz u +ime o +ĠTom ato +ĠH oms +ex amination +Ġqu arry +ĠVal iant +ĠJer icho +ĠIN CLUD +Ġ18 40 +5 19 +Ġres ists +Ġsnap shots +ĠSp ur +ĠAnt iqu +Log in +Ġbest selling +Ġant ic +ĠS utherland +ãĤ¢ ãĥ« +Ġ~ / +ĠP arm +è ĥ +P ages +int ensity +Ġimm obil +Ġ18 65 +zz o +Ġn ifty +Ġf entanyl +ĠPres ervation +op hen +Ġd arts +ĠD inosaur +po inters +ĠR ite +s uggest +aware ness +ĠSher idan +Ġst ances +Ġsor cery +Ġper jury +ĠNik ola +ie ver +Ġf iance +ĠJordan ian +ĠBall oon +Ġn ab +Ġk b +Ġhuman ities +ĠTan aka +hill ary +Ġconsult ancy +ĠZ ub +Ġrem ission +Ġconf id +CH Q +ĠF ug +Ġimpro vis +Y ep +/ _ +Ġunwilling ness +Ġport folios +05 5 +ĠInstruct or +aim an +Ġclaim ants +M bps +ĠBy e +re ceived +T weet +Ġind emn +ri z +am ara +N at +Ġeval uates +ĠL ur +ep ad +FO X +ĠTh ro +Ġrust y +Ġbed rock +ĠOp rah +J B +Ġmanip ulative +Ġwill ful +Ġrel apse +Ġext ant +The me +S ensor +ĠSt ability +go vern +Ġpo ppy +Ġkn ack +Ġins ulated +ĠT ile +ĠExt rem +Ġunt old +Ġconver ge +Ġref uel +ig roup +Ġdistort ions +Ġrav aged +Ġmechan ically +ĠRe illy +ĠN ose +ĠIncarn ation +ĠBeck y +abb ling +Ġt aco +Ġr ake +Ġmelanch oly +Ġillust rious +ĠDart mouth +Gu ide +ĠR azer +ĠBen z +Ult imate +ĠSur prise +Ġpage ant +off er +Who ever +Ġw iser +Ġchem ist +ĠHE LL +ĠBul k +Ġpl utonium +ĠCO VER +Ö ¼ +f ailed +Ġtire lessly +Ġinf ertility +ĠTr ident +ĠShow time +ĠC iv +V ice +requ ires +itt ance +Ġun controlled +interest ing +56 1 +Ġinnov ate +ateg ic +L ie +ĠS elling +U l +Ġsav ior +ĠT osh +Ġsw ast +P ASS +Ġr ink +Ġcard io +ĠI ro +ud i +Ġv antage +Ġv ans +ĠNi ño ++ = +Ġpropag ate +< ? +Ġmethod ological +204 39 +Ġtrig lycer +Ġing rained +ĠAn notations +arr anted +6 17 +ĠS odium +ĠA AC +techn ical +mult ipl +Ġ3 73 +å ĭ +Ġdec isively +Ġboost ers +Ġdessert s +ĠGren ade +Ġtest ifying +ĠSc ully +ID s +Ġlock down +ĠSc her +ĠR é +ĠWhit man +ĠRams ay +rem ote +Ġh ikers +ĠHy undai +Ġcons cientious +Ġcler ics +ĠSiber ian +ut i +is bury +Ġrel ayed +Ġqu artz +ĠC BI +seek ers +ull a +Ġweld ing +ĠSh al +ble acher +T ai +ĠSam son +Ġt umble +ĠInvest or +Ġsub contract +ĠShin ra +ow icz +j andro +d ad +Ġtermin ating +ĠNe ural +ä» £ +Ġleak age +ĠMid lands +ĠCaucas us +í ķ +c it +ll an +iv ably +ĠAlb ion +Ġ4 57 +Ġregist rations +Ġcomr ade +Ġclip board +0 47 +Ġdiscour aging +ĠO ops +Ad apt +Ġem path +n v +ĠPR OT +ĠDon n +ĠP ax +ĠB ayer +t is +Squ are +Ġfoot prints +part icip +ĠChile an +B rend +ind ucing +M agn +Ġclub house +ĠMagn um +Ġenc amp +ĠEth nic +uch a +ere y +Ġw atered +ĠCal ais +Ġcomplex ion +Ġsect s +Ġren ters +Ġbr as +oÄŁ an +Time out +Man agement +Ġinf ographic +P okemon +Cl ar +Ġloc ality +Ġfl ora +as el +P ont +Ġpop ulate +ĠO ng +Ġsubs istence +Ġa uctions +ĠMcA uliffe +ĠL OOK +br inger +Ġtit an +Ġmanif old +ĠâĹ ı +Ġcalibr ated +Ġcal iphate +ĠSH E +ĠCommission ers +ce ivable +j c +W inner +5 24 +Ġcond one +Other wise +Ġp iling +Ġem body +ĠCrime an +ut ics +ĠEx hibition +Ġ4 26 +e ering +Ġv ying +ĠH UGE +* =- +Ġprin cipled +à ¦ +Ġquir ks +ĠEdit ors +put ing +G ES +ĠF TA +ठ¾ +add on +ĠH AM +ĠFrie za +W oman +. $ +Ġc rib +ĠHer od +Ġtim ers +ĠSp aces +ĠMac intosh +at aka +Ġgl ide +Ġsmell ing +ĠB AL +Ġun su +Ġcond os +Ġbicy cl +ĠRev ival +55 3 +Ġjugg ling +H ug +ĠKardash ian +ĠBalk ans +mult iple +Ġnutrit ious +oc ry +19 00 +Ġinteg rates +Ġad joining +ĠF older +roll ment +ven ient +Ġu ber +y i +Ġwh iff +ĠJu ven +ĠB orough +net te +Ġb ilingual +ĠSp arks +ph thal +man ufact +Ġt outing +ĠPH I +Ke efe +Rew ard +Ġinf all +ĠTem per +typ ically +ĠNik ol +Ġregular s +Ġpseud onym +Ġexhib itions +Ġbl aster +Ġ40 9 +w arming +Ġrever ber +Ġrecip rocal +Ġ6 70 +ip ient +b ett +ĠBe gins +Ġit ching +ĠPh ar +Ass uming +Ġem itting +ĠML G +Ġbirth place +Ġt aunt +ĠL uffy +ĠAm it +Ġcir cled +ĠN ost +enn ett +Ġde forestation +ĠHist orically +ĠEvery day +Ġovert ake +79 2 +Ġn un +ĠLuc ia +Ġaccompan ies +ĠSe eking +ĠTr ash +an ism +R ogue +Ġnorth western +ĠSupplement al +ĠNY U +ĠF RI +ĠSat isf +x es +5 17 +Ġreass ured +Ġspor adic +Ġ7 01 +Ġmed ial +Ġcannabin oid +Ġbarbar ic +Ġep is +ĠExplos ive +ĠD ough +Ġuns olved +Support ed +Ġacknowled gment +sp awn +Ġkit chens +Ġ- = +talk ing +ic ist +ĠPeg asus +ĠPS U +Ġphot on +ĠAuthent ication +R G +@# & +76 2 +ĠCl air +Ġdi aper +Ġbr ist +ĠProsecut ors +ĠJ em +6 28 +ĠEvery where +ĠJean ne +equ ality +ãĥ© ãĥ³ +object s +ĠPel icans +Ġ39 2 +Ġbl u +b ys +ĠA go +Ġinstruction al +Ġdiscrim inating +ĠTR AN +ĠCorn el +ag os +Ġty re +Ġas piration +ĠBrid gewater +": - +! ". +ĠEn s +ĠCoc o +P ie +Ġdet ach +ĠC ouch +Ġphys ique +ĠOccup ations +osc opic +en ough +B uzz +App earance +Y P +Ġrac er +Ġcompl icity +r pm +T oy +Ġinterrupt s +ĠCat alyst +Ġut ilitarian +imp act +Ġsp aghetti +Ġp orous +Ġeste emed +Ġinc iner +ĠI OC +7 48 +Ġesp resso +ĠSm ile +abil ia +6 35 +Ġmathematic ian +Ġ4 24 +ĠK L +ĠH IP +Ġover heard +ĠT ud +ĠT ec +Ġqu izz +Ġfl attering +Ġcon n +âĢ İ +Ġatt aches +ĠR OS +ĠAC S +Ġt cp +ĠSh ame +sk ip +res pected +ĠTrin idad +gr ain +Ġfooth old +ĠUnch arted +ĠJul io +z l +av ored +ĠAn xiety +er rors +ĠCent auri +its ch +D addy +Ġclutch ing +ĠIm plement +ĠGut ierrez +Ġ7 60 +Ġtele portation +end ra +Ġrevers ible +st ros +Ad venture +08 3 +Ġliber ating +Ġas phalt +ĠSp end +AR DS +im sy +PR ES +ĠEmer ging +Ġwild fires +Ġtechn ologically +Ġem its +ĠART ICLE +Ġirregular ities +Ġcher ish +çī Ī +Ġst ink +ĠR ost +Econom ic +Ġcough ing +ĠMcC ann +pro perties +ilant ro +Ġreneg oti +Trans lation +Ġin quest +ĠGra pe +oot ers +gu i +ĠSwords man +ace ae +h itting +Ġr c +Ġexert ed +ĠS AP +it ent +Ġperil ous +Ġobsc urity +Ġassass inate +Ġab original +Ġresc uing +ĠSh attered +lock ing +all ion +Ch anging +ĠHar rington +ĠB ord +ĠAfgh ans +Jam ie +aret z +ĠAugust us +Ġ38 6 +8 30 +Ġj og +ok ingly +Tr igger +ĠH OR +Stat istics +Ġviewers hip +Ġadd itives +h ur +Ġmaxim izing +ĠR ove +ĠLou ie +ĠBuck et +ĠCHR IST +ou sel +Ġstre aks +ir ted +Ġt ert +Ġcolonial ism +Ġbur ying +y k +Cond ition +ĠDPR K +By Id +75 1 +âĹ ¼ +Ġwor risome +Ġvoc ational +sl ice +Ġsa ils +ĠCorrection al +95 4 +Ġt ul +K id +l uster +Ġfam ilial +ĠSp it +ĠEp iscopal +Specific ally +ĠVol cano +run s +q s +Ġve tted +Ġcram med +t rop +here r +Thank fully +Ġper cussion +Ġor anges +Ġround up +Ġ4 99 +x ious +Char acters +ĠZion ism +ĠR ao +ÃĽ ÃĽ +W F +Ġunintention al +ONE Y +Gr ab +Com mercial +Ġglut amate +ĠMcK enna +ru ciating +ning ton +ih u +Ch an +ĠSw ap +Ġleaf lets +Ġfunction ally +er ous +F arm +Ġcal oric +ĠLiter ally +con cert +Ġshe nan +Ġrep aid +ey es +Ġbas hing +ĠG orge +Ġcollabor ations +Ġun account +itch ie +Ġteam work +pp elin +Ġpip ing +Ġmin ced +Ġd iam +ri eg +Ġmasc ara +Ġsuck er +ĠMo ons +App s +ĠPe ck +Ġper v +ĠFl oat +o ley +ĠN ish +im ize +Ġarom atic +u in +end ish +! / +ĠB icycle +ĠAS IC +ile ged +ĠQuad ro +ios yn +Ġlock out +ĠW ink +SP EC +Attempt s +Ġseed ed +red o +ias is +Ġsn ag +ãĥķ ãĤ© +ãĤ ¶ +Ġground ing +Ġrelie ver +Ġfrivol ous +ĠG ifts +ĠF aces +Es pecially +Ġmicrobi ome +im ag +ĠSch l +ĠP les +ĠBle ach +ĠIr win +ĠE aton +ĠDisc iple +Ġmultipl ication +Ġcoer ced +Ġ4 19 +st h +E vil +B omb +Ġex orc +Ġstag gered +L ESS +Ġinert ia +ĠED IT +Ġgo b +Tr aditional +Ġclass y +Lear y +ĠP AGE +yr s +Ġtrans porter +Ġmat ured +Ġhij ab +Ġbi ome +Where as +Ġex termination +ĠT ues +ĠT akeru +ĠAud rey +er ial +ĠAd en +aff les +Ġnarciss istic +ĠB aird +UT F +I re +ĠCon nie +Ch amp +Ġwhis pering +ĠH att +D K +Ġdis infect +Ġdeduct ed +Ġpart ake +Ġdown grade +ĠEs ports +ĠContin uing +Ġdemocr atically +icro bial +itt a +Ġlim estone +Ġexempt ed +ĠFren zy +H erm +7 28 +Ġfled gling +Met a +765 61 +69 3 +% : +w ake +5 26 +ĠDis cipline +Ġvirgin ity +ĠLeg ions +ĠFrank ie +int ent +Ġrest rooms +ĠRou ter +da q +Ġobjection able +âĨ ij +w ark +ĠRah ul +g ain +activ ation +abs olute +ĠAccess ed +Ġ24 00 +ogg les +Ġsecond ly +ĠDEF ENSE +Ġpost age +wra pper +sh arp +7 29 +Ġcommun icates +Ġadd on +ĠMil itia +H ong +Ġsl umped +ĠJP EG +ĠI car +ad ish +68 1 +Ġmaj esty +ĠWolf gang +ĠEl astic +u per +Ġv iz +Ġunconscious ly +ĠST D +ĠS ass +Ġflower ing +ĠHel ic +ĠDra per +ĠAm ateur +Ġman ure +Ġdis ingen +ĠLe i +br ing +9 49 +Ġinhib ited +Ġhead quartered +Ġen igmatic +�� � +Ġred ress +R H +Ġratt led +Ġd iction +l io +ĠT BA +ĠSN AP +C alling +Ġfasc ists +ĠD ove +iew icz +0 36 +Ġco asts +ĠR ect +Ġ) ] +L ot +6 29 +ĠS EM +ĠPeters en +ĠExpl ain +ĠBo ards +ĠBe zos +ĠJ ournals +Ġ20 24 +p arser +Ġmist rust +Ġgr ate +ĠL ocked +bo a +S aint +g aming +Ġvow el +in ately +bl ow +All ah +Ġun matched +Ġb ordering +ĠExp end +n r +Or acle +rou ch +Ġcont iguous +ac us +Ġdist raught +58 1 +Ġanat omical +O X +ap ixel +8 33 +ĠPL US +Ġres usc +Ġab iding +57 3 +Ġvac ancies +Em ily +Ġhyp othal +ĠWer ner +ĠWe e +ĠDJ s +5 13 +Ġwitch craft +Ġac upuncture +ent ary +benef it +Product s +ĠP SP +ĠMP G +ĠJ inn +ĠJ arrett +Ġ4 45 +ĠIm aging +ĠP yth +Fin ish +Ġte x +Ġjuven iles +Ġhero ism +Ġdoubt less +ĠA ki +ĠT end +ĠPatri arch +Ġbit ters +ĠTele communications +it atively +ag na +Ġr g +ĠS OLD +Ġcomp ulsion +ĠN asa +ĠKath ryn +Ġmillion aires +Ġintrins ically +Ġbolst ered +time out +fl o +Ġtut or +p our +Stat ement +Ġ{ * +ĠRud olph +ĠKimber ly +rog ens +adi q +] + +Ġindign ation +Ġfract uring +ĠRe leases +ĠGr ain +pro tein +L ago +Ġvac ations +Ġboot ed +ĠTH REE +ĠH G +oresc ence +Ġt f +Ġso ar +iosyn cr +Ġgl ances +ĠSp oon +ĠJ ury +ĠCow boy +Ġcreat ively +Hig her +Ġsolic itor +Ġhaw k +ac io +89 6 +Ġsuperf lu +Ġbombs hell +ct ure +Ġbroker age +Ġraid ing +Ġf rench +Ġang led +Trans action +ĠGen ocide +u pe +ĠHait ian +57 2 +! : +Ġunwitting ly +iter ator +sc roll +Ġtall ied +Ġbi omedical +ĠC ARD +Ġe uphem +Ġbrain storm +a quin +K o +Mic helle +ĠR unes +ĠBall istic +ud ers +Ġmod esty +ĠiP ads +ĠEzek iel +Y E +Ġstars hip +Ġpower fully +Ġper l +ĠSh ade +ĠQu art +ĠE EG +Ġfisher man +OS ED +ĠTyp ical +df x +Ġmes hes +Ġet ched +worth iness +Ġtopp led +Ġ3 96 +or ius +We iss +Ġmy sql +ĠVal halla +Ù Ĵ +le asing +Ġrec omp +rap nel +S el +04 3 +Ġder ailed +ĠGu ides +IR T +Ġde human +ĠBritt any +" )) +Ġex claim +Ġb alk +Ġ8 40 +CLA IM +int el +L AB +Ġpe gged +Ġast roph +sm oking +Ġrig ging +Ġfix ation +Ġcat apult +ins ide +ĠC ascade +ĠBolshe vik +G aza +Dep th +Ġloud spe +Ġalmond s +me yer +l eness +j en +f resh +Ġunbeat en +ĠSqu id +ĠPres umably +Tim er +B W +Ġro sters +Ġell ipt +ĠHar riet +dat abase +ĠMut ual +ĠComm odore +uk ed +kn ife +ĠCOMM UN +h ya +Ġmel ts +arch ives +Ġrat ification +Ġmultip lying +Ġinter oper +Ġasc ert +w ings +ver ting +ĠScorp ion +ay e +ĠPorts mouth +ĠM TA +n it +iaz ep +Ġqu arantine +Ġslides how +Ġcent imeters +Ġsyn opsis +Ġsp ate +th irst +Ġnom inating +ĠMel vin +Pre view +Ġthro b +Ġgener ational +ĠRad ius +rest ling +put able +aw ar +N ECT +Ġunlaw fully +ĠRevel ations +Wik ipedia +sur v +Ġeye ing +ij n +ĠF W +Ġbr unt +Ġinter stellar +Ġcl itor +ĠCroat ian +ĠCh ic +ev a +ĠDis app +ĠA kin +iner ies +d ust +Interest ed +Ġgen esis +ĠE ucl +ö n +p icking +Ġmut ated +Ġdisappro ve +ĠHD L +Ġ6 25 +Ì ¶ +c ancer +Ġsqu ats +Ġle vers +Disc uss += ] +D ex +ĠVIDE OS +A UD +Ġtrans act +ĠKin ect +ĠK uala +ĠC yp +7 47 +Ġsh attering +Ġarsen ic +ĠInt ake +ĠAngel o +ĠQu it +ĠK he +Ġ18 93 +M aker +0 29 +ĠPain ting +Dis able +9 16 +Ġanal ges +Ġtact ile +Ġprop hes +Ġd iced +ĠTravel s +ĠHe ader +ĠClub s +Ass istant +Ġinc rim +Ġd ips +Ġcruc ifix +ĠShan ahan +ĠInter pret +Ġ40 90 +al ogy +abb a +Ġsimul ac +hus band +S IM +Ġrecy cle +uc er +ed ged +Ġre naissance +ĠBomb ay +Cath olic +ĠL INE +ĠCl othing +re ports +Ġpl aus +Ġd ag +ĠM ace +Z I +Ġintr uder +ĠVeter inary +g ru +Ġsne aky +ĠS ie +ĠC innamon +P OSE +Ġcou rier +ĠC NS +Ġemanc ipation +s it +Ġplay through +ĠFac ilities +v irt +ĠG auntlet +Thom pson +Ġunbeliev ably +Param eters +Ġst itching +ign e +ĠTH ESE +Priv acy +Ġshenan igans +Ġvit ri +ĠVal id +59 1 +Ń · +ĠProt otype +ink a +SC P +ĠT id +è Ī +old ed +Ġindividual ity +Ġbark ing +Ġm ars +ĠW D +Ġ8 20 +Ġt ir +Ġsl apping +Ġdisgr untled +ĠAng ola +ri us +ĠTorn ado +ĠTh urs +Ġcapt cha +Ġang st +ĠP og +ĠAssass ins +ĠAd idas +Ġjoy ful +Ġwh ining +Emer gency +Ġphosph orus +Ġatt rition +oph on +ĠTimber wolves +ĠJ ah +ĠBr inging +ĠW ad +ĠEn sure +oh l +ĠX ie +omm el +c mp +Ġz ipper +Ġrel at +ĠCor ridor +m ilo +T ING +Av g +Ġcro pped +] } +Ġr aged +ĠLump ur +ĠGuer rero +our ke +N ut +Ġoff sets +og lu +dr m +Ġmort als +lat able +Ġdismiss ive +ä¸ ī +Ġthro ats +Ġchips et +ĠSpot light +Catal og +art ist +G b +Ġch illy +Ġst oked +Ġ3 74 +W ard +L atin +Ġf iasco +Ġble ach +Ġb rav +Enh anced +Ġin oc +ĠFior ina +_ > +Ġle ukemia +Ġel uc +Ġannoun cer +ĠLith uan +ĠArm ageddon +å ĩ +Len in +ĠR uk +Ġpe pp +ĠRom antic +ĠP IT +ĠInter stellar +ĠAt kinson +R aid +J s +Go al +C ourse +Ġvan ishing +es ley +ĠR ounds +Els a +59 3 +Ġredund ancy +ĠST AND +Ġprop hetic +Ġhabit able +ry u +Ġfaint ly +M ODE +Ġfl anked +IR C +Aw esome +Ġsp urious +ĠZ ah +ĠMS G +Ġsh ading +Ġmotiv ational +ĠSant ana +ĠS PR +Ġexc ruciating +om ial +ĠM iko +ĠLe opard +A byss +Ġ[ | +d irty +Ġbath s +Ġdem oral +and re +P B +Ġun ification +Ġsac rament +Ġ[ & +Ġpric eless +Ġgel atin +Ġeman ating +ĠAll aah +98 6 +Ġout burst +Ġer as +ĠX VI +ĠSP I +O tt +ĠLaz arus +PL IED +F lying +blog s +W isconsin +R aven +Ġreb ate +Ġcreep s +ĠSp an +ĠPain ter +ĠKir a +ĠAm os +ĠCor vette +Cons umer +ĠRec over +ck i +Ġpes ky +ĠIn vention +Compan ies +Ġchalleng ers +ad emic +ĠUkrain ians +ĠNeuro log +ĠFors aken +Ġent rants +Ġemb attled +Ġdef unct +ĠGlac ier +Ġpo isons +ĠH orses +m akes +ĠD irt +Ġ4 23 +hh h +ĠTrans formation +QUI RE +................ .. +Ġtrave ller +ĠSe xy +ĠK ern +ip olar +Ġransom ware +oooooooo oooooooo +E c +rub y +Prof essional +ĠOut break +arg ument +G rey +ĠFif a +ĠCH O +ĠFOR M +ĠAm trak +- [ +Ġcr adle +Ġantioxid ants +ãģ®å ® +7 36 +ĠNAS L +ĠContribut ions +Ind iana +ĠST EP +C SS +Ġsal ient +Ġall ocations +yr ights +Ġm ashed +ĠCut ter +Sex ual +Ġp ounded +Ġfan base +Ġc asc +ĠTrans parency +Ġanaly tic +ĠSummon er +× ŀ +ĠAD C +det ail +Ġvan quished +Ġcr abs +ar ie +Dest roy +ĠS ack +Ġtrans istor +Al abama +ĠK oen +ĠFisher ies +c one +Ġannex ed +ĠM GM +es a +Ġf aked +ĠCong ratulations +Ġhind ered +Ġcorrection al +ĠI TV +lee ve +Ġin appropriately +lic ks +Ġtresp ass +Ġp aws +Ġnegoti ator +ĠChrist ensen +lim its +ĠDian ne +Ġeleg ance +ĠContract s +an ke +Ob j +Ġvigil ance +Ġcast les +ĠN AD +ĠHol o +Ġemph atically +ĠTit us +ĠServ ing +ĠRich ie +ĠP igs +5 68 +Ġanim osity +ĠAtt ributes +ĠU riel +M Q +my ra +ĠApplic ant +Ġpsychiat rists +ĠV ij +ĠAb by +ag ree +P ush +Ġk Wh +hib a +Ġinc ite +ĠWe asley +ĠTax i +minist ic +hy per +ĠF arn +Ġ6 01 +ĠNation wide +F ake +95 2 +Ġma ize +Ġinteract ed +Ġtransition ed +Ġparas itic +Ġharm onic +Ġdec aying +Ġbas eless +ns ics +Ġtrans pired +Ġabund antly +ĠFore nsic +Ġtread mill +ĠJ av +ab and +Ġssh d +Ġfront man +ĠJak arta +oll er +dro ps +ĠSERV ICES +rompt u +oph ical +h ospital +bled on +6 45 +Ġmid range +ĠEV ENT +cul ated +raw led +Ġper ched +Ġover board +ĠPe el +ĠP wr +ĠCar th +ĠCOM PLE +co e +sh all +Ġdeter rence +M ETHOD +ĠAbs ent +M EN +Ġs ill +ĠLE VEL +Y ork +Ġsin ners +ĠOP EC +ĠN ur +ĠDesign s +se lection +Ġunw orthy +CH A +Ġstreng thens +88 3 +ed ly +Ġslic ing +Ġmal nutrition +Ġfilm making +ĠPol k +ur ated +Ġ4 21 +bre akers +!' " +Ġwet lands +ĠDisc rimination +Ġallow able +Ġste ered +ĠSic ily +S AM +Ġmust ache +Ġm ids +Ġcl ipped +Ġcirc ulate +Ġbr ittle +ĠBuild ings +ra ised +ĠRound up +Ġwealth ier +Ġoverw rite +Ġover powered +ĠGerr ard +s ites +PD ATED +Ġacute ly +ĠGam ble +Ġp im +ĠK us +Typ ically +De ploy +ĠMoroc can +p otion +com be +Ġvigil ante +Ġ36 3 +St ew +ĠB agg +Ġres ided +ĠSp o +Ġrem nant +Ġempt iness +br ainer +Ġout patient +pri ority +Ġle ptin +ĠPay ton +ĠGle aming +ĠS hed +ĠPol o +ĠMormon ism +rest ricted +arl ane +w x +Ġcreat ine +ĠAn on +ĠST UD +ĠJ UL +ĠT ee +5 28 +08 9 +Ġhat ched +Dis patch +ĠCompos ite +Ġ45 1 +p uff +ĠX COM +ĠOr n +ĠTH ANK +END ED +ĠAshe ville +Ġà ľ +Ġman go +ĠS lightly +world ly +ĠW ander +ĠExp and +ĠCh r +M ist +Ġorthodox y +ĠUN ESCO +reg ate +Else where +k ie +ir led +Ġtopp le +Ġadopt ive +ĠLeg s +d ress +ĠS agan +b are +ĠGl ou +Cr unch +Ġhelp ers +Ġchron ically +ĠH uma +1 0000 +Ġaccommod ating +äº Ķ +Ġwrink les +Ġdod ged +four th +Ġpre con +Ġcompress or +ĠK are +Ġev ict +ĠWar wick +im ar +Ġmodern ization +Ġband wagon +Ġref uted +Ġnet ted +ĠNa ples +ĠGen ie +per ors +Ġfield ed +Ġde re +ĠPar ables +le es +Ġtr out +asp ers +Ġn ihil +Ġhapp iest +Ġflo ppy +ĠLo ft +ĠHe ard +Ġun ison +Ġl ug +ĠRed mond +class ic +Supp orters +SH IP +G MT +Ġfue lled +ç IJ +Ġd d +ĠEmin em +Ġ18 97 +NY SE +Ġsecret aries +ĠF IA +ĠCanaver al +F avorite +Ġp omp +Ġdetain ee +ers hip +aim on +i our +ĠA pex +Ġplant ations +am ia +ac ion +R ust +Ġtow ed +ĠTru ly +5 77 +Ġshel tered +r ider +W o +Ġl air +ĠInt elligent +impro ve +m atically +Ġet iquette +ad ra +all o +ĠJun o +any thing +ĠStru ggle +ĠPred ict +ĠGr imes +ĠAMER ICA +ct x +ĠSit uation +W OOD +Ġsol uble +me ier +Ġintoler able +ang ering +Ġun interrupted +Ġtool tip +Ġinterrog ated +Ġgun ned +ĠSne ak +æŃ ¦ +Ġt ether +Ġcr umble +L ens +Ġclust ered +ĠSy l +ĠHas an +Ġdystop ian +w ana +Ġjoy stick +ĠTh ib +amm u +Tom orrow +5 46 +Ġoverc ame +Ġminim ized +cept or +Run ner +ENG TH +ĠBrend a +ĠAchieve ments +Ġtor ches +Ġrapp ort +ĠInvestig ator +ĠHand ling +rel ation +g rey +8 15 +Ġk cal +ĠComm ands +d q +Ġcur ls +Ġbe arer +Ġcyn icism +it ri +ĠUse ful +B ee +D CS +Ġab ras +P ract +BIL ITIES +7 12 +Ġdebug ger +Ġdebt or +ĠL ia +ĠK ers +Ġexacerb ate +ĠSt acy +ĠB land +ĠSc enes +Ġbranch ing +âĸĪâĸĪâĸĪâĸĪ âĸĪâĸĪâĸĪâĸĪ +ape ake +Ġs alsa +Ġmish and +ĠKon ami +ĠN ib +Ġanecd ote +Ġagree able +Ï ī +ĠNath aniel +ĠHe isman +ĠB eware +Ġ18 86 +spect ive +69 1 +5 22 +Ġinhib its +Ġhas hing +Ġ18 89 +å° Ĩ +v ich +P ure +Ġsolid ly +Ġaspir in +im aru +Ġstreet car +ĠU CS +ĠJ udd +Ġflash backs +p ins +Ġ14 40 +ĠUN HCR +ĠSym ptoms +T IT +5 38 +F ra +% ); +Ġo oz +Ġcur few +Ġcal med +Ġparticip ates +Te X +Ġnons ensical +Ġfull back +ĠDe L +mon key +h ari +Ġmetabol ites +Ġloot ed +ĠAL WAYS +ĠB CC +L t +oc het +B one +Ġveto ed +Ġg cc +ĠCL ICK +Ġ18 88 +s af +Ġstiff ness +Ġlow ly +ĠGe h +vers on +ors et +Ġun foreseen +Ġan esthesia +ĠOpt ical +Ġrecon structed +ĠT up +sh ows +NEW S +ĠNewsp aper +ĠA SA +ter a +N umbers +Ġinexpl icable +× ij +Ġhard ness +unt arily +ĠA cer +grad ient +ARD IS +Ġwood land +Ġmetaph ors +ĠWem bley +ĠPa vel +phil is +Ġre writing +Ġpercept ual +Ġ10 70 +worm s +ĠDown s +Ġunsur prisingly +Ġtag ging +fl ame +Ġlit res +Ġboun ces +ĠB abe +sh ut +Ġoverd oses +ĠShe ila +ĠCh au +ĠBl ess +Capt ure +ĠSign ificant +ĠSc ion +Ġ38 9 +ĠMc H +ĠTitan ium +ĠMe al +amed a +ag ents +agg ressive +B illy +76 3 +ĠS aying +DER R +it one +Coll ins +B ound +Ġbol ted +ĠDM CA +95 3 +Ġun iqueness +Ġep igen +un ci +ant am +Ġreck oning +ch airs +OG R +ĠSen egal +Ġ18 62 +re levant +Ġ ¯ +Ġpharm acies +ĠG eral +v ier +Y an +OR PG +Ġrab id +b ending +ĠUN ITED +Ġ4 65 +As sembly +Ġwe ep +Ġbe hest +ĠMother s +ĠJ ace +h id +Ġwh irlwind +ĠUN IVERS +Ġut opian +Ġkidn ap +Ph ilipp +K in +89 3 +Ġlivest ream +ĠM ISS +Ġsub versive +ĠTechn iques +ĠJUST ICE +ĠB ASE +Ġ38 7 +Ġassail ants +ĠHard core +Ġsprink led +ĠP se +é ļ +print ed +ĠH au +OR GE +ĠT OUR +Ġl aced +Ġit ch +G iving +Ġport ed +78 1 +//////////////// //////////////// +bre eding +Ġlog ger +ĠH OL +inn ie +First ly +Ġembry onic +Ġdeleg ated +p ai +O IL +Ġcentr ally +ĠR x +ĠSc outing +D utch +Ġhe reditary +ĠCru iser +s at +5 29 +ĠMar riott +other mal +Ġprohib itions +E arn +ĠSt ab +ĠColleg es +ĠBel ief +st retched +ĠL H +ĠEntity Item +C IA +Ġun rem +Ġlaure ate +Ġdenomin ations +sum mary +h ler +S pect +ĠK laus +ĠBe ans +Ġins ur +ĠPA X +Ġfield er +ĠV et +ĠSp arrow +z ie +ĠS Q +ĠMond ays +ĠOff line +ĠLer ner +ĠExt ensions +Ire land +Ġpatron age +Ġcontrast ed +ĠMan ia +h irt +Mos cow +Ġcondem ns +ĠAn ge +Ġcomp osing +ĠPe pe +ĠP addock +Ġheter ogeneity +Ġide ologically +Ġf ishes +Ġcur sing +ĠR utherford +ĠFlo ating +ĠAm elia +Te a +Syn opsis +Ġstun ts +Ġbe ad +Ġstock ing +ĠM ILL +ob ook +mass ive +\ < +Ġh ump +ĠPref erences +Engine Debug +ge ist +ĠNiet o +ome ver +ish y +eval uate +col onial +Altern ative +ĠGo Pro +ĠV ortex +ĠNET WORK +ans ky +Sec ure +ĠTh rust +Sn ake +Ġparcel s +Ġsam urai +Ġactress es +N ap +M F +ifer ation +Be er +5 23 +ĠI ly +oint ment +P ing +Ġstri ped +ĠMell on +oss ession +Ġneut ron +end ium +Ġa ph +ĠFlav oring +Ġ38 3 +Ġrespons iveness +ĠJ indal +ĠHitch cock +Den ver +ĠDRAG ON +sm anship +ĠDu pl +Ġs ly +Ġweb cam +ĠTw ain +ĠDar ling +ili ate +cons umer +D IT +Ġnames ake +Ġun orthodox +Ġfun er +ĠPL oS +ĠCONTR OL +ozy g +ogl obin +F ACE +ER G +ĠD ia +ĠF iesta +ce le +0 34 +Ġencl ave +âĸ¬ âĸ¬ +on ement +al ist +M and +Ġhome grown +ĠF ancy +Ġconcept ions +ĠCont ains +ure en +Ġreiter ate +Ġme ager +Ġinstall ments +Sp awn +6 27 +Ġphot oc +ĠCab rera +ĠRos enthal +ĠLans ing +is ner +Ġinvest s +ĠUFO s +EX P +Hard ware +Ġtr agically +Ġconced es +ie ft +ch am +bor gh +ĠSch r +ĠMel anie +ĠH oy +Ġvisit ation +Ġid iosyncr +Ġfract ions +Ġfore skin +ob os +Ġpo aching +ĠVI EW +Ġstimul ates +ĠG ork +can on +M IC +ĠNem esis +ĠInd ra +ĠDM V +Ġ5 29 +Ġinspect ing +Ġgrand ma +ĠW hedon +ĠSh ant +ĠP urg +ik an +ĠT eg +ĠCL R +z ac +Vict oria +ĠVer ify +ion ics +Ġpart ying +ĠM ou +col our +Ġtestim onies +l ations +Ġpress uring +hi ro +ac ers +Ġf id +ang ler +ĠCS I +Ġhere after +Ġdiss idents +report ing +iph any +che v +Ġsol itude +Ġl obe +Ġind is +Ġcred ential +re cent +ad ult +ĠNir vana +ĠFranch ise +L ayer +H yp +ĠBerks hire +Ġwill s +t if +Ġtot em +ĠJud ah +rep air +Inst ant +5 48 +Ġemb assies +Ġbott leneck +Ġb ount +Ġtyp ew +ĠAl vin +j ing +im ilar +R ush +Ġbr im +ĠHEL P +A im +] ' +Ġpass ively +Ġbound ed +ĠR ated +Ġcriminal ity +Ġbiom ark +Ġdisp atcher +ĠTow ards +Ġ+ ++ +right eous +f rog +ĠP anc +C arter +0 32 +æ© Ł +Ġult raviolet +ĠLic ensed +ĠT ata +ĠBl essing +ĠG AM +Ġchem ically +ĠSe af +ĠRE LE +ĠMerc enary +capital ist +Ġform ulations +Ġann ihilation +ĠVer b +ĠAr gon +Ġun loaded +Ġmorp hed +Ġconqu ering +back er +I ELD +Ġtheft s +Ġfront runner +ĠRoy ale +ĠFund amental +el ight +C hip +necess ary +ay n +ĠSl ip +Ġ4 48 +cern ed +P ause +Ġshock ingly +ĠAB V +Ġcomp osure +7 33 +ĠMotors port +ah ime +Mur ray +M ach +Ġgr ids +Ġdeb ian +Ġfurther more +Ġdexter ity +ĠCollect ions +os lov +il age +b j +ĠMont eneg +Ġstrut Connector +Ġmassac res +Ġbrief s +fet ched +uv ian +ol ition +Fail ure +emon ic +Ġfl ared +Ġclaim ant +Ġc ures +Ġgive aways +ĠSubst ance +al ions +Ġcr inge +ĠK ul +Ġarist ocracy +ĠUl ster +ol ated +h ousing +ĠM IS +Ġgl ared +ĠWil helm +ne eds +lam bda +build ers +ĠV IS +Ġradi ator +ĠGhost busters +Ġ4 36 +act ual +Ġher ds +ç a +watch ing +Ġcounter ing +Ch arge +Ġchar red +Ġwar heads +Ġiod ine +ĠM acy +04 1 +Ġdepart ures +ĠS ins +Ġdy ed +ĠConcept s +g ado +7 13 +Ġquot ations +Ġg ist +ĠChrist y +Ġant igen +ĠHem p +ĠD rawn +ĠB arg +ez vous +Ġp aternity +Ġar du +ĠAnch orage +ĠR ik +Ġover loaded +ĠUs ername +ĠTam my +ĠN au +ĠCell ular +Ġw aning +Ġrod ent +ĠWor cester +il ts +ĠT ad +Ġdwell ings +Ġbull ish +4 31 +Ġretali ate +Ġmig raine +ĠChev ron +CH ECK +Ġdon key +c rim +SP A +ĠAn alog +Ġmarqu ee +ĠHa as +B ir +ĠGD DR +ĠDownload s +Ġwill power +ĠFor th +ĠRecord ed +Ġimp ossibility +ĠLog ged +ĠFr anks +ĠR att +in itions +Ġclean ers +Ġsore ly +Ġflick ering +ĠEx amination +c atching +allow een +Ms g +Ġdun no +F a +Ġdys ph +c razy +.' '. +Ġmain line +Ġc s +Ġp tr +ĠW ally +ig un +95 1 +ĠBig foot +f ights +Ġretrie ving +J r +Ġdupl ication +ĠExpl an +Ġrel ational +Ġqu aint +Ġbisc uits +Ġad o +Ġsh udder +Ġantid ote +blood ed +ks h +Ġsa uces +Ġrein vest +Ġdispens ary +ĠD iver +Ġ9 000 +stud ent +Ġin separ +esc ap +Ġtodd lers +ĠGP IO +ĠAss ignment +head ers +Ġlack luster +Ġab ack +95 6 +Ġtool bar +7 45 +Ġo ust +Ġcontempl ation +ĠPRES IDENT +Ġ4 58 +==== == +Ġguarantee ing +ĠHe ist +ĠCann es +Ļ ½ +Ġcollabor ator +ĠAm p +Ġg ou +ĠSH ALL +st ories +78 3 +Ġmobil ized +Ġbro od +ĠL U +ĠðŁ ij +Ġref in +ĠAnthrop ology +v ind +ill i +Ġwarrant ies +ĠB abel +Ġsw ath +Ġc aches +Ġantagon ists +art ifacts +Ġhot ly +ĠSt arts +ĠG ö +z ag +!! !!! +Ġsc ourge +Ġcons piring +ru its +re verse +ĠShe en +ĠJes uit +ĠGiov anni +ad ies +Ġbutt ocks +ear cher +ac an +Ġvolley ball +Ġshroud ed +Ġscore board +b ats +ĠI PM +Ġass es +Ġde regulation +ĠTe legram +ĠReb oot +Ġ7 000 +ĠCan ary +Ġk ernels +ĠFranç ois +ĠD uff +ĠP on +ĠLe ica +ĠGar min +Ġor phans +ĠClaud ia +Ġcal endars +ĠLe ilan +ent o +R ocket +Ġbr unch +ĠHaw king +ain ers +Ġsens ibilities +Ġk W +ĠK and +Ġre claimed +Ġinteresting ly +× © +rom y +J M +ĠEnhance ment +b ush +Sk ip +Ġrapp ers +Ġg azing +p edia +ath lon +Rev olution +Ġsn ipers +Ġre verted +Ġconglomer ate +T erry +79 4 +Ġhars her +Ġdes olate +ĠHit man +Comm ission +Ġ( / +âĢ¦ ." +Com par +Ġampl ification +om inated +Ġreg ress +ĠColl ider +Ġinform ants +Ġg azed diff --git a/nlp_data/gpt2-vocab.json b/nlp_data/gpt2-vocab.json new file mode 100644 index 0000000000000000000000000000000000000000..1f1d9aaca301414e7f6c9396df506798ff4eb9a6 --- /dev/null +++ b/nlp_data/gpt2-vocab.json @@ -0,0 +1 @@ +{"!": 0, "\"": 1, "#": 2, "$": 3, "%": 4, "&": 5, "'": 6, "(": 7, ")": 8, "*": 9, "+": 10, ",": 11, "-": 12, ".": 13, "/": 14, "0": 15, "1": 16, "2": 17, "3": 18, "4": 19, "5": 20, "6": 21, "7": 22, "8": 23, "9": 24, ":": 25, ";": 26, "<": 27, "=": 28, ">": 29, "?": 30, "@": 31, "A": 32, "B": 33, "C": 34, "D": 35, "E": 36, "F": 37, "G": 38, "H": 39, "I": 40, "J": 41, "K": 42, "L": 43, "M": 44, "N": 45, "O": 46, "P": 47, "Q": 48, "R": 49, "S": 50, "T": 51, "U": 52, "V": 53, "W": 54, "X": 55, "Y": 56, "Z": 57, "[": 58, "\\": 59, "]": 60, "^": 61, "_": 62, "`": 63, "a": 64, "b": 65, "c": 66, "d": 67, "e": 68, "f": 69, "g": 70, "h": 71, "i": 72, "j": 73, "k": 74, "l": 75, "m": 76, "n": 77, "o": 78, "p": 79, "q": 80, "r": 81, "s": 82, "t": 83, "u": 84, "v": 85, "w": 86, "x": 87, "y": 88, "z": 89, "{": 90, "|": 91, "}": 92, "~": 93, "\u00a1": 94, "\u00a2": 95, "\u00a3": 96, "\u00a4": 97, "\u00a5": 98, "\u00a6": 99, "\u00a7": 100, "\u00a8": 101, "\u00a9": 102, "\u00aa": 103, "\u00ab": 104, "\u00ac": 105, "\u00ae": 106, "\u00af": 107, "\u00b0": 108, "\u00b1": 109, "\u00b2": 110, "\u00b3": 111, "\u00b4": 112, "\u00b5": 113, "\u00b6": 114, "\u00b7": 115, "\u00b8": 116, "\u00b9": 117, "\u00ba": 118, "\u00bb": 119, "\u00bc": 120, "\u00bd": 121, "\u00be": 122, "\u00bf": 123, "\u00c0": 124, "\u00c1": 125, "\u00c2": 126, "\u00c3": 127, "\u00c4": 128, "\u00c5": 129, "\u00c6": 130, "\u00c7": 131, "\u00c8": 132, "\u00c9": 133, "\u00ca": 134, "\u00cb": 135, "\u00cc": 136, "\u00cd": 137, "\u00ce": 138, "\u00cf": 139, "\u00d0": 140, "\u00d1": 141, "\u00d2": 142, "\u00d3": 143, "\u00d4": 144, "\u00d5": 145, "\u00d6": 146, "\u00d7": 147, "\u00d8": 148, "\u00d9": 149, "\u00da": 150, "\u00db": 151, "\u00dc": 152, "\u00dd": 153, "\u00de": 154, "\u00df": 155, "\u00e0": 156, "\u00e1": 157, "\u00e2": 158, "\u00e3": 159, "\u00e4": 160, "\u00e5": 161, "\u00e6": 162, "\u00e7": 163, "\u00e8": 164, "\u00e9": 165, "\u00ea": 166, "\u00eb": 167, "\u00ec": 168, "\u00ed": 169, "\u00ee": 170, "\u00ef": 171, "\u00f0": 172, "\u00f1": 173, "\u00f2": 174, "\u00f3": 175, "\u00f4": 176, "\u00f5": 177, "\u00f6": 178, "\u00f7": 179, "\u00f8": 180, "\u00f9": 181, "\u00fa": 182, "\u00fb": 183, "\u00fc": 184, "\u00fd": 185, "\u00fe": 186, "\u00ff": 187, "\u0100": 188, "\u0101": 189, "\u0102": 190, "\u0103": 191, "\u0104": 192, "\u0105": 193, "\u0106": 194, "\u0107": 195, "\u0108": 196, "\u0109": 197, "\u010a": 198, "\u010b": 199, "\u010c": 200, "\u010d": 201, "\u010e": 202, "\u010f": 203, "\u0110": 204, "\u0111": 205, "\u0112": 206, "\u0113": 207, "\u0114": 208, "\u0115": 209, "\u0116": 210, "\u0117": 211, "\u0118": 212, "\u0119": 213, "\u011a": 214, "\u011b": 215, "\u011c": 216, "\u011d": 217, "\u011e": 218, "\u011f": 219, "\u0120": 220, "\u0121": 221, "\u0122": 222, "\u0123": 223, "\u0124": 224, "\u0125": 225, "\u0126": 226, "\u0127": 227, "\u0128": 228, "\u0129": 229, "\u012a": 230, "\u012b": 231, "\u012c": 232, "\u012d": 233, "\u012e": 234, "\u012f": 235, "\u0130": 236, "\u0131": 237, "\u0132": 238, "\u0133": 239, "\u0134": 240, "\u0135": 241, "\u0136": 242, "\u0137": 243, "\u0138": 244, "\u0139": 245, "\u013a": 246, "\u013b": 247, "\u013c": 248, "\u013d": 249, "\u013e": 250, "\u013f": 251, "\u0140": 252, "\u0141": 253, "\u0142": 254, "\u0143": 255, "\u0120t": 256, "\u0120a": 257, "he": 258, "in": 259, "re": 260, "on": 261, "\u0120the": 262, "er": 263, "\u0120s": 264, "at": 265, "\u0120w": 266, "\u0120o": 267, "en": 268, "\u0120c": 269, "it": 270, "is": 271, "an": 272, "or": 273, "es": 274, "\u0120b": 275, "ed": 276, "\u0120f": 277, "ing": 278, "\u0120p": 279, "ou": 280, "\u0120an": 281, "al": 282, "ar": 283, "\u0120to": 284, "\u0120m": 285, "\u0120of": 286, "\u0120in": 287, "\u0120d": 288, "\u0120h": 289, "\u0120and": 290, "ic": 291, "as": 292, "le": 293, "\u0120th": 294, "ion": 295, "om": 296, "ll": 297, "ent": 298, "\u0120n": 299, "\u0120l": 300, "st": 301, "\u0120re": 302, "ve": 303, "\u0120e": 304, "ro": 305, "ly": 306, "\u0120be": 307, "\u0120g": 308, "\u0120T": 309, "ct": 310, "\u0120S": 311, "id": 312, "ot": 313, "\u0120I": 314, "ut": 315, "et": 316, "\u0120A": 317, "\u0120is": 318, "\u0120on": 319, "im": 320, "am": 321, "ow": 322, "ay": 323, "ad": 324, "se": 325, "\u0120that": 326, "\u0120C": 327, "ig": 328, "\u0120for": 329, "ac": 330, "\u0120y": 331, "ver": 332, "ur": 333, "\u0120u": 334, "ld": 335, "\u0120st": 336, "\u0120M": 337, "'s": 338, "\u0120he": 339, "\u0120it": 340, "ation": 341, "ith": 342, "ir": 343, "ce": 344, "\u0120you": 345, "il": 346, "\u0120B": 347, "\u0120wh": 348, "ol": 349, "\u0120P": 350, "\u0120with": 351, "\u01201": 352, "ter": 353, "ch": 354, "\u0120as": 355, "\u0120we": 356, "\u0120(": 357, "nd": 358, "ill": 359, "\u0120D": 360, "if": 361, "\u01202": 362, "ag": 363, "ers": 364, "ke": 365, "\u0120\"": 366, "\u0120H": 367, "em": 368, "\u0120con": 369, "\u0120W": 370, "\u0120R": 371, "her": 372, "\u0120was": 373, "\u0120r": 374, "od": 375, "\u0120F": 376, "ul": 377, "ate": 378, "\u0120at": 379, "ri": 380, "pp": 381, "ore": 382, "\u0120The": 383, "\u0120se": 384, "us": 385, "\u0120pro": 386, "\u0120ha": 387, "um": 388, "\u0120are": 389, "\u0120de": 390, "ain": 391, "and": 392, "\u0120or": 393, "igh": 394, "est": 395, "ist": 396, "ab": 397, "rom": 398, "\u0120N": 399, "th": 400, "\u0120com": 401, "\u0120G": 402, "un": 403, "op": 404, "00": 405, "\u0120L": 406, "\u0120not": 407, "ess": 408, "\u0120ex": 409, "\u0120v": 410, "res": 411, "\u0120E": 412, "ew": 413, "ity": 414, "ant": 415, "\u0120by": 416, "el": 417, "os": 418, "ort": 419, "oc": 420, "qu": 421, "\u0120from": 422, "\u0120have": 423, "\u0120su": 424, "ive": 425, "ould": 426, "\u0120sh": 427, "\u0120this": 428, "nt": 429, "ra": 430, "pe": 431, "ight": 432, "art": 433, "ment": 434, "\u0120al": 435, "ust": 436, "end": 437, "--": 438, "all": 439, "\u0120O": 440, "ack": 441, "\u0120ch": 442, "\u0120le": 443, "ies": 444, "red": 445, "ard": 446, "\u00e2\u0122": 447, "out": 448, "\u0120J": 449, "\u0120ab": 450, "ear": 451, "iv": 452, "ally": 453, "our": 454, "ost": 455, "gh": 456, "pt": 457, "\u0120pl": 458, "ast": 459, "\u0120can": 460, "ak": 461, "ome": 462, "ud": 463, "The": 464, "\u0120his": 465, "\u0120do": 466, "\u0120go": 467, "\u0120has": 468, "ge": 469, "'t": 470, "\u0120U": 471, "rou": 472, "\u0120sa": 473, "\u0120j": 474, "\u0120but": 475, "\u0120wor": 476, "\u0120all": 477, "ect": 478, "\u0120k": 479, "ame": 480, "\u0120will": 481, "ok": 482, "\u0120whe": 483, "\u0120they": 484, "ide": 485, "01": 486, "ff": 487, "ich": 488, "pl": 489, "ther": 490, "\u0120tr": 491, "..": 492, "\u0120int": 493, "ie": 494, "ure": 495, "age": 496, "\u0120ne": 497, "ial": 498, "ap": 499, "ine": 500, "ice": 501, "\u0120me": 502, "\u0120out": 503, "ans": 504, "one": 505, "ong": 506, "ions": 507, "\u0120who": 508, "\u0120K": 509, "\u0120up": 510, "\u0120their": 511, "\u0120ad": 512, "\u01203": 513, "\u0120us": 514, "ated": 515, "ous": 516, "\u0120more": 517, "ue": 518, "og": 519, "\u0120St": 520, "ind": 521, "ike": 522, "\u0120so": 523, "ime": 524, "per": 525, ".\"": 526, "ber": 527, "iz": 528, "act": 529, "\u0120one": 530, "\u0120said": 531, "\u0120-": 532, "are": 533, "\u0120your": 534, "cc": 535, "\u0120Th": 536, "\u0120cl": 537, "ep": 538, "ake": 539, "able": 540, "ip": 541, "\u0120cont": 542, "\u0120which": 543, "ia": 544, "\u0120im": 545, "\u0120about": 546, "\u0120were": 547, "very": 548, "ub": 549, "\u0120had": 550, "\u0120en": 551, "\u0120comp": 552, ",\"": 553, "\u0120In": 554, "\u0120un": 555, "\u0120ag": 556, "ire": 557, "ace": 558, "au": 559, "ary": 560, "\u0120would": 561, "ass": 562, "ry": 563, "\u0120\u00e2\u0122": 564, "cl": 565, "ook": 566, "ere": 567, "so": 568, "\u0120V": 569, "ign": 570, "ib": 571, "\u0120off": 572, "\u0120te": 573, "ven": 574, "\u0120Y": 575, "ile": 576, "ose": 577, "ite": 578, "orm": 579, "\u0120201": 580, "\u0120res": 581, "\u0120man": 582, "\u0120per": 583, "\u0120other": 584, "ord": 585, "ult": 586, "\u0120been": 587, "\u0120like": 588, "ase": 589, "ance": 590, "ks": 591, "ays": 592, "own": 593, "ence": 594, "\u0120dis": 595, "ction": 596, "\u0120any": 597, "\u0120app": 598, "\u0120sp": 599, "int": 600, "ress": 601, "ations": 602, "ail": 603, "\u01204": 604, "ical": 605, "\u0120them": 606, "\u0120her": 607, "ount": 608, "\u0120Ch": 609, "\u0120ar": 610, "\u0120if": 611, "\u0120there": 612, "\u0120pe": 613, "\u0120year": 614, "av": 615, "\u0120my": 616, "\u0120some": 617, "\u0120when": 618, "ough": 619, "ach": 620, "\u0120than": 621, "ru": 622, "ond": 623, "ick": 624, "\u0120over": 625, "vel": 626, "\u0120qu": 627, "\u010a\u010a": 628, "\u0120sc": 629, "reat": 630, "ree": 631, "\u0120It": 632, "ound": 633, "port": 634, "\u0120also": 635, "\u0120part": 636, "fter": 637, "\u0120kn": 638, "\u0120bec": 639, "\u0120time": 640, "ens": 641, "\u01205": 642, "ople": 643, "\u0120what": 644, "\u0120no": 645, "du": 646, "mer": 647, "ang": 648, "\u0120new": 649, "----": 650, "\u0120get": 651, "ory": 652, "ition": 653, "ings": 654, "\u0120just": 655, "\u0120into": 656, "\u01200": 657, "ents": 658, "ove": 659, "te": 660, "\u0120people": 661, "\u0120pre": 662, "\u0120its": 663, "\u0120rec": 664, "\u0120tw": 665, "ian": 666, "irst": 667, "ark": 668, "ors": 669, "\u0120work": 670, "ade": 671, "ob": 672, "\u0120she": 673, "\u0120our": 674, "wn": 675, "ink": 676, "lic": 677, "\u012019": 678, "\u0120He": 679, "ish": 680, "nder": 681, "ause": 682, "\u0120him": 683, "ons": 684, "\u0120[": 685, "\u0120ro": 686, "form": 687, "ild": 688, "ates": 689, "vers": 690, "\u0120only": 691, "oll": 692, "\u0120spe": 693, "ck": 694, "ell": 695, "amp": 696, "\u0120acc": 697, "\u0120bl": 698, "ious": 699, "urn": 700, "ft": 701, "ood": 702, "\u0120how": 703, "hed": 704, "\u0120'": 705, "\u0120after": 706, "aw": 707, "\u0120att": 708, "ov": 709, "ne": 710, "\u0120play": 711, "erv": 712, "ict": 713, "\u0120could": 714, "itt": 715, "\u0120am": 716, "\u0120first": 717, "\u01206": 718, "\u0120act": 719, "\u0120$": 720, "ec": 721, "hing": 722, "ual": 723, "ull": 724, "\u0120comm": 725, "oy": 726, "old": 727, "ces": 728, "ater": 729, "\u0120fe": 730, "\u0120bet": 731, "we": 732, "iff": 733, "\u0120two": 734, "ock": 735, "\u0120back": 736, ").": 737, "ident": 738, "\u0120under": 739, "rough": 740, "sel": 741, "xt": 742, "\u0120may": 743, "round": 744, "\u0120po": 745, "ph": 746, "iss": 747, "\u0120des": 748, "\u0120most": 749, "\u0120did": 750, "\u0120add": 751, "ject": 752, "\u0120inc": 753, "fore": 754, "\u0120pol": 755, "ont": 756, "\u0120again": 757, "clud": 758, "tern": 759, "\u0120know": 760, "\u0120need": 761, "\u0120cons": 762, "\u0120co": 763, "\u0120.": 764, "\u0120want": 765, "\u0120see": 766, "\u01207": 767, "ning": 768, "iew": 769, "\u0120This": 770, "ced": 771, "\u0120even": 772, "\u0120ind": 773, "ty": 774, "\u0120We": 775, "ath": 776, "\u0120these": 777, "\u0120pr": 778, "\u0120use": 779, "\u0120because": 780, "\u0120fl": 781, "ng": 782, "\u0120now": 783, "\u0120\u00e2\u0122\u0135": 784, "com": 785, "ise": 786, "\u0120make": 787, "\u0120then": 788, "ower": 789, "\u0120every": 790, "\u0120Un": 791, "\u0120sec": 792, "oss": 793, "uch": 794, "\u0120em": 795, "\u0120=": 796, "\u0120Re": 797, "ied": 798, "rit": 799, "\u0120inv": 800, "lect": 801, "\u0120supp": 802, "ating": 803, "\u0120look": 804, "man": 805, "pect": 806, "\u01208": 807, "row": 808, "\u0120bu": 809, "\u0120where": 810, "ific": 811, "\u0120years": 812, "ily": 813, "\u0120diff": 814, "\u0120should": 815, "\u0120rem": 816, "Th": 817, "In": 818, "\u0120ev": 819, "day": 820, "'re": 821, "rib": 822, "\u0120rel": 823, "ss": 824, "\u0120def": 825, "\u0120right": 826, "\u0120sy": 827, "),": 828, "les": 829, "000": 830, "hen": 831, "\u0120through": 832, "\u0120Tr": 833, "__": 834, "\u0120way": 835, "\u0120don": 836, "\u0120,": 837, "\u012010": 838, "ased": 839, "\u0120ass": 840, "ublic": 841, "\u0120reg": 842, "\u0120And": 843, "ix": 844, "\u0120very": 845, "\u0120includ": 846, "other": 847, "\u0120imp": 848, "oth": 849, "\u0120sub": 850, "\u0120\u00e2\u0122\u0136": 851, "\u0120being": 852, "arg": 853, "\u0120Wh": 854, "==": 855, "ible": 856, "\u0120does": 857, "ange": 858, "ram": 859, "\u01209": 860, "ert": 861, "ps": 862, "ited": 863, "ational": 864, "\u0120br": 865, "\u0120down": 866, "\u0120many": 867, "aking": 868, "\u0120call": 869, "uring": 870, "ities": 871, "\u0120ph": 872, "ics": 873, "als": 874, "\u0120dec": 875, "ative": 876, "ener": 877, "\u0120before": 878, "ility": 879, "\u0120well": 880, "\u0120much": 881, "erson": 882, "\u0120those": 883, "\u0120such": 884, "\u0120ke": 885, "\u0120end": 886, "\u0120But": 887, "ason": 888, "ting": 889, "\u0120long": 890, "ef": 891, "\u0120think": 892, "ys": 893, "\u0120bel": 894, "\u0120sm": 895, "its": 896, "ax": 897, "\u0120own": 898, "\u0120prov": 899, "\u0120set": 900, "ife": 901, "ments": 902, "ble": 903, "ward": 904, "\u0120show": 905, "\u0120pres": 906, "ms": 907, "omet": 908, "\u0120ob": 909, "\u0120say": 910, "\u0120Sh": 911, "ts": 912, "ful": 913, "\u0120eff": 914, "\u0120gu": 915, "\u0120inst": 916, "und": 917, "ren": 918, "cess": 919, "\u0120ent": 920, "\u0120You": 921, "\u0120good": 922, "\u0120start": 923, "ince": 924, "\u0120made": 925, "tt": 926, "stem": 927, "olog": 928, "up": 929, "\u0120|": 930, "ump": 931, "\u0120hel": 932, "vern": 933, "ular": 934, "ually": 935, "\u0120ac": 936, "\u0120mon": 937, "\u0120last": 938, "\u0120200": 939, "10": 940, "\u0120stud": 941, "ures": 942, "\u0120Ar": 943, "self": 944, "ars": 945, "meric": 946, "ues": 947, "cy": 948, "\u0120min": 949, "ollow": 950, "\u0120col": 951, "io": 952, "\u0120mod": 953, "\u0120count": 954, "\u0120Com": 955, "hes": 956, "\u0120fin": 957, "air": 958, "ier": 959, "\u00e2\u0122\u0136": 960, "read": 961, "ank": 962, "atch": 963, "ever": 964, "\u0120str": 965, "\u0120point": 966, "ork": 967, "\u0120New": 968, "\u0120sur": 969, "ool": 970, "alk": 971, "ement": 972, "\u0120used": 973, "ract": 974, "ween": 975, "\u0120same": 976, "oun": 977, "\u0120Al": 978, "ci": 979, "\u0120differe": 980, "\u0120while": 981, "--------": 982, "\u0120game": 983, "cept": 984, "\u0120sim": 985, "...": 986, "\u0120inter": 987, "ek": 988, "\u0120report": 989, "\u0120produ": 990, "\u0120still": 991, "led": 992, "ah": 993, "\u0120here": 994, "\u0120world": 995, "\u0120though": 996, "\u0120num": 997, "arch": 998, "imes": 999, "ale": 1000, "\u0120Se": 1001, "\u0120If": 1002, "//": 1003, "\u0120Le": 1004, "\u0120ret": 1005, "\u0120ref": 1006, "\u0120trans": 1007, "ner": 1008, "ution": 1009, "ters": 1010, "\u0120take": 1011, "\u0120Cl": 1012, "\u0120conf": 1013, "way": 1014, "ave": 1015, "\u0120going": 1016, "\u0120sl": 1017, "ug": 1018, "\u0120Americ": 1019, "\u0120spec": 1020, "\u0120hand": 1021, "\u0120between": 1022, "ists": 1023, "\u0120De": 1024, "oot": 1025, "It": 1026, "\u0120ear": 1027, "\u0120against": 1028, "\u0120high": 1029, "gan": 1030, "az": 1031, "ather": 1032, "\u0120exp": 1033, "\u0120op": 1034, "\u0120ins": 1035, "\u0120gr": 1036, "\u0120help": 1037, "\u0120requ": 1038, "ets": 1039, "ins": 1040, "\u0120Pro": 1041, "ism": 1042, "\u0120found": 1043, "land": 1044, "ata": 1045, "uss": 1046, "ames": 1047, "\u0120person": 1048, "\u0120great": 1049, "pr": 1050, "\u0120sign": 1051, "\u0120An": 1052, "'ve": 1053, "\u0120somet": 1054, "\u0120ser": 1055, "hip": 1056, "\u0120run": 1057, "\u0120:": 1058, "\u0120ter": 1059, "irect": 1060, "\u0120follow": 1061, "\u0120det": 1062, "ices": 1063, "\u0120find": 1064, "12": 1065, "\u0120mem": 1066, "\u0120cr": 1067, "ered": 1068, "ex": 1069, "\u0120ext": 1070, "uth": 1071, "ense": 1072, "co": 1073, "\u0120team": 1074, "ving": 1075, "ouse": 1076, "ash": 1077, "att": 1078, "ved": 1079, "\u0120system": 1080, "\u0120As": 1081, "der": 1082, "ives": 1083, "min": 1084, "\u0120lead": 1085, "\u0120Bl": 1086, "cent": 1087, "\u0120around": 1088, "\u0120govern": 1089, "\u0120cur": 1090, "velop": 1091, "any": 1092, "\u0120cour": 1093, "alth": 1094, "ages": 1095, "ize": 1096, "\u0120car": 1097, "ode": 1098, "\u0120law": 1099, "\u0120read": 1100, "'m": 1101, "con": 1102, "\u0120real": 1103, "\u0120support": 1104, "\u012012": 1105, "....": 1106, "\u0120really": 1107, "ness": 1108, "\u0120fact": 1109, "\u0120day": 1110, "\u0120both": 1111, "ying": 1112, "\u0120serv": 1113, "\u0120For": 1114, "\u0120three": 1115, "\u0120wom": 1116, "\u0120med": 1117, "ody": 1118, "\u0120They": 1119, "50": 1120, "\u0120exper": 1121, "ton": 1122, "\u0120each": 1123, "akes": 1124, "\u0120che": 1125, "\u0120cre": 1126, "ines": 1127, "\u0120rep": 1128, "19": 1129, "gg": 1130, "illion": 1131, "\u0120grou": 1132, "ute": 1133, "ik": 1134, "We": 1135, "get": 1136, "ER": 1137, "\u0120met": 1138, "\u0120says": 1139, "ox": 1140, "\u0120during": 1141, "ern": 1142, "ized": 1143, "ared": 1144, "\u0120fam": 1145, "ically": 1146, "\u0120happ": 1147, "\u0120Is": 1148, "\u0120char": 1149, "med": 1150, "vent": 1151, "\u0120gener": 1152, "ient": 1153, "ple": 1154, "iet": 1155, "rent": 1156, "11": 1157, "ves": 1158, "ption": 1159, "\u012020": 1160, "formation": 1161, "\u0120cor": 1162, "\u0120offic": 1163, "ield": 1164, "\u0120too": 1165, "ision": 1166, "\u0120inf": 1167, "\u0120Z": 1168, "the": 1169, "oad": 1170, "\u0120public": 1171, "\u0120prog": 1172, "ric": 1173, "**": 1174, "\u0120war": 1175, "\u0120power": 1176, "view": 1177, "\u0120few": 1178, "\u0120loc": 1179, "\u0120different": 1180, "\u0120state": 1181, "\u0120head": 1182, "'ll": 1183, "\u0120poss": 1184, "\u0120stat": 1185, "ret": 1186, "ants": 1187, "\u0120val": 1188, "\u0120iss": 1189, "\u0120cle": 1190, "ivers": 1191, "anc": 1192, "\u0120expl": 1193, "\u0120another": 1194, "\u0120Q": 1195, "\u0120av": 1196, "thing": 1197, "nce": 1198, "Wh": 1199, "\u0120child": 1200, "\u0120since": 1201, "ired": 1202, "less": 1203, "\u0120life": 1204, "\u0120develop": 1205, "ittle": 1206, "\u0120dep": 1207, "\u0120pass": 1208, "\u00e3\u0125": 1209, "\u0120turn": 1210, "orn": 1211, "This": 1212, "bers": 1213, "ross": 1214, "\u0120Ad": 1215, "\u0120fr": 1216, "\u0120resp": 1217, "\u0120second": 1218, "oh": 1219, "\u0120/": 1220, "\u0120disc": 1221, "\u0120&": 1222, "\u0120something": 1223, "\u0120comple": 1224, "\u0120ed": 1225, "\u0120fil": 1226, "\u0120month": 1227, "aj": 1228, "uc": 1229, "\u0120government": 1230, "\u0120without": 1231, "\u0120leg": 1232, "\u0120dist": 1233, "\u0120put": 1234, "\u0120quest": 1235, "ann": 1236, "\u0120prot": 1237, "20": 1238, "\u0120never": 1239, "ience": 1240, "\u0120level": 1241, "\u0120art": 1242, "\u0120things": 1243, "\u0120might": 1244, "\u0120effect": 1245, "\u0120contro": 1246, "\u0120cent": 1247, "\u012018": 1248, "\u0120allow": 1249, "\u0120belie": 1250, "chool": 1251, "ott": 1252, "\u0120incre": 1253, "\u0120feel": 1254, "\u0120result": 1255, "\u0120lot": 1256, "\u0120fun": 1257, "ote": 1258, "\u0120ty": 1259, "erest": 1260, "\u0120contin": 1261, "\u0120using": 1262, "\u0120big": 1263, "201": 1264, "\u0120ask": 1265, "\u0120best": 1266, "\u0120)": 1267, "IN": 1268, "\u0120opp": 1269, "30": 1270, "\u0120number": 1271, "iness": 1272, "St": 1273, "lease": 1274, "\u0120ca": 1275, "\u0120must": 1276, "\u0120direct": 1277, "\u0120gl": 1278, "\u0120<": 1279, "\u0120open": 1280, "\u0120post": 1281, "\u0120come": 1282, "\u0120seem": 1283, "ording": 1284, "\u0120week": 1285, "ately": 1286, "ital": 1287, "\u0120el": 1288, "riend": 1289, "\u0120far": 1290, "\u0120tra": 1291, "inal": 1292, "\u0120pri": 1293, "\u0120US": 1294, "\u0120place": 1295, "\u0120form": 1296, "\u0120told": 1297, "\":": 1298, "ains": 1299, "ature": 1300, "\u0120Trump": 1301, "\u0120stand": 1302, "\u0120#": 1303, "ider": 1304, "\u0120Fr": 1305, "\u0120next": 1306, "\u0120soc": 1307, "\u0120pur": 1308, "\u0120let": 1309, "\u0120little": 1310, "\u0120hum": 1311, "\u0120i": 1312, "ron": 1313, "15": 1314, "\u012015": 1315, "\u0120commun": 1316, "\u0120mark": 1317, "\u0120There": 1318, "\u0120wr": 1319, "\u0120That": 1320, "\u0120information": 1321, "ways": 1322, "\u0120bus": 1323, "app": 1324, "\u0120invest": 1325, "me": 1326, "\u0120hard": 1327, "ained": 1328, "ead": 1329, "\u0120import": 1330, "\u0120appro": 1331, "\u0120test": 1332, "\u0120tri": 1333, "\u0120rest": 1334, "osed": 1335, "\u0120full": 1336, "\u0120care": 1337, "\u0120Sp": 1338, "\u0120case": 1339, "ON": 1340, "\u0120sk": 1341, "\u0120less": 1342, "\u0120+": 1343, "\u0120partic": 1344, "\u0120Pl": 1345, "ably": 1346, "uck": 1347, "ished": 1348, "chn": 1349, "be": 1350, "\u0120list": 1351, "ator": 1352, "\u0120top": 1353, "\u0120adv": 1354, "\u0120Be": 1355, "ruct": 1356, "\u0120dem": 1357, "ration": 1358, "ling": 1359, "gy": 1360, "reen": 1361, "ger": 1362, "\u0120home": 1363, "\u0120left": 1364, "\u0120better": 1365, "\u0120data": 1366, "\u012011": 1367, "\u0120attack": 1368, "\u0120proble": 1369, "line": 1370, "ards": 1371, "\u0120beh": 1372, "ral": 1373, "\u0120How": 1374, "\u0120She": 1375, "arge": 1376, "\u0120--": 1377, "://": 1378, "\u0120bro": 1379, "\u0120Ph": 1380, "ats": 1381, "\u0120build": 1382, "ww": 1383, "ided": 1384, "aim": 1385, "ases": 1386, "ency": 1387, "\u0120main": 1388, "ined": 1389, "\u0120including": 1390, "\u0120{": 1391, "\u0120got": 1392, "\u0120interest": 1393, "\u0120keep": 1394, "\u0120X": 1395, "\u0120eas": 1396, "aining": 1397, "\u0120class": 1398, "\u00e2\u0122\u00a6": 1399, "\u0120No": 1400, "\u0120var": 1401, "\u0120small": 1402, "ample": 1403, "AT": 1404, "\u0120ide": 1405, "\u0120So": 1406, "\u0120rece": 1407, "\u0120polit": 1408, "\u0120mov": 1409, "\u0120plan": 1410, "\u0120percent": 1411, "iving": 1412, "\u0120camp": 1413, "\u0120pay": 1414, "14": 1415, "sc": 1416, "ised": 1417, "\u0120unt": 1418, "oney": 1419, "ploy": 1420, "====": 1421, "\u0120didn": 1422, "\u0120Ind": 1423, "els": 1424, "ertain": 1425, "\u0120pos": 1426, "____": 1427, "iver": 1428, "\u0120process": 1429, "\u0120program": 1430, "ified": 1431, "\u0120Rep": 1432, "16": 1433, "uro": 1434, "ology": 1435, "atter": 1436, "ina": 1437, "\u0120name": 1438, "\u0120All": 1439, "\u0120four": 1440, "\u0120return": 1441, "vious": 1442, "bs": 1443, "\u0120called": 1444, "\u0120move": 1445, "\u0120Sc": 1446, "ird": 1447, "\u0120group": 1448, "\u0120bre": 1449, "\u0120men": 1450, "\u0120cap": 1451, "ten": 1452, "ee": 1453, "\u0120dri": 1454, "leg": 1455, "here": 1456, "uthor": 1457, "\u0120pat": 1458, "\u0120current": 1459, "ides": 1460, "\u0120pop": 1461, "to": 1462, "ention": 1463, "\u0120always": 1464, "\u0120mil": 1465, "\u0120women": 1466, "\u012016": 1467, "\u0120old": 1468, "iven": 1469, "raph": 1470, "\u0120Or": 1471, "ror": 1472, "ently": 1473, "\u0120near": 1474, "\u0120Ex": 1475, "ream": 1476, "sh": 1477, "\u012014": 1478, "\u0120free": 1479, "ission": 1480, "stand": 1481, "\u0120Con": 1482, "ality": 1483, "used": 1484, "13": 1485, "\u0120design": 1486, "\u0120change": 1487, "\u0120chang": 1488, "\u0120bo": 1489, "\u0120vis": 1490, "ember": 1491, "\u0120book": 1492, "ready": 1493, "\u0120kill": 1494, "25": 1495, "pped": 1496, "\u0120away": 1497, "\u0120able": 1498, "\u0120country": 1499, "\u0120const": 1500, "arn": 1501, "\u0120order": 1502, "AR": 1503, "ior": 1504, "ium": 1505, "orth": 1506, "18": 1507, "ailable": 1508, "\u0120sw": 1509, "\u0120million": 1510, "\u012013": 1511, "atic": 1512, "ted": 1513, "\u0120Go": 1514, "\u0120oper": 1515, "eng": 1516, "\u0120thing": 1517, "ajor": 1518, "conom": 1519, "\u0120Comm": 1520, "\u0120why": 1521, "ured": 1522, "ural": 1523, "\u0120school": 1524, "by": 1525, "\u0120Mar": 1526, "\u0120aff": 1527, "\u0120days": 1528, "\u0120ann": 1529, "ush": 1530, "ane": 1531, "If": 1532, "eg": 1533, "\u0120prof": 1534, "\u0120health": 1535, "outh": 1536, "But": 1537, "ional": 1538, ".,": 1539, "\u0120sol": 1540, "\u0120already": 1541, "\u012030": 1542, "\u0120charact": 1543, "He": 1544, "\u0120friend": 1545, "ES": 1546, "ians": 1547, "icle": 1548, "'d": 1549, "\u0120On": 1550, "\u0120least": 1551, "\u0120prom": 1552, "\u0120dr": 1553, "\u0120hist": 1554, "ither": 1555, "\u0120est": 1556, "iqu": 1557, "17": 1558, "son": 1559, "\u0120tell": 1560, "\u0120talk": 1561, "ohn": 1562, "oint": 1563, "lection": 1564, "AN": 1565, "\u0120until": 1566, "augh": 1567, "\u0120later": 1568, "\u0120ve": 1569, "\u0120view": 1570, "ending": 1571, "ived": 1572, "\u0120word": 1573, "ware": 1574, "\u0120cost": 1575, "\u0120enough": 1576, "\u0120give": 1577, "\u0120United": 1578, "\u0120techn": 1579, "arent": 1580, "OR": 1581, "\u0120par": 1582, "\u0120Dr": 1583, "\u01202016": 1584, "rist": 1585, "ering": 1586, "\u0120\u00c2": 1587, "\u0120large": 1588, "side": 1589, "acy": 1590, "ccess": 1591, "\u0120win": 1592, "\u0120important": 1593, "\u0120199": 1594, "\u0120doesn": 1595, "\u012017": 1596, "\u0120business": 1597, "\u0120clear": 1598, "\u0120rese": 1599, "\",": 1600, "ury": 1601, "\u0120equ": 1602, "aster": 1603, "alf": 1604, "\u0120American": 1605, "nect": 1606, "\u0120expect": 1607, "iversity": 1608, "\u0120occ": 1609, "\u0120Fl": 1610, "\u0120kind": 1611, "\u0120mean": 1612, "\u0120past": 1613, "\u0120dev": 1614, "\u0120bas": 1615, "let": 1616, "raft": 1617, "\u0120organ": 1618, "\u0120del": 1619, "\u0120perform": 1620, "\u0120story": 1621, "\u0120season": 1622, "\u0120Col": 1623, "\u0120claim": 1624, "\u0120came": 1625, "\u0120within": 1626, "\u0120line": 1627, "\u0120project": 1628, "\u0120At": 1629, "\u0120control": 1630, "ended": 1631, "\u0120Sy": 1632, "\u0120air": 1633, "ization": 1634, "\u0120*": 1635, "ley": 1636, "\u0120money": 1637, "idd": 1638, "You": 1639, "for": 1640, "\u0120family": 1641, "\u0120making": 1642, "\u0120bit": 1643, "\u0120police": 1644, "\u0120happen": 1645, "\u0120vers": 1646, "ony": 1647, "uff": 1648, "\u0120When": 1649, "\u0120sit": 1650, "ideo": 1651, "lf": 1652, "ison": 1653, "\u0120sure": 1654, "gin": 1655, "\u0120appear": 1656, "\u0120light": 1657, "\u0120es": 1658, "of": 1659, "\u0120water": 1660, "\u0120times": 1661, "not": 1662, "\u0120grow": 1663, "\u0120company": 1664, "\u0120Te": 1665, "ows": 1666, "\u0120mar": 1667, "ource": 1668, "iol": 1669, "arm": 1670, "br": 1671, "\u0120example": 1672, "\u0120conc": 1673, "\u0120fore": 1674, "\u0120To": 1675, "pro": 1676, "EN": 1677, "ries": 1678, "\u012025": 1679, "\u0120Can": 1680, "ney": 1681, "\u0120actually": 1682, "\u0120ever": 1683, "urity": 1684, "aken": 1685, "aps": 1686, "\u0120tax": 1687, "\u0120major": 1688, "ama": 1689, "\u0120often": 1690, "eral": 1691, "\u0120human": 1692, "\u0120job": 1693, "ister": 1694, "\u0120available": 1695, "ocr": 1696, "enn": 1697, "aid": 1698, "ivid": 1699, "\u0120record": 1700, "?\"": 1701, "\u0120sing": 1702, "\u0120Am": 1703, "idence": 1704, "\u0120news": 1705, "ster": 1706, "\u0120econom": 1707, "\u0120following": 1708, "\u0120Br": 1709, "ising": 1710, "\u0120hour": 1711, "most": 1712, "ument": 1713, "\u0120sex": 1714, "\u0120desc": 1715, "\u0120become": 1716, "\u0120Ed": 1717, "\u0120took": 1718, "\u0120having": 1719, "\u0120product": 1720, "ault": 1721, "As": 1722, "aring": 1723, "\u0120means": 1724, "\u0120hop": 1725, "une": 1726, "\u0120cho": 1727, "\u0120certain": 1728, "\u0120non": 1729, "\u0120deal": 1730, "24": 1731, "lement": 1732, "oci": 1733, "ene": 1734, "\u0120side": 1735, "\u0120Pr": 1736, "\u0120May": 1737, "\u0120reason": 1738, "ued": 1739, "ched": 1740, "ulation": 1741, "\u0120elect": 1742, "\u0120official": 1743, "\u0120possible": 1744, "\u0120hold": 1745, "ands": 1746, "ots": 1747, "\u0120city": 1748, "ories": 1749, "\u0120sever": 1750, "\u0120children": 1751, "\u0120once": 1752, "\u0120activ": 1753, "ler": 1754, "\u0120night": 1755, "itions": 1756, "\u0120John": 1757, "ape": 1758, "play": 1759, "\u0120done": 1760, "\u0120lim": 1761, "\u0120working": 1762, "\u0120Pres": 1763, "orld": 1764, "eb": 1765, "\u0120Co": 1766, "\u0120body": 1767, "ails": 1768, "utes": 1769, "\u0120Mr": 1770, "\u0120whether": 1771, "\u0120author": 1772, "rop": 1773, "\u0120proper": 1774, "\u0120seen": 1775, ");": 1776, "\u0120fac": 1777, "\u0120Su": 1778, "\u0120cond": 1779, "iting": 1780, "\u0120course": 1781, "\u0120}": 1782, "----------------": 1783, "aign": 1784, "\u0120event": 1785, "\u0120eng": 1786, "\u0120pot": 1787, "\u0120intern": 1788, "iam": 1789, "\u0120short": 1790, "empt": 1791, "\u00e3\u0124": 1792, "\u0120God": 1793, "ilar": 1794, "80": 1795, "\u0120orig": 1796, "IS": 1797, "ourn": 1798, "ability": 1799, "itive": 1800, "\u0120dam": 1801, "\u0120100": 1802, "\u0120press": 1803, "\u0120doing": 1804, "\u0120protect": 1805, "ring": 1806, "\u0120thought": 1807, "\u0120question": 1808, "rew": 1809, "\u0120War": 1810, "\u0120several": 1811, "\u0120State": 1812, "\u0120given": 1813, "\u0120fund": 1814, "\u0120Tw": 1815, "\u0120went": 1816, "ances": 1817, "work": 1818, "por": 1819, "my": 1820, "40": 1821, "\u0120arg": 1822, "artment": 1823, "ustom": 1824, "\u0120polic": 1825, "\u0120meet": 1826, "\u0120creat": 1827, "22": 1828, "\u0120States": 1829, "\u0120games": 1830, "raw": 1831, "uture": 1832, "\u0120understand": 1833, "urs": 1834, "\u0120Ob": 1835, "lish": 1836, "sy": 1837, "\u0120makes": 1838, "\u0120won": 1839, "agon": 1840, "\u0120htt": 1841, "\u0120love": 1842, "ential": 1843, "\u0120complete": 1844, "par": 1845, "\u0120Im": 1846, "AL": 1847, "\u0120account": 1848, "\u00c2\u0142": 1849, "ored": 1850, "vert": 1851, "\u0120ident": 1852, "\u01202015": 1853, "\u0120others": 1854, "\u0120Min": 1855, "iber": 1856, "verage": 1857, "There": 1858, "itional": 1859, "dd": 1860, "\u0120prob": 1861, "\u0120young": 1862, "\u0120along": 1863, "\u0120according": 1864, "\u0120yet": 1865, "\u0120members": 1866, "\u0120What": 1867, "oid": 1868, "\u0120Man": 1869, "And": 1870, "\u0120among": 1871, "ai": 1872, "\u0120employ": 1873, "\u0120Res": 1874, "\u0120>": 1875, "\u0120invol": 1876, "\u0120low": 1877, "af": 1878, "\u0120Car": 1879, "\u0120hig": 1880, "\u0120One": 1881, "\u0120Sec": 1882, "ination": 1883, "\u0120likely": 1884, "\u0120ant": 1885, "aged": 1886, "\u0120Russ": 1887, "\u0120ben": 1888, "\u0120rele": 1889, "For": 1890, "back": 1891, "\u0120Not": 1892, "\u0120president": 1893, "ball": 1894, "\u0120access": 1895, "ividual": 1896, "\u0120Dem": 1897, "\u0120Euro": 1898, "60": 1899, "\u0120known": 1900, "irl": 1901, "\u0120Gr": 1902, "\u0120early": 1903, "use": 1904, "iety": 1905, "\u00e2\u0122\u0135": 1906, "\u0120fight": 1907, "\u0120sent": 1908, "\u0120today": 1909, "\u0120market": 1910, "\".": 1911, "\u0120based": 1912, "\u0120strong": 1913, "urther": 1914, "\u0120deb": 1915, "mber": 1916, "\u0120problem": 1917, "\u0120death": 1918, "\u0120social": 1919, "imate": 1920, "AS": 1921, "ortun": 1922, "\u0120campaign": 1923, "ery": 1924, "Ch": 1925, "\u0120ey": 1926, "ially": 1927, "\u0120mus": 1928, "wh": 1929, "pos": 1930, "\u0120er": 1931, "\u0120saf": 1932, "\u0120months": 1933, "iron": 1934, "\u0120viol": 1935, "\u0120five": 1936, "\u0120stre": 1937, "\u0120players": 1938, "inc": 1939, "ald": 1940, "year": 1941, "aun": 1942, "\u0120success": 1943, "\u0120present": 1944, "erence": 1945, "\u01202014": 1946, "\u0120sugg": 1947, "\u0120particular": 1948, "\u0120try": 1949, "\u0120suggest": 1950, "\u0120Christ": 1951, "ones": 1952, "\u0120priv": 1953, "23": 1954, "\u0120crit": 1955, "\u0120land": 1956, "\u0120local": 1957, "ify": 1958, "29": 1959, "\u0120aut": 1960, "ED": 1961, "\u0120Gu": 1962, "\u0120mult": 1963, "\u0120political": 1964, "\u0120asked": 1965, "\u0120former": 1966, "itter": 1967, "ript": 1968, "\u0120close": 1969, "\u0120pract": 1970, "\u0120York": 1971, "\u0120getting": 1972, "\u0120across": 1973, "\u0120comb": 1974, "\u0120believe": 1975, "\u0120z": 1976, "\u0120toget": 1977, "\u0120together": 1978, "\u0120Cent": 1979, "irc": 1980, "\u0120individual": 1981, "\u0120Mc": 1982, "27": 1983, "isk": 1984, "\u0120Eng": 1985, "\u0120face": 1986, "\u012024": 1987, "\u0120value": 1988, "\u0120area": 1989, "ev": 1990, "\u0120writ": 1991, "\u0120President": 1992, "\u0120vot": 1993, "\u0120key": 1994, "\u0120mom": 1995, "put": 1996, "\u0120anything": 1997, "\u0120experience": 1998, "attle": 1999, "\u0120mind": 2000, "aff": 2001, "omm": 2002, "\u0120future": 2003, "ged": 2004, "\u0120cut": 2005, "\u0120tot": 2006, "itch": 2007, "\u0120video": 2008, "\u0120investig": 2009, "\u0120net": 2010, "\u0120My": 2011, "rict": 2012, "ien": 2013, ".)": 2014, "\u0120impro": 2015, "though": 2016, "wards": 2017, "\u0120connect": 2018, "\u0120Med": 2019, "selves": 2020, "ensive": 2021, "mb": 2022, "ober": 2023, "ators": 2024, "An": 2025, "\u012050": 2026, "\u0120redu": 2027, "resent": 2028, "\u0120above": 2029, "\u0120fre": 2030, "\u0120Europe": 2031, "sw": 2032, "\u0120amount": 2033, "\u0120App": 2034, "\u0120either": 2035, "\u0120milit": 2036, "\u0120anal": 2037, "\u0120fail": 2038, "\u0120En": 2039, "ales": 2040, "\u0120special": 2041, "\u0120black": 2042, "IT": 2043, "cher": 2044, "\u0120looking": 2045, "\u0120fire": 2046, "yn": 2047, "\u0120almost": 2048, "oon": 2049, "\u0120study": 2050, "\u0120miss": 2051, "ches": 2052, "rown": 2053, "\u0120tre": 2054, "\u0120community": 2055, "\u0120media": 2056, "\u0120food": 2057, "\u0120comes": 2058, "\u0120University": 2059, "\u0120single": 2060, "What": 2061, "uly": 2062, "\u0120half": 2063, "ague": 2064, "hod": 2065, "\u0120Republic": 2066, "\u0120started": 2067, "\u0120quick": 2068, "oto": 2069, "book": 2070, "\u0120issue": 2071, "itor": 2072, "\u0120else": 2073, "\u0120consider": 2074, "26": 2075, "rodu": 2076, "\u0120taken": 2077, "28": 2078, "99": 2079, "\u0120With": 2080, "\u0120true": 2081, "\u0120wa": 2082, "\u0120trad": 2083, "\u0120ago": 2084, "\u0120mess": 2085, "ief": 2086, "\u0120added": 2087, "oke": 2088, "\u0120bad": 2089, "\u0120fav": 2090, "33": 2091, "\u0120similar": 2092, "ask": 2093, "\u0120Don": 2094, "\u0120character": 2095, "orts": 2096, "\u0120House": 2097, "\u0120reported": 2098, "\u0120type": 2099, "val": 2100, "iod": 2101, "\u0120However": 2102, "\u0120targ": 2103, "\u0120entire": 2104, "pping": 2105, "\u0120history": 2106, "\u0120live": 2107, "ffic": 2108, "........": 2109, "ederal": 2110, "\u0120trying": 2111, "\u0120discuss": 2112, "\u0120Har": 2113, "aces": 2114, "lished": 2115, "\u0120self": 2116, "osp": 2117, "rest": 2118, "\u0120room": 2119, "elt": 2120, "\u0120fall": 2121, "olution": 2122, "\u0120et": 2123, "\u0120x": 2124, "\u0120isn": 2125, "\u0120idea": 2126, "bo": 2127, "\u0120sound": 2128, "\u0120Dep": 2129, "\u0120someone": 2130, "cially": 2131, "ully": 2132, "\u0120foc": 2133, "\u0120object": 2134, "ift": 2135, "aper": 2136, "\u0120player": 2137, "\u0120rather": 2138, "\u0120service": 2139, "ashing": 2140, "\u0120Do": 2141, "\u0120Part": 2142, "rug": 2143, "mon": 2144, "ply": 2145, "\u0120mor": 2146, "\u0120nothing": 2147, "\u0120provide": 2148, "IC": 2149, "ung": 2150, "\u0120party": 2151, "\u0120exist": 2152, "\u0120mag": 2153, "70": 2154, "\u0120rul": 2155, "\u0120house": 2156, "\u0120behind": 2157, "\u0120however": 2158, "\u0120World": 2159, "\u0120sum": 2160, "\u0120applic": 2161, "\u0120;": 2162, "\u0120function": 2163, "gr": 2164, "\u0120Pol": 2165, "\u0120front": 2166, "200": 2167, "\u0120series": 2168, "\u0120tem": 2169, "\u0120typ": 2170, "ills": 2171, "\u0120opt": 2172, "\u0120points": 2173, "\u0120below": 2174, "itted": 2175, "\u0120specific": 2176, "\u01202017": 2177, "umb": 2178, "\u0120ra": 2179, "\u0120previous": 2180, "\u0120pret": 2181, "reme": 2182, "\u0120custom": 2183, "\u0120court": 2184, "\u0120Me": 2185, "\u0120repl": 2186, "\u0120whole": 2187, "go": 2188, "cer": 2189, "\u0120treat": 2190, "\u0120Act": 2191, "\u0120probably": 2192, "\u0120learn": 2193, "ender": 2194, "\u0120Ass": 2195, "\u0120version": 2196, "now": 2197, "\u0120check": 2198, "\u0120Cal": 2199, "RE": 2200, "minist": 2201, "On": 2202, "ources": 2203, "\u0120benef": 2204, "\u0120doc": 2205, "\u0120deter": 2206, "\u0120enc": 2207, "\u0120super": 2208, "\u0120address": 2209, "\u0120vict": 2210, "\u01202013": 2211, "\u0120meas": 2212, "tr": 2213, "\u0120field": 2214, "When": 2215, "\u0120signific": 2216, "uge": 2217, "\u0120feat": 2218, "\u0120common": 2219, "load": 2220, "\u0120begin": 2221, "\u0120bring": 2222, "\u0120action": 2223, "erman": 2224, "\u0120describ": 2225, "\u0120indust": 2226, "\u0120wanted": 2227, "ried": 2228, "ming": 2229, "\u0120attempt": 2230, "45": 2231, "fer": 2232, "\u0120due": 2233, "ression": 2234, "##": 2235, "\u0120shall": 2236, "\u0120six": 2237, "oo": 2238, "\u0120step": 2239, "\u0120pub": 2240, "\u0120himself": 2241, "\u012023": 2242, "\u0120cop": 2243, "\u0120dest": 2244, "\u0120stop": 2245, "AC": 2246, "ibility": 2247, "\u0120lab": 2248, "icult": 2249, "\u0120hours": 2250, "\u0120create": 2251, "\u0120further": 2252, "\u0120America": 2253, "\u0120City": 2254, "\u0120dou": 2255, "head": 2256, "ST": 2257, "\u0120North": 2258, "cing": 2259, "\u0120national": 2260, "ule": 2261, "\u0120Inst": 2262, "\u0120taking": 2263, "\u0120Qu": 2264, "irt": 2265, "\u0120red": 2266, "\u0120research": 2267, "viron": 2268, "\u0120Ge": 2269, "\u0120break": 2270, "ana": 2271, "\u0120space": 2272, "aterial": 2273, "\u0120recent": 2274, "\u0120Ab": 2275, "\u0120general": 2276, "\u0120hit": 2277, "\u0120period": 2278, "\u0120everything": 2279, "ively": 2280, "\u0120phys": 2281, "\u0120saying": 2282, "anks": 2283, "\u0120cou": 2284, "\u0120cult": 2285, "aced": 2286, "eal": 2287, "uation": 2288, "\u0120coun": 2289, "lu": 2290, "\u0120include": 2291, "\u0120position": 2292, "\u0120After": 2293, "\u0120Canad": 2294, "\u0120Em": 2295, "\u0120imm": 2296, "\u0120Red": 2297, "\u0120pick": 2298, "\u0120compl": 2299, "\u0120matter": 2300, "reg": 2301, "ext": 2302, "angu": 2303, "isc": 2304, "ole": 2305, "aut": 2306, "\u0120compet": 2307, "eed": 2308, "fect": 2309, "\u012021": 2310, "\u0120Sen": 2311, "\u0120These": 2312, "asing": 2313, "\u0120cannot": 2314, "\u0120init": 2315, "\u0120relations": 2316, "ached": 2317, "\u0120bar": 2318, "\u012040": 2319, "\u0120TH": 2320, "\u01202012": 2321, "\u0120vol": 2322, "\u0120ground": 2323, "\u0120security": 2324, "\u0120upd": 2325, "ilt": 2326, "35": 2327, "\u0120concern": 2328, "\u0120Just": 2329, "\u0120white": 2330, "\u0120seems": 2331, "\u0120Her": 2332, "pecially": 2333, "ients": 2334, "\u0120announ": 2335, "\u0120fig": 2336, "ights": 2337, "\u0120stri": 2338, "like": 2339, "ids": 2340, "\u0120sus": 2341, "\u0120watch": 2342, "\u0120\u00e2": 2343, "\u0120wind": 2344, "\u0120Cont": 2345, "\u0120itself": 2346, "\u0120mass": 2347, "Al": 2348, "yle": 2349, "ique": 2350, "\u0120National": 2351, "\u0120abs": 2352, "\u0120pack": 2353, "\u0120outside": 2354, "\u0120anim": 2355, "\u0120pain": 2356, "eter": 2357, "\u0120manag": 2358, "duct": 2359, "ogn": 2360, "\u0120]": 2361, "\u0120Sept": 2362, "sec": 2363, "off": 2364, "\u0120Jan": 2365, "\u0120foot": 2366, "ades": 2367, "\u0120third": 2368, "\u0120mot": 2369, "\u0120evidence": 2370, "inton": 2371, "\u0120threat": 2372, "apt": 2373, "ples": 2374, "cle": 2375, "\u0120lo": 2376, "\u0120decl": 2377, "\u0120item": 2378, "medi": 2379, "\u0120represent": 2380, "omb": 2381, "amer": 2382, "\u0120significant": 2383, "ograph": 2384, "su": 2385, "\u0120cal": 2386, "ires": 2387, "0000": 2388, "ID": 2389, "AM": 2390, "\u0120simply": 2391, "\u0120longer": 2392, "\u0120file": 2393, "OT": 2394, "che": 2395, "So": 2396, "ateg": 2397, "org": 2398, "\u0120His": 2399, "\u0120ener": 2400, "\u0120dom": 2401, "\u0120upon": 2402, "ili": 2403, "\":\"": 2404, "\u0120themselves": 2405, "\u0120coming": 2406, "\u0120quite": 2407, "\u0120difficult": 2408, "\u0120Bar": 2409, "ilities": 2410, "rel": 2411, "ends": 2412, "cial": 2413, "64": 2414, "\u0120woman": 2415, "rap": 2416, "yr": 2417, "\u0120necess": 2418, "ips": 2419, "\u0120text": 2420, "\u0120require": 2421, "\u0120military": 2422, "\u0120review": 2423, "\u0120respons": 2424, "75": 2425, "\u0120subject": 2426, "\u0120instead": 2427, "\u0120issues": 2428, "\u0120gen": 2429, "\",\"": 2430, "\u0120minutes": 2431, "\u0120weap": 2432, "ray": 2433, "amed": 2434, "time": 2435, "bl": 2436, "How": 2437, "\u0120code": 2438, "\u0120Sm": 2439, "\u0120higher": 2440, "\u0120Ste": 2441, "ris": 2442, "\u0120page": 2443, "\u0120students": 2444, "\u0120Intern": 2445, "\u0120method": 2446, "\u0120Aug": 2447, "\u0120Per": 2448, "\u0120Ag": 2449, "\u0120policy": 2450, "\u0120Sw": 2451, "\u0120exec": 2452, "\u0120accept": 2453, "ume": 2454, "ribut": 2455, "\u0120words": 2456, "\u0120final": 2457, "\u0120changes": 2458, "\u0120Democr": 2459, "\u0120friends": 2460, "\u0120respect": 2461, "\u0120ep": 2462, "\u0120compan": 2463, "ivil": 2464, "\u0120damage": 2465, "****": 2466, "ogle": 2467, "vironment": 2468, "\u0120neg": 2469, "ental": 2470, "\u0120ap": 2471, "\u0120total": 2472, "ival": 2473, "!\"": 2474, "lim": 2475, "\u0120needs": 2476, "\u0120agre": 2477, "\u0120development": 2478, "\u0120age": 2479, "iple": 2480, "21": 2481, "\u0120results": 2482, "\u0120Af": 2483, "Sh": 2484, "\u0120gun": 2485, "\u0120Obama": 2486, "roll": 2487, "\u0120@": 2488, "\u0120rights": 2489, "\u0120Brit": 2490, "\u0120running": 2491, "\u0120wasn": 2492, "\u0120port": 2493, "\u0120rate": 2494, "\u0120pretty": 2495, "\u0120target": 2496, "\u0120saw": 2497, "\u0120circ": 2498, "\u0120works": 2499, "icro": 2500, "alt": 2501, "over": 2502, "www": 2503, "That": 2504, "lier": 2505, "\u0120everyone": 2506, "ude": 2507, "\u0120pie": 2508, "iddle": 2509, "rael": 2510, "\u0120rad": 2511, "\u0120block": 2512, "\u0120walk": 2513, "To": 2514, "\u00e3\u0123": 2515, "nes": 2516, "\u0120Aust": 2517, "aul": 2518, "rote": 2519, "\u0120South": 2520, "ession": 2521, "oph": 2522, "\u0120shows": 2523, "\u0120site": 2524, "\u0120jo": 2525, "\u0120risk": 2526, "clus": 2527, "lt": 2528, "\u0120inj": 2529, "iding": 2530, "\u0120Spe": 2531, "\u0120chall": 2532, "irm": 2533, "\u012022": 2534, "itting": 2535, "str": 2536, "\u0120hy": 2537, "LE": 2538, "key": 2539, "\u0120began": 2540, "atur": 2541, "ashington": 2542, "lam": 2543, "\u0120Dav": 2544, "bit": 2545, "\u0120size": 2546, "\u0120Par": 2547, "38": 2548, "ournal": 2549, "face": 2550, "\u0120decision": 2551, "\u0120larg": 2552, "\u0120jud": 2553, "rect": 2554, "\u0120continue": 2555, "\u0120Oct": 2556, "overed": 2557, "\u0120Int": 2558, "========": 2559, "\u0120parent": 2560, "\u0120Will": 2561, "\u0120easy": 2562, "\u0120drug": 2563, "anger": 2564, "\u0120sense": 2565, "\u0120di": 2566, "iday": 2567, "\u0120energy": 2568, "istic": 2569, "\u0120associ": 2570, "arter": 2571, "obal": 2572, "eks": 2573, "\u0120El": 2574, "urch": 2575, "\u0120girl": 2576, "oe": 2577, "itle": 2578, "\u012028": 2579, "\u0120Che": 2580, "\u0120request": 2581, "\u0120soon": 2582, "\u0120host": 2583, "ky": 2584, "\u0120states": 2585, "omes": 2586, "\u0120material": 2587, "lex": 2588, "\u0120moment": 2589, "\u0120answ": 2590, "onse": 2591, "\u0120especially": 2592, "\u0120norm": 2593, "\u0120services": 2594, "pite": 2595, "ran": 2596, "\u0120role": 2597, "44": 2598, "):": 2599, "\u0120cred": 2600, "Cl": 2601, "________": 2602, "\u0120mat": 2603, "\u0120log": 2604, "\u0120Clinton": 2605, "OU": 2606, "\u0120office": 2607, "\u012026": 2608, "\u0120charg": 2609, "\u0120track": 2610, "ma": 2611, "\u0120heart": 2612, "\u0120ball": 2613, "\u0120personal": 2614, "\u0120building": 2615, "na": 2616, "set": 2617, "body": 2618, "\u0120Black": 2619, "\u0120increase": 2620, "itten": 2621, "\u0120needed": 2622, "36": 2623, "32": 2624, "=\"": 2625, "\u0120lost": 2626, "\u0120became": 2627, "\u0120groups": 2628, "\u0120Mus": 2629, "\u0120wrote": 2630, "\u0120Pe": 2631, "\u0120prop": 2632, "joy": 2633, "\u00c3\u00a9": 2634, "\u0120White": 2635, "\u0120dead": 2636, ".'": 2637, "\u0120http": 2638, "\u0120webs": 2639, "OS": 2640, "\u0120inside": 2641, "\u0120wrong": 2642, "\u0120statement": 2643, "\u0120...": 2644, "yl": 2645, "\u0120film": 2646, "\u0120music": 2647, "\u0120share": 2648, "ification": 2649, "\u0120release": 2650, "\u0120forward": 2651, "\u0120stay": 2652, "\u0120comput": 2653, "itte": 2654, "ser": 2655, "\u0120original": 2656, "\u0120card": 2657, "\u0120cand": 2658, "\u0120div": 2659, "atural": 2660, "\u0120favor": 2661, "OM": 2662, "\u0120cases": 2663, "uses": 2664, "\u0120section": 2665, "\u0120leave": 2666, "ging": 2667, "oved": 2668, "\u0120Washington": 2669, "39": 2670, "\u0120Gl": 2671, "\u0120required": 2672, "action": 2673, "apan": 2674, "oor": 2675, "iter": 2676, "\u0120King": 2677, "\u0120countries": 2678, "\u0120German": 2679, "lling": 2680, "\u012027": 2681, "34": 2682, "\u0120questions": 2683, "\u0120prim": 2684, "\u0120cell": 2685, "\u0120shoot": 2686, "\u0120anyone": 2687, "\u0120West": 2688, "\u0120affect": 2689, "epend": 2690, "\u0120online": 2691, "\u0120Israel": 2692, "\u0120September": 2693, "\u0120ability": 2694, "\u0120content": 2695, "ises": 2696, "\u0120reve": 2697, "\u0120laun": 2698, "\u0120indic": 2699, "\u0120force": 2700, "cast": 2701, "\u0120sold": 2702, "aving": 2703, "fl": 2704, "\u0120soft": 2705, "\u0120companies": 2706, "ceed": 2707, "\u0120article": 2708, "\u0120aud": 2709, "\u0120rev": 2710, "\u0120educ": 2711, "\u0120playing": 2712, "05": 2713, "\u0120held": 2714, "ctor": 2715, "\u0120released": 2716, "\u0120federal": 2717, "37": 2718, "\u0120administ": 2719, "\u0120interview": 2720, "\u0120install": 2721, "\u0120received": 2722, "\u0120source": 2723, "uk": 2724, "Ph": 2725, "\u0120serious": 2726, "\u0120created": 2727, "\u0120cause": 2728, "\u0120immedi": 2729, "\u0120defin": 2730, "uel": 2731, "\u0120Department": 2732, "ctions": 2733, "\u0120Cour": 2734, "\u0120Now": 2735, "ze": 2736, "ites": 2737, "itution": 2738, "\u0120late": 2739, "\u0120speak": 2740, "ners": 2741, "\u0120legal": 2742, "ari": 2743, "\u0120Cor": 2744, "\u0120weeks": 2745, "\u0120model": 2746, "\u0120pred": 2747, "\u0120exact": 2748, "BC": 2749, "\u0120By": 2750, "ING": 2751, "osing": 2752, "\u0120takes": 2753, "\u0120regard": 2754, "\u0120opportun": 2755, "\u0120price": 2756, "\u0120198": 2757, "\u0120Apr": 2758, "fully": 2759, "\u0120ord": 2760, "\u0120problems": 2761, "ruction": 2762, "ham": 2763, "\u0120Count": 2764, "lege": 2765, "\u0120leaders": 2766, "ET": 2767, "lev": 2768, "\u0120deep": 2769, "ological": 2770, "ese": 2771, "haps": 2772, "\u0120Some": 2773, "\u0120pers": 2774, "\u0120contract": 2775, "\u0120relationship": 2776, "sp": 2777, "oud": 2778, "\u0120base": 2779, "48": 2780, "mit": 2781, "Ad": 2782, "ancial": 2783, "\u0120consum": 2784, "\u0120potential": 2785, "\u0120langu": 2786, "rem": 2787, "eth": 2788, "\u0120relig": 2789, "ressed": 2790, "66": 2791, "\u0120link": 2792, "\u0120lower": 2793, "ayer": 2794, "\u0120June": 2795, "\u0120fem": 2796, "unt": 2797, "erc": 2798, "urd": 2799, "\u0120contact": 2800, "\u0120ill": 2801, "\u0120mother": 2802, "\u0120estab": 2803, "htt": 2804, "\u0120March": 2805, "\u0120Bro": 2806, "\u0120China": 2807, "\u012029": 2808, "\u0120squ": 2809, "\u0120provided": 2810, "\u0120average": 2811, "asons": 2812, "\u01202011": 2813, "\u0120exam": 2814, "lin": 2815, "55": 2816, "ned": 2817, "\u0120perfect": 2818, "\u0120tou": 2819, "alse": 2820, "ux": 2821, "\u0120buy": 2822, "\u0120shot": 2823, "\u0120collect": 2824, "\u0120phot": 2825, "\u0120played": 2826, "\u0120surpr": 2827, "\u0120officials": 2828, "\u0120simple": 2829, "avy": 2830, "\u0120industry": 2831, "\u0120hands": 2832, "ground": 2833, "\u0120pull": 2834, "\u0120round": 2835, "\u0120user": 2836, "\u0120range": 2837, "uary": 2838, "\u0120private": 2839, "ops": 2840, "ees": 2841, "\u0120ways": 2842, "\u0120Mich": 2843, "\u0120veh": 2844, "\u0120except": 2845, "\u0120terms": 2846, "imum": 2847, "pper": 2848, "ION": 2849, "ores": 2850, "\u0120Dragon": 2851, "oul": 2852, "\u0120den": 2853, "\u0120performance": 2854, "\u0120bill": 2855, "cil": 2856, "47": 2857, "\u0120environment": 2858, "\u0120exc": 2859, "add": 2860, "\u0120worth": 2861, "\u0120pict": 2862, "\u0120chance": 2863, "\u01202018": 2864, "bor": 2865, "\u0120speed": 2866, "iction": 2867, "\u0120alleg": 2868, "\u0120Japan": 2869, "atory": 2870, "reet": 2871, "\u0120match": 2872, "\u0120II": 2873, "\u0120stru": 2874, "order": 2875, "\u0120ste": 2876, "\u0120living": 2877, "\u0120struct": 2878, "ino": 2879, "\u0120separ": 2880, "hern": 2881, "\u0120response": 2882, "\u0120enjoy": 2883, "\u0120via": 2884, "AD": 2885, "uments": 2886, "acebook": 2887, "\u0120member": 2888, "ibr": 2889, "izing": 2890, "\u0120tool": 2891, "\u0120Mon": 2892, "\u0120While": 2893, "hood": 2894, "\u0120Ang": 2895, "\u0120Def": 2896, "\u0120offer": 2897, "Tr": 2898, "aur": 2899, "\u0120turned": 2900, "\u0120July": 2901, "down": 2902, "anced": 2903, "\u0120recently": 2904, "\u0120Ear": 2905, "\u0120ce": 2906, "\u0120Star": 2907, "\u0120Cong": 2908, "rought": 2909, "\u0120blood": 2910, "\u0120hope": 2911, "\u0120comment": 2912, "aint": 2913, "\u0120arri": 2914, "iles": 2915, "\u0120particip": 2916, "ought": 2917, "ription": 2918, "08": 2919, "49": 2920, "\u0120gave": 2921, "\u0120select": 2922, "\u0120killed": 2923, "sych": 2924, "\u0120goes": 2925, "ij": 2926, "\u0120coll": 2927, "\u0120impact": 2928, "atives": 2929, "\u0120Ser": 2930, "09": 2931, "\u0120August": 2932, "\u0120boy": 2933, "de": 2934, "\u0120Des": 2935, "\u0120felt": 2936, "US": 2937, "\u0120expected": 2938, "\u0120image": 2939, "\u0120Mark": 2940, "ccording": 2941, "oice": 2942, "EC": 2943, "\u0120Mag": 2944, "ened": 2945, "hold": 2946, "\u0120Post": 2947, "\u0120prevent": 2948, "No": 2949, "\u0120involved": 2950, "\u0120eyes": 2951, "\u0120quickly": 2952, "At": 2953, "unk": 2954, "\u0120behav": 2955, "\u0120ur": 2956, "\u0120led": 2957, "come": 2958, "ey": 2959, "\u0120candid": 2960, "\u0120earlier": 2961, "\u0120focus": 2962, "ety": 2963, "Pro": 2964, "ledge": 2965, "ixed": 2966, "illed": 2967, "\u0120popular": 2968, "AP": 2969, "\u0120sett": 2970, "light": 2971, "\u0120various": 2972, "inks": 2973, "\u0120levels": 2974, "\u0120road": 2975, "ellig": 2976, "ables": 2977, "hel": 2978, "ittee": 2979, "\u0120Gener": 2980, "ype": 2981, "\u0120heard": 2982, "icles": 2983, "\u0120mis": 2984, "\u0120users": 2985, "\u0120San": 2986, "\u0120improve": 2987, "\u0120father": 2988, "\u0120search": 2989, "They": 2990, "vil": 2991, "\u0120profess": 2992, "\u0120knew": 2993, "\u0120loss": 2994, "\u0120events": 2995, "65": 2996, "\u0120billion": 2997, "07": 2998, "02": 2999, "\u0120News": 3000, "\u0120AM": 3001, "\u0120cover": 3002, "where": 3003, "ension": 3004, "\u0120bott": 3005, "\u0120areas": 3006, "ences": 3007, "ope": 3008, "\u0120Twitter": 3009, "ael": 3010, "\u0120gets": 3011, "\u0120Google": 3012, "\u0120sn": 3013, "iant": 3014, "\u0120vote": 3015, "\u0120nearly": 3016, "\u0120included": 3017, "\u0120recogn": 3018, "zz": 3019, "mm": 3020, "aled": 3021, "\u0120happened": 3022, "04": 3023, "\u0120hot": 3024, "\u0120whose": 3025, "\u0120civil": 3026, "\u0120suff": 3027, "oes": 3028, "itiz": 3029, "\u0120Syri": 3030, "\u0120respond": 3031, "\u0120hon": 3032, "\u0120features": 3033, "\u0120economic": 3034, "\u0120April": 3035, "rim": 3036, "\u0120technology": 3037, "\u0120option": 3038, "aging": 3039, "\u0120purch": 3040, "Re": 3041, "\u0120lat": 3042, "chie": 3043, "isl": 3044, "\u0120recomm": 3045, "uf": 3046, "\u0120training": 3047, "\u0120effects": 3048, "\u0120fast": 3049, "\u01202010": 3050, "\u0120occur": 3051, "\u0120website": 3052, "\u0120email": 3053, "\u0120sens": 3054, "ech": 3055, "\u0120oil": 3056, "\u0120influ": 3057, "\u0120currently": 3058, "\u0120Sch": 3059, "\u0120Add": 3060, "\u0120goal": 3061, "\u0120scient": 3062, "\u0120conv": 3063, "100": 3064, "emy": 3065, "\u0120decided": 3066, "\u0120travel": 3067, "\u0120mention": 3068, "LL": 3069, "03": 3070, "\u0120election": 3071, "\u0120phone": 3072, "\u0120looks": 3073, "\u0120situation": 3074, "\u0120cy": 3075, "\u0120hor": 3076, "bed": 3077, "\u0120Court": 3078, "aily": 3079, "aves": 3080, "\u0120quality": 3081, "\u0120Comp": 3082, "wise": 3083, "\u0120table": 3084, "\u0120staff": 3085, "\u0120Wind": 3086, "ett": 3087, "\u0120tried": 3088, "idered": 3089, "\u0120addition": 3090, "\u0120box": 3091, "\u0120lack": 3092, "arily": 3093, "\u0120wide": 3094, "\u0120mid": 3095, "\u0120board": 3096, "ysis": 3097, "\u0120anti": 3098, "ha": 3099, "\u0120dig": 3100, "ening": 3101, "\u0120dro": 3102, "Con": 3103, "68": 3104, "\u0120slow": 3105, "based": 3106, "sequ": 3107, "\u0120path": 3108, "Ex": 3109, "aker": 3110, "\u0120worked": 3111, "\u0120pen": 3112, "\u0120engine": 3113, "\u0120looked": 3114, "\u0120Super": 3115, "\u0120Serv": 3116, "\u0120victim": 3117, "Un": 3118, "\u0120property": 3119, "\u0120introdu": 3120, "\u0120execut": 3121, "\u0120PM": 3122, "Le": 3123, "\u0120color": 3124, "\u0120More": 3125, "\u012060": 3126, "\u0120network": 3127, "\u0120date": 3128, "cul": 3129, "idge": 3130, "\u0120extra": 3131, "31": 3132, "\u0120sle": 3133, "67": 3134, "\u0120wond": 3135, "\u0120reports": 3136, "just": 3137, "\u0120Austral": 3138, "\u0120capital": 3139, "\u0120ens": 3140, "\u0120command": 3141, "\u0120allowed": 3142, "\u0120prep": 3143, "\u0120capt": 3144, "hib": 3145, "\u0120numbers": 3146, "chan": 3147, "\u0120fair": 3148, "mp": 3149, "oms": 3150, "\u0120reach": 3151, "With": 3152, "tain": 3153, "\u0120broad": 3154, "\u0120couple": 3155, "ecause": 3156, "lying": 3157, "\u0120Feb": 3158, "\u0120screen": 3159, "\u0120lives": 3160, "\u0120prior": 3161, "\u0120Congress": 3162, "Ar": 3163, "\u0120approach": 3164, "\u0120emer": 3165, "aries": 3166, "\u0120Dis": 3167, "serv": 3168, "\u0120Ne": 3169, "\u0120built": 3170, "cies": 3171, "\u0120repe": 3172, "\u0120rules": 3173, "force": 3174, "\u0120Pal": 3175, "\u0120financial": 3176, "\u0120considered": 3177, "\u0120Char": 3178, "nces": 3179, "\u0120IS": 3180, "\u0120brought": 3181, "\u0120bi": 3182, "iers": 3183, "\u0120Sim": 3184, "OP": 3185, "\u0120products": 3186, "\u0120visit": 3187, "\u0120document": 3188, "\u0120conduct": 3189, "\u0120completely": 3190, "ining": 3191, "\u0120Calif": 3192, "ibly": 3193, "\u0120written": 3194, "\u0120TV": 3195, "ements": 3196, "\u0120draw": 3197, "One": 3198, "\u0120published": 3199, "\u0120secret": 3200, "rain": 3201, "het": 3202, "\u0120Facebook": 3203, "onday": 3204, "\u0120Up": 3205, "\u0120sexual": 3206, "\u0120thous": 3207, "\u0120Pat": 3208, "\u0120ess": 3209, "\u0120standard": 3210, "\u0120arm": 3211, "ges": 3212, "ection": 3213, "\u0120fell": 3214, "\u0120foreign": 3215, "ani": 3216, "\u0120Friday": 3217, "\u0120regular": 3218, "inary": 3219, "\u0120increased": 3220, "\u0120usually": 3221, "\u0120demon": 3222, "\u0120dark": 3223, "\u0120additional": 3224, "rol": 3225, "\u0120Of": 3226, "\u0120production": 3227, "!!": 3228, "undred": 3229, "\u0120international": 3230, "idents": 3231, "\u0120Free": 3232, "roup": 3233, "\u0120race": 3234, "\u0120mach": 3235, "\u0120huge": 3236, "All": 3237, "lear": 3238, "ovember": 3239, "\u0120town": 3240, "\u0120attention": 3241, "\u0120Off": 3242, "yond": 3243, "\u0120Then": 3244, "field": 3245, "\u0120terror": 3246, "raz": 3247, "\u0120Bo": 3248, "\u0120meeting": 3249, "\u0120Park": 3250, "\u0120arrest": 3251, "\u0120fear": 3252, "\u0120aw": 3253, "\u0120Val": 3254, "oring": 3255, "',": 3256, "\u0120extreme": 3257, "arr": 3258, "\u0120workers": 3259, "After": 3260, "\u012031": 3261, "net": 3262, "ament": 3263, "\u0120directly": 3264, "\u0120population": 3265, "ube": 3266, "\u0120October": 3267, "\u0120IN": 3268, "\u0120January": 3269, "59": 3270, "\u0120David": 3271, "\u0120cross": 3272, "cember": 3273, "\u0120First": 3274, "\u0120message": 3275, "irit": 3276, "\u0120nation": 3277, "\u0120poll": 3278, "isions": 3279, "\u0120answer": 3280, "ny": 3281, "isode": 3282, "\u0120carry": 3283, "\u0120Russia": 3284, "\u0120hear": 3285, "ength": 3286, "roy": 3287, "\u0120natural": 3288, "inally": 3289, "\u0120dog": 3290, "mitted": 3291, "\u0120trade": 3292, "\u0120subst": 3293, "\u0120multiple": 3294, "\u0120Afric": 3295, "\u0120fans": 3296, "\u0120sort": 3297, "\u0120global": 3298, "ication": 3299, "\u0120Wed": 3300, "ara": 3301, "\u0120achie": 3302, "\u0120language": 3303, "vey": 3304, "\u0120tal": 3305, "\u0120necessary": 3306, "\u0120details": 3307, "\u0120sen": 3308, "\u0120Sund": 3309, "\u0120Reg": 3310, "\u0120Rec": 3311, "06": 3312, "\u0120sil": 3313, "ressive": 3314, "\u0120medical": 3315, "unch": 3316, "ornia": 3317, "\u0120und": 3318, "fort": 3319, "ocks": 3320, "\u0120Monday": 3321, "uesday": 3322, "craft": 3323, "77": 3324, "urt": 3325, "\u0120ver": 3326, "\u0120Hill": 3327, "\u0120receive": 3328, "\u0120morning": 3329, "estern": 3330, "\u0120bank": 3331, "\u0120sat": 3332, "irth": 3333, "\u0120High": 3334, "\u0120device": 3335, "\u0120THE": 3336, "\u0120Center": 3337, "\u0120safe": 3338, "\u0120ple": 3339, "\u0120Canada": 3340, "\u0120systems": 3341, "\u0120assist": 3342, "\u0120surv": 3343, "\u0120battle": 3344, "\u0120Soc": 3345, "vertis": 3346, "She": 3347, "\u0120paper": 3348, "\u0120growth": 3349, "\u0120cast": 3350, "Sc": 3351, "\u0120plans": 3352, "lled": 3353, "\u0120parts": 3354, "\u0120wall": 3355, "\u0120movement": 3356, "\u0120practice": 3357, "imately": 3358, "\u0120display": 3359, "\u0120sometimes": 3360, "omp": 3361, "\u0120Paul": 3362, "\u0120Yes": 3363, "king": 3364, "58": 3365, "oly": 3366, "\u0120son": 3367, "\u0120avoid": 3368, "okes": 3369, "\u0120Jew": 3370, "\u0120towards": 3371, "asc": 3372, "\u0120//": 3373, "\u0120Kore": 3374, "\u0120talking": 3375, "\u0120correct": 3376, "\u0120spent": 3377, "icks": 3378, "iable": 3379, "eared": 3380, "\u0120term": 3381, "\u0120wants": 3382, "oming": 3383, "\u0120ut": 3384, "\u0120doub": 3385, "\u0120forces": 3386, "\u0120please": 3387, "69": 3388, "\u0120November": 3389, "atform": 3390, "ondon": 3391, "\u0120ones": 3392, "\u0120immediately": 3393, "\u0120Russian": 3394, "\u0120Met": 3395, "\u0120deg": 3396, "\u0120parents": 3397, "CH": 3398, "\u0120Americans": 3399, "aly": 3400, "\u0120Mod": 3401, "\u0120shown": 3402, "\u0120conditions": 3403, "\u0120stuff": 3404, "\u0120reb": 3405, "\u0120Your": 3406, "\u0120includes": 3407, "nown": 3408, "\u0120Sam": 3409, "\u0120experien": 3410, "mission": 3411, "\u0120Even": 3412, "aught": 3413, "\u0120announced": 3414, "\u0120Republican": 3415, "\u0120determin": 3416, "\u0120described": 3417, "\u0120County": 3418, "()": 3419, "\u0120door": 3420, "\u0120changed": 3421, "\u0120neigh": 3422, "\u0120Here": 3423, "\u0120clean": 3424, "\u0120pan": 3425, "\u0120December": 3426, "\u0120European": 3427, "iring": 3428, "apter": 3429, "\u0120club": 3430, "\u0120Tuesday": 3431, "\u0120paid": 3432, "\u0120Net": 3433, "\u0120attacks": 3434, "\u0120characters": 3435, "\u0120alone": 3436, "\u0120director": 3437, "dom": 3438, "\u012035": 3439, "\u0120load": 3440, "\u0120rout": 3441, "\u0120California": 3442, "\u0120finally": 3443, "\u0120rac": 3444, "\u0120contr": 3445, "\u0120exactly": 3446, "resh": 3447, "pri": 3448, "\u0120Islam": 3449, "\u0120nature": 3450, "\u0120career": 3451, "\u0120latest": 3452, "\u0120convers": 3453, "\u0120Sl": 3454, "pose": 3455, "cient": 3456, "\u0120Inc": 3457, "ivity": 3458, "88": 3459, "\u0120Att": 3460, "\u0120Mor": 3461, "nesday": 3462, "\u0120weight": 3463, "ken": 3464, "\u0120note": 3465, "\u0120teams": 3466, "\u0120\\": 3467, "airs": 3468, "\u0120Green": 3469, "\u0120hundred": 3470, "onent": 3471, "\u0120streng": 3472, "\u0120consist": 3473, "icated": 3474, "\u0120regul": 3475, "\u0120lic": 3476, "astic": 3477, "\u0120ten": 3478, "ursday": 3479, "elligence": 3480, "ously": 3481, "\u0120UK": 3482, "BI": 3483, "\u0120costs": 3484, "\u0120independ": 3485, "\u0120AP": 3486, "\u0120normal": 3487, "\u0120hom": 3488, "\u0120obvious": 3489, "\u0120swe": 3490, "\u0120star": 3491, "\u0120ready": 3492, "acher": 3493, "\u0120implement": 3494, "gest": 3495, "\u0120song": 3496, "\u0120Get": 3497, "\u0120Lab": 3498, "\u0120interesting": 3499, "using": 3500, "\u0120giving": 3501, "\u0120Sunday": 3502, "\u0120etc": 3503, "\u0120middle": 3504, "\u0120remember": 3505, "right": 3506, "osition": 3507, "utions": 3508, "\u0120max": 3509, "46": 3510, "\u0120yourself": 3511, "\u0120demand": 3512, "\u0120treatment": 3513, "\u0120danger": 3514, "\u0120Cons": 3515, "\u0120guy": 3516, "\u0120British": 3517, "\u0120physical": 3518, "\u0120related": 3519, "\u0120remain": 3520, "\u0120couldn": 3521, "\u0120refer": 3522, "\u0120citiz": 3523, "box": 3524, "ENT": 3525, "board": 3526, "\u0120inn": 3527, "IG": 3528, "ero": 3529, "\u0120Street": 3530, "ospital": 3531, "rench": 3532, "chers": 3533, "\u0120stra": 3534, "OL": 3535, "ager": 3536, "\u0120AN": 3537, "\u0120easily": 3538, "IA": 3539, "enge": 3540, "iny": 3541, "\u0120clos": 3542, "ocked": 3543, "\u0120uses": 3544, "\u0120Coun": 3545, "Im": 3546, "uild": 3547, "??": 3548, "more": 3549, "\u0120ang": 3550, "\u0120write": 3551, "olute": 3552, "57": 3553, "\u0120leader": 3554, "\u0120reading": 3555, "": 3784, "\u0120figure": 3785, "\u0120disapp": 3786, "enty": 3787, "\u0120software": 3788, "\u0120ult": 3789, "\u0120officers": 3790, "New": 3791, "Is": 3792, "\u0120remains": 3793, "\u0120India": 3794, "\u0120psych": 3795, "rief": 3796, "\u0120cat": 3797, "esc": 3798, "\u0120observ": 3799, "\u0120stage": 3800, "\u0120Dark": 3801, "\u0120enter": 3802, "change": 3803, "\u0120passed": 3804, "\u0120despite": 3805, "\u0120Out": 3806, "\u0120movie": 3807, "rs": 3808, "\u0120voice": 3809, "mine": 3810, "\u0120Play": 3811, "\u0120toward": 3812, "\u0120Ter": 3813, "\u0120region": 3814, "\u0120values": 3815, "orters": 3816, "\u0120mount": 3817, "\u0120officer": 3818, "\u0120Other": 3819, "ban": 3820, "\u0120hous": 3821, "wood": 3822, "room": 3823, "IV": 3824, "\u0120Sun": 3825, "see": 3826, "\u0120Over": 3827, "rog": 3828, "90": 3829, "\u0120lay": 3830, "\u0120Tur": 3831, "awn": 3832, "\u0120pressure": 3833, "\u0120Sub": 3834, "\u0120books": 3835, "edom": 3836, "\u0120Sand": 3837, "AA": 3838, "ago": 3839, "\u0120reasons": 3840, "ford": 3841, "\u0120activity": 3842, "UT": 3843, "Now": 3844, "\u0120Senate": 3845, "cell": 3846, "night": 3847, "\u0120calls": 3848, "inter": 3849, "\u0120letter": 3850, "\u0120Rob": 3851, "\u0120Je": 3852, "\u0120choose": 3853, "\u0120Law": 3854, "Get": 3855, "Be": 3856, "\u0120rob": 3857, "\u0120types": 3858, "\u0120platform": 3859, "\u0120quarter": 3860, "RA": 3861, "\u0120Time": 3862, "\u0120maybe": 3863, "\u0120Cr": 3864, "95": 3865, "pre": 3866, "\u0120moving": 3867, "\u0120lif": 3868, "\u0120gold": 3869, "\u0120som": 3870, "\u0120patients": 3871, "\u0120truth": 3872, "\u0120Ke": 3873, "urance": 3874, "antly": 3875, "mar": 3876, "\u0120charge": 3877, "\u0120Great": 3878, "\u0120cele": 3879, "--------------------------------": 3880, "\u0120rock": 3881, "roid": 3882, "ancy": 3883, "\u0120credit": 3884, "aud": 3885, "By": 3886, "\u0120Every": 3887, "\u0120moved": 3888, "inger": 3889, "ribution": 3890, "\u0120names": 3891, "\u0120straight": 3892, "\u0120Health": 3893, "\u0120Well": 3894, "\u0120feature": 3895, "\u0120rule": 3896, "\u0120sche": 3897, "inated": 3898, "\u0120Michael": 3899, "berg": 3900, "41": 3901, "iled": 3902, "band": 3903, "\u0120click": 3904, "\u0120Angel": 3905, "onents": 3906, "\u00c2\u0143": 3907, "\u0120Iraq": 3908, "\u0120Saturday": 3909, "\u0120aware": 3910, "part": 3911, "\u0120pattern": 3912, "OW": 3913, "\u0120Let": 3914, "\u0120grad": 3915, "igned": 3916, "\u0120associated": 3917, "\u0120style": 3918, "no": 3919, "iation": 3920, "aith": 3921, "ilies": 3922, "\u0120stories": 3923, "uration": 3924, "\u0120individuals": 3925, "\u0120\u00e2\u0122\u00a6": 3926, "miss": 3927, "\u0120Associ": 3928, "ishing": 3929, "aby": 3930, "\u0120summer": 3931, "\u0120Ben": 3932, "\u012032": 3933, "\u0120arch": 3934, "uty": 3935, "\u0120Texas": 3936, "hol": 3937, "\u0120fully": 3938, "\u0120mill": 3939, "\u0120followed": 3940, "\u0120Bill": 3941, "\u0120Indian": 3942, "\u0120Secret": 3943, "\u0120Bel": 3944, "\u0120February": 3945, "\u0120jobs": 3946, "\u0120seemed": 3947, "\u0120Govern": 3948, "ipped": 3949, "\u0120reality": 3950, "\u0120lines": 3951, "\u0120park": 3952, "\u0120measure": 3953, "\u0120Our": 3954, "IM": 3955, "\u0120brother": 3956, "\u0120growing": 3957, "\u0120ban": 3958, "\u0120estim": 3959, "\u0120cry": 3960, "\u0120School": 3961, "\u0120mechan": 3962, "\u0120OF": 3963, "\u0120Windows": 3964, "\u0120rates": 3965, "\u0120Oh": 3966, "\u0120positive": 3967, "\u0120culture": 3968, "istics": 3969, "ica": 3970, "\u0120har": 3971, "ya": 3972, "itely": 3973, "ipp": 3974, "\u0120map": 3975, "encies": 3976, "\u0120William": 3977, "II": 3978, "akers": 3979, "56": 3980, "\u0120Mart": 3981, "\u0120Rem": 3982, "\u0120altern": 3983, "itude": 3984, "\u0120coach": 3985, "rowd": 3986, "Don": 3987, "\u0120kids": 3988, "\u0120journal": 3989, "\u0120corpor": 3990, "\u0120false": 3991, "\u0120web": 3992, "\u0120sleep": 3993, "\u0120contain": 3994, "\u0120sto": 3995, "\u0120bed": 3996, "iverse": 3997, "\u0120Rich": 3998, "\u0120Chinese": 3999, "\u0120pun": 4000, "\u0120meant": 4001, "known": 4002, "\u0120notice": 4003, "\u0120favorite": 4004, "aven": 4005, "\u0120condition": 4006, "\u0120purpose": 4007, "))": 4008, "\u0120organization": 4009, "\u0120challeng": 4010, "\u0120manufact": 4011, "\u0120susp": 4012, "\u0120Ac": 4013, "\u0120critic": 4014, "unes": 4015, "uclear": 4016, "\u0120mer": 4017, "vention": 4018, "\u012080": 4019, "\u0120mist": 4020, "\u0120Us": 4021, "\u0120Tor": 4022, "http": 4023, "olf": 4024, "\u0120larger": 4025, "\u0120advant": 4026, "\u0120resear": 4027, "\u0120actions": 4028, "ml": 4029, "\u0120kept": 4030, "\u0120aim": 4031, ",'": 4032, "col": 4033, "\u0120benefits": 4034, "ifying": 4035, "\u0120actual": 4036, "\u0120International": 4037, "\u0120vehicle": 4038, "\u0120chief": 4039, "\u0120efforts": 4040, "\u0120League": 4041, "\u0120Most": 4042, "\u0120wait": 4043, "\u0120adult": 4044, "\u0120overall": 4045, "\u0120speech": 4046, "\u0120highly": 4047, "\u0120female": 4048, "\u0120error": 4049, "\u0120effective": 4050, "54": 4051, "\u0120encour": 4052, "well": 4053, "\u0120failed": 4054, "\u0120conserv": 4055, "\u0120programs": 4056, "\u0120trou": 4057, "\u0120ahead": 4058, "500": 4059, "vertisement": 4060, "IP": 4061, "\u0120Found": 4062, "pir": 4063, "\u0120%": 4064, "\u0120crime": 4065, "ander": 4066, "\u0120location": 4067, "\u0120Iran": 4068, "\u0120behavior": 4069, "azing": 4070, "\u0120rare": 4071, "\u0120emb": 4072, "\u0120caused": 4073, "\u0120ship": 4074, "\u0120active": 4075, "\u0120contribut": 4076, "\u0120green": 4077, "\u0120acqu": 4078, "\u0120reflect": 4079, "venue": 4080, "\u0120firm": 4081, "\u0120birth": 4082, "].": 4083, "\u0120clearly": 4084, "\u0120emot": 4085, "\u0120agency": 4086, "riage": 4087, "\u0120memory": 4088, "98": 4089, "SA": 4090, "\u0120See": 4091, "acing": 4092, "CC": 4093, "\u0120biggest": 4094, "\u0120rap": 4095, "\u0120basic": 4096, "\u0120band": 4097, "eat": 4098, "\u0120suspect": 4099, "\u0120Mac": 4100, "\u012090": 4101, "mark": 4102, "istan": 4103, "\u0120spread": 4104, "ams": 4105, "ki": 4106, "asy": 4107, "rav": 4108, "\u0120Rober": 4109, "\u0120demonstr": 4110, "rated": 4111, "\u0120absolute": 4112, "\u0120places": 4113, "\u0120impl": 4114, "ibrary": 4115, "\u0120cards": 4116, "\u0120destroy": 4117, "\u0120virt": 4118, "vere": 4119, "\u0120appeared": 4120, "yan": 4121, "point": 4122, "\u0120beg": 4123, "\u0120temper": 4124, "spe": 4125, "anted": 4126, "ears": 4127, "\u0120Direct": 4128, "\u0120length": 4129, "\u0120blog": 4130, "amb": 4131, "\u0120integ": 4132, "\u0120resources": 4133, "acc": 4134, "iful": 4135, "\u0120spot": 4136, "\u0120forced": 4137, "\u0120thousands": 4138, "\u0120Minister": 4139, "\u0120qual": 4140, "\u0120French": 4141, "atically": 4142, "\u0120generally": 4143, "\u0120drink": 4144, "\u0120thus": 4145, "IL": 4146, "odes": 4147, "\u0120appropri": 4148, "\u0120Read": 4149, "\u0120whom": 4150, "\u0120eye": 4151, "\u0120college": 4152, "\u012045": 4153, "irection": 4154, "\u0120ensure": 4155, "\u0120apparent": 4156, "iders": 4157, "\u0120religious": 4158, "\u0120minor": 4159, "olic": 4160, "\u0120tro": 4161, "\u0120Why": 4162, "ribute": 4163, "met": 4164, "\u0120primary": 4165, "\u0120developed": 4166, "\u0120peace": 4167, "\u0120skin": 4168, "ste": 4169, "ava": 4170, "\u0120blue": 4171, "\u0120families": 4172, "\u0120ir": 4173, "\u0120apply": 4174, "\u0120inform": 4175, "\u0120Smith": 4176, "CT": 4177, "ii": 4178, "\u0120limit": 4179, "\u0120resist": 4180, "................": 4181, "umn": 4182, "\u0120conflic": 4183, "\u0120twe": 4184, "udd": 4185, "\u0120Tom": 4186, "\u0120liter": 4187, "que": 4188, "bon": 4189, "\u0120hair": 4190, "\u0120eventually": 4191, "\u0120pus": 4192, "\u0120helped": 4193, "\u0120agg": 4194, "orney": 4195, "\u0120Apple": 4196, "\u0120fit": 4197, "\u0120Sur": 4198, "\u0120prem": 4199, "\u0120sales": 4200, "\u0120seconds": 4201, "\u0120strength": 4202, "\u0120feeling": 4203, "\u00bf\u00bd": 4204, "\u0120tour": 4205, "\u0120knows": 4206, "oom": 4207, "\u0120exerc": 4208, "\u0120somew": 4209, "\u00ef\u00bf\u00bd": 4210, ">>": 4211, "\u0120spokes": 4212, "\u0120ideas": 4213, "\u0120regist": 4214, "soft": 4215, "\u0120Del": 4216, "\u0120PC": 4217, "\u0120propos": 4218, "\u0120launch": 4219, "\u0120bottom": 4220, "TH": 4221, "\u0120Please": 4222, "vest": 4223, "itz": 4224, "\u0120Inter": 4225, "\u0120script": 4226, "\u0120rat": 4227, "arning": 4228, "\u0120il": 4229, "\u0120Jer": 4230, "\u0120Are": 4231, "\u0120whatever": 4232, "oken": 4233, "cience": 4234, "\u0120mode": 4235, "\u0120agree": 4236, "\u0120sources": 4237, "\u0120initial": 4238, "\u0120restrict": 4239, "\u0120wonder": 4240, "usion": 4241, "####": 4242, "\u0120Sil": 4243, "ville": 4244, "\u0120burn": 4245, "tw": 4246, "asion": 4247, "\u0120\u00c2\u00a3": 4248, "\u0120nor": 4249, "uing": 4250, "\u0120reached": 4251, "\u0120sun": 4252, "\u0120categ": 4253, "igration": 4254, "\u0120cook": 4255, "\u0120promot": 4256, "\u0120male": 4257, "\u0120climate": 4258, "\u0120fix": 4259, "\u0120alleged": 4260, "UR": 4261, "alled": 4262, "\u0120images": 4263, "Cont": 4264, "ota": 4265, "\u0120schools": 4266, "ios": 4267, "\u0120drop": 4268, "\u0120stream": 4269, "\u0120Mo": 4270, "\u0120previously": 4271, "aling": 4272, "\u0120pet": 4273, "\u0120double": 4274, "\u0120(@": 4275, "annel": 4276, "\u0120default": 4277, "ties": 4278, "\u0120rank": 4279, "\u0120Dec": 4280, "\u0120Council": 4281, "\u0120weapon": 4282, "\u0120stock": 4283, "\u0120analy": 4284, "\u0120Str": 4285, "\u0120picture": 4286, "\u0120Police": 4287, "ference": 4288, "\u0120century": 4289, "\u0120citizens": 4290, "\u0120onto": 4291, "\u0120expand": 4292, "\u0120hero": 4293, "\u0120Sol": 4294, "\u0120wild": 4295, "\u0120update": 4296, "\u0120customers": 4297, "ront": 4298, "def": 4299, "\u0120lik": 4300, "\u0120criminal": 4301, "\u0120Christian": 4302, "SP": 4303, "76": 4304, "\u0120leaving": 4305, "\u0120otherwise": 4306, "\u0120Dist": 4307, "\u0120basis": 4308, "52": 4309, "53": 4310, "icip": 4311, "\u0120Ber": 4312, "\u0120recommend": 4313, "\u0120floor": 4314, "\u0120crowd": 4315, "oles": 4316, "\u012070": 4317, "\u0120central": 4318, "\u0120Ev": 4319, "\u0120dream": 4320, "\u0120download": 4321, "\u0120confir": 4322, "\u0120Thom": 4323, "\u0120window": 4324, "\u0120happens": 4325, "\u0120unit": 4326, "\u0120tend": 4327, "\u0120spl": 4328, "\u0120becomes": 4329, "\u0120fighting": 4330, "\u0120predict": 4331, "\u0120Press": 4332, "\u0120Power": 4333, "\u0120heavy": 4334, "aked": 4335, "\u0120fan": 4336, "orter": 4337, "ategy": 4338, "BA": 4339, "izes": 4340, "\u0120spend": 4341, "Here": 4342, "\u01202007": 4343, "\u0120adop": 4344, "\u0120Ham": 4345, "\u0120football": 4346, "\u0120Port": 4347, "oday": 4348, "51": 4349, "ampions": 4350, "\u0120transfer": 4351, "ht": 4352, "\u012038": 4353, "term": 4354, "acity": 4355, "\u0120bur": 4356, "],": 4357, "ternal": 4358, "rig": 4359, "but": 4360, "\u0120therefore": 4361, "\u0120Because": 4362, "resp": 4363, "rey": 4364, "\u0120mission": 4365, "Some": 4366, "\u0120noted": 4367, "\u0120assum": 4368, "\u0120disease": 4369, "\u0120edit": 4370, "\u0120progress": 4371, "rd": 4372, "\u0120Brown": 4373, "ocal": 4374, "\u0120adding": 4375, "\u0120raised": 4376, "\u0120Any": 4377, "\u0120tick": 4378, "\u0120seeing": 4379, "\u0120People": 4380, "\u0120agreement": 4381, "\u0120server": 4382, "\u0120wat": 4383, "\u0120debate": 4384, "\u0120supposed": 4385, "iling": 4386, "\u0120largest": 4387, "\u0120successful": 4388, "\u0120Pri": 4389, "\u0120Democratic": 4390, "\u0120jump": 4391, "\u0120Syria": 4392, "\u0120owners": 4393, "\u0120offers": 4394, "\u0120shooting": 4395, "\u0120effic": 4396, "sey": 4397, "\u0120haven": 4398, "verse": 4399, "tered": 4400, "\u0120Light": 4401, "imal": 4402, "\u0120Big": 4403, "\u0120defend": 4404, "\u0120beat": 4405, "\u0120records": 4406, "%)": 4407, "\u0120scen": 4408, "\u0120employees": 4409, "\u0120devices": 4410, "hem": 4411, "\u0120commer": 4412, "\u0120Mex": 4413, "\u0120benefit": 4414, "\u0120Prof": 4415, "\u0120illeg": 4416, "\u0120surface": 4417, "\u0120Also": 4418, "\u0120harm": 4419, "ingly": 4420, "wide": 4421, "\u0120Alex": 4422, "\u0120shut": 4423, "\u0120Cur": 4424, "\u0120lose": 4425, "pm": 4426, "\u0120challenge": 4427, "semb": 4428, "\u0120station": 4429, "\u0120intelligence": 4430, "\u0120accur": 4431, "\u0120Flor": 4432, "\u0120requires": 4433, "\u0120Mal": 4434, "bum": 4435, "\u0120hospital": 4436, "\u0120spirit": 4437, "\u0120offered": 4438, "\u0120produce": 4439, "\u0120Commun": 4440, "\u0120creating": 4441, "\u0120cris": 4442, "spect": 4443, "\u0120ended": 4444, "\u0120daily": 4445, "\u0120voters": 4446, "lands": 4447, "ias": 4448, "ih": 4449, "ona": 4450, "\u0120smart": 4451, "\u0120Office": 4452, "\u0120Lord": 4453, "rial": 4454, "\u0120Internet": 4455, "\u0120circum": 4456, "\u0120extremely": 4457, "'.": 4458, "\u0120opinion": 4459, "\u0120Mil": 4460, "\u0120gain": 4461, "BS": 4462, "\u0120Fin": 4463, "yp": 4464, "\u0120useful": 4465, "\u0120budget": 4466, "\u0120comfort": 4467, "isf": 4468, "\u0120background": 4469, "eline": 4470, "\u0120episode": 4471, "\u0120enemy": 4472, "\u0120trial": 4473, "\u0120establish": 4474, "date": 4475, "\u0120Cap": 4476, "\u0120continues": 4477, "\u0120showing": 4478, "\u0120Union": 4479, "with": 4480, "\u0120posted": 4481, "\u0120System": 4482, "\u0120eat": 4483, "rian": 4484, "\u0120rise": 4485, "\u0120Germany": 4486, "ils": 4487, "\u0120signed": 4488, "\u0120vill": 4489, "\u0120grand": 4490, "mor": 4491, "\u0120England": 4492, "\u0120projects": 4493, "umber": 4494, "\u0120conference": 4495, "za": 4496, "\u0120responsible": 4497, "\u0120Arab": 4498, "\u0120learned": 4499, "\u00e2\u0122\u0136\u00e2\u0122\u0136": 4500, "ipping": 4501, "\u0120George": 4502, "OC": 4503, "\u0120returned": 4504, "\u0120Australia": 4505, "\u0120brief": 4506, "Qu": 4507, "\u0120brand": 4508, "illing": 4509, "abled": 4510, "\u0120highest": 4511, "\u0120train": 4512, "\u0120Commission": 4513, "while": 4514, "\u0120nom": 4515, "ception": 4516, "\u0120mut": 4517, "\u0120Blue": 4518, "\u0120incident": 4519, "vant": 4520, "86": 4521, "\u0120ID": 4522, "\u0120nuclear": 4523, "74": 4524, "\u0120Like": 4525, "\u0120RE": 4526, "\u0120Micro": 4527, "li": 4528, "mail": 4529, "\u0120charges": 4530, "89": 4531, "\u0120adjust": 4532, "ado": 4533, "\u0120earth": 4534, "NA": 4535, "\u0120prices": 4536, "PA": 4537, "\u0120draft": 4538, "\u0120runs": 4539, "\u0120candidate": 4540, "enses": 4541, "\u0120management": 4542, "\u0120Phil": 4543, "\u0120Miss": 4544, "\u0120teach": 4545, "gram": 4546, "\u0120understanding": 4547, "ait": 4548, "icago": 4549, "Add": 4550, "\u0120Ep": 4551, "secut": 4552, "\u0120separate": 4553, "\u0120instance": 4554, "\u0120eth": 4555, "\u0120unless": 4556, "********": 4557, "\u0120Fore": 4558, "inate": 4559, "\u0120operations": 4560, "Sp": 4561, "\u0120faith": 4562, "gar": 4563, "\u0120Church": 4564, "ronic": 4565, "\u0120config": 4566, "osure": 4567, "\u0120activities": 4568, "\u0120traditional": 4569, "\u012036": 4570, "\u0120direction": 4571, "\u0120machine": 4572, "\u0120surround": 4573, "\u0120push": 4574, "unction": 4575, "\u0120EU": 4576, "\u0120easier": 4577, "\u0120argument": 4578, "GB": 4579, "\u0120micro": 4580, "\u0120spending": 4581, "izations": 4582, "\u0120theory": 4583, "adow": 4584, "\u0120calling": 4585, "\u0120Last": 4586, "\u0120der": 4587, "\u0120influence": 4588, "\u0120commit": 4589, "\u0120photo": 4590, "\u0120unc": 4591, "istry": 4592, "gn": 4593, "aste": 4594, "acks": 4595, "\u0120disp": 4596, "ady": 4597, "do": 4598, "\u0120Good": 4599, "\u0120`": 4600, "\u0120wish": 4601, "\u0120revealed": 4602, "\u00c2\u0142\u00c2\u0142": 4603, "lig": 4604, "\u0120enforce": 4605, "\u0120Committee": 4606, "\u0120chem": 4607, "\u0120miles": 4608, "\u0120interested": 4609, "\u0120solution": 4610, "icy": 4611, "inct": 4612, "\u0120->": 4613, "\u0120Det": 4614, "\u0120removed": 4615, "\u0120compar": 4616, "eah": 4617, "\u0120plant": 4618, "\u0120Since": 4619, "\u0120achieve": 4620, "\u0120advantage": 4621, "\u0120slightly": 4622, "bing": 4623, "\u0120placed": 4624, "under": 4625, "2015": 4626, "\u0120Mad": 4627, "\u0120tim": 4628, "oses": 4629, "\u0120cru": 4630, "\u0120Rock": 4631, "\u0120mostly": 4632, "\u0120negative": 4633, "\u0120setting": 4634, "\u0120produced": 4635, "\u0120mur": 4636, "\u0120connection": 4637, "\u0120Mer": 4638, "\u0120driver": 4639, "\u0120executive": 4640, "\u0120assault": 4641, "\u0120born": 4642, "\u0120Ver": 4643, "tained": 4644, "\u0120structure": 4645, "\u0120reduce": 4646, "\u0120decades": 4647, "\u0120ded": 4648, "uke": 4649, "\u0120Many": 4650, "idden": 4651, "\u0120league": 4652, "Se": 4653, "\u0120join": 4654, "\u0120disco": 4655, "\u0120die": 4656, "cks": 4657, "actions": 4658, "\u0120assess": 4659, "agn": 4660, "\u0120goals": 4661, "ours": 4662, "IR": 4663, "\u0120senior": 4664, "iller": 4665, "mod": 4666, "ipment": 4667, "ocol": 4668, "uy": 4669, "\u0120Que": 4670, "\u0120parties": 4671, "irgin": 4672, "\u0120learning": 4673, "itable": 4674, "\u0120street": 4675, "\u0120camera": 4676, "App": 4677, "\u0120skills": 4678, "bre": 4679, "cious": 4680, "\u0120celebr": 4681, "\u0120Franc": 4682, "\u0120existing": 4683, "\u0120willing": 4684, "lor": 4685, "\u0120id": 4686, "\u0120Space": 4687, "\u0120critical": 4688, "\u0120La": 4689, "ortunately": 4690, "\u0120serve": 4691, "\u0120cold": 4692, "\u0120species": 4693, "TS": 4694, "\u0120animals": 4695, "\u0120Bay": 4696, "\u0120older": 4697, "\u0120Under": 4698, "estic": 4699, "\u0120Tre": 4700, "\u0120teacher": 4701, "\u0120prefer": 4702, "vis": 4703, "\u0120thread": 4704, "\u0120Matt": 4705, "\u0120manager": 4706, "\u00e3\u0125\u00bb": 4707, "\u0120professional": 4708, "\u0120Vol": 4709, "\u0120notes": 4710, "These": 4711, "ula": 4712, "\u0120fresh": 4713, "ented": 4714, "uzz": 4715, "edy": 4716, "clusion": 4717, "\u0120Rel": 4718, "\u0120doubt": 4719, "EO": 4720, "\u0120opened": 4721, "\u0120Bit": 4722, "Advertisement": 4723, "\u0120guess": 4724, "\u0120UN": 4725, "\u0120sequ": 4726, "\u0120explain": 4727, "otten": 4728, "\u0120attract": 4729, "aks": 4730, "\u0120string": 4731, "\u0120context": 4732, "ossible": 4733, "\u0120Republicans": 4734, "\u0120solid": 4735, "\u0120cities": 4736, "\u0120asking": 4737, "\u0120random": 4738, "ups": 4739, "uries": 4740, "arant": 4741, "dden": 4742, "gl": 4743, "\u0120Florida": 4744, "\u0120depend": 4745, "\u0120Scott": 4746, "\u012033": 4747, "\u0120iT": 4748, "icon": 4749, "\u0120mentioned": 4750, "\u01202000": 4751, "\u0120claimed": 4752, "\u0120definitely": 4753, "ulf": 4754, "\u0120core": 4755, "\u0120opening": 4756, "\u0120Const": 4757, "which": 4758, "\u0120Tra": 4759, "AG": 4760, "72": 4761, "\u0120believed": 4762, "ada": 4763, "\u012048": 4764, "\u0120Security": 4765, "yright": 4766, "\u0120Pet": 4767, "\u0120Lou": 4768, "\u0120holding": 4769, "================": 4770, "\u0120ice": 4771, "\u0120brow": 4772, "\u0120authorities": 4773, "host": 4774, "word": 4775, "\u0120score": 4776, "\u0120Div": 4777, "\u0120cells": 4778, "\u0120transl": 4779, "\u0120neighbor": 4780, "\u0120remove": 4781, "uct": 4782, "\u0120district": 4783, "\u0120According": 4784, "\u0120worse": 4785, "\u0120concerns": 4786, "\u0120presidential": 4787, "\u0120policies": 4788, "\u0120Hall": 4789, "73": 4790, "\u0120hus": 4791, "AY": 4792, "\u01202006": 4793, "\u0120Jud": 4794, "\u0120independent": 4795, "\u0120Justice": 4796, "iliar": 4797, "print": 4798, "ighter": 4799, "\u0120protection": 4800, "zen": 4801, "\u0120sudden": 4802, "house": 4803, "\u0120Jes": 4804, "PR": 4805, "\u0120Inf": 4806, "\u0120bul": 4807, "\u0120_": 4808, "\u0120Service": 4809, "\u0120PR": 4810, "\u0120strategy": 4811, "ffect": 4812, "\u0120girls": 4813, "\u0120missing": 4814, "oyal": 4815, "\u0120Team": 4816, "ulated": 4817, "\u0120dat": 4818, "\u0120politics": 4819, "abor": 4820, "According": 4821, "\u0120spell": 4822, "\u0120graph": 4823, "orthern": 4824, "TC": 4825, "Ab": 4826, "\u0120labor": 4827, "isher": 4828, "\u0120kick": 4829, "\u0120iTunes": 4830, "\u0120steps": 4831, "poses": 4832, "\u0120smaller": 4833, "En": 4834, "bert": 4835, "\u0120roll": 4836, "\u0120researchers": 4837, "\u0120closed": 4838, "\u0120transport": 4839, "\u0120lawy": 4840, "________________": 4841, "\u0120Chicago": 4842, "\u0120aspect": 4843, "\u0120none": 4844, "\u0120marriage": 4845, "96": 4846, "\u0120elements": 4847, "\u0120Fre": 4848, "\u0120Sal": 4849, "\u0120dram": 4850, "FC": 4851, "top": 4852, "equ": 4853, "\u0120hearing": 4854, "\u0120supported": 4855, "\u0120testing": 4856, "cohol": 4857, "\u0120massive": 4858, "\u0120stick": 4859, "\u0120guard": 4860, "isco": 4861, "phone": 4862, "From": 4863, "However": 4864, "\u0120border": 4865, "\u0120copy": 4866, "ography": 4867, "list": 4868, "71": 4869, "\u0120owner": 4870, "class": 4871, "ruit": 4872, "rate": 4873, "\u0120Once": 4874, "\u0120digital": 4875, "\u0120task": 4876, "ERS": 4877, "\u0120incred": 4878, "tes": 4879, "++": 4880, "\u0120France": 4881, "\u0120breat": 4882, "owl": 4883, "\u0120issued": 4884, "\u0120Western": 4885, "\u0120detect": 4886, "\u0120partners": 4887, "\u0120shared": 4888, "\u0120Call": 4889, "\u0120cancer": 4890, "ache": 4891, "ribe": 4892, "\u0120explained": 4893, "\u0120heat": 4894, "{\"": 4895, "\u0120investment": 4896, "\u0120Book": 4897, "\u0120wood": 4898, "\u0120tools": 4899, "\u0120Although": 4900, "\u0120belief": 4901, "\u0120crisis": 4902, "\u0120ge": 4903, "\u0120MP": 4904, "\u0120operation": 4905, "type": 4906, "~~": 4907, "ga": 4908, "\u0120contains": 4909, "anta": 4910, "\u0120express": 4911, "\u0120Group": 4912, "\u0120Journal": 4913, "ka": 4914, "\u0120amb": 4915, "\u0120USA": 4916, "\u0120finding": 4917, "\u0120funding": 4918, "how": 4919, "\u0120established": 4920, "ideos": 4921, "\u0120degree": 4922, "\u0120dangerous": 4923, "anging": 4924, "\u0120freedom": 4925, "pport": 4926, "outhern": 4927, "\u0120church": 4928, "\u0120catch": 4929, "\u0120Two": 4930, "\u0120presence": 4931, "\u0120Guard": 4932, "Up": 4933, "\u0120authority": 4934, "\u0120Project": 4935, "\u0120button": 4936, "\u0120consequ": 4937, "\u0120valid": 4938, "\u0120weak": 4939, "\u0120starts": 4940, "\u0120reference": 4941, "\u0120Mem": 4942, "\")": 4943, "UN": 4944, "orage": 4945, "\u0120Open": 4946, "\u0120collection": 4947, "ym": 4948, "gency": 4949, "\u0120beautiful": 4950, "ros": 4951, "\u0120tells": 4952, "\u0120waiting": 4953, "nel": 4954, "\u0120providing": 4955, "\u0120Democrats": 4956, "\u0120daughter": 4957, "\u0120master": 4958, "\u0120purposes": 4959, "\u0120Japanese": 4960, "\u0120equal": 4961, "\u0120turns": 4962, "\u0120documents": 4963, "\u0120watching": 4964, "Res": 4965, "\u0120ran": 4966, "2014": 4967, "\u0120reject": 4968, "\u0120Korea": 4969, "\u0120victims": 4970, "Level": 4971, "erences": 4972, "\u0120witness": 4973, "\u012034": 4974, "\u0120reform": 4975, "coming": 4976, "\u0120occup": 4977, "\u0120caught": 4978, "\u0120traffic": 4979, "ading": 4980, "\u0120models": 4981, "ario": 4982, "\u0120served": 4983, "\u0120batter": 4984, "uate": 4985, "\u0120Secretary": 4986, "\u0120agreed": 4987, "\u0120truly": 4988, "ynam": 4989, "\u0120Ret": 4990, "\u0120units": 4991, "\u0120Research": 4992, "hand": 4993, "azine": 4994, "\u0120Mike": 4995, "\u0120variety": 4996, "otal": 4997, "\u0120amazing": 4998, "\u0120confirmed": 4999, "\u0120entirely": 5000, "\u0120purchase": 5001, "\u0120element": 5002, "\u0120cash": 5003, "\u0120determine": 5004, "De": 5005, "\u0120cars": 5006, "\u0120Wall": 5007, "\u00e2\u0138": 5008, "\u0120views": 5009, "\u0120drugs": 5010, "\u0120department": 5011, "\u0120Step": 5012, "uit": 5013, "\u012039": 5014, "asure": 5015, "\u0120Class": 5016, "\u0120covered": 5017, "\u0120Bank": 5018, "\u0120mere": 5019, "uana": 5020, "\u0120multi": 5021, "\u0120mix": 5022, "\u0120unlike": 5023, "levision": 5024, "\u0120stopped": 5025, "\u0120sem": 5026, "\u0120Gal": 5027, "ules": 5028, "\u0120wel": 5029, "\u0120Johnson": 5030, "la": 5031, "\u0120skill": 5032, "\u0120becoming": 5033, "rie": 5034, "\u0120appropriate": 5035, "fe": 5036, "ellow": 5037, "\u0120Prot": 5038, "ulate": 5039, "ocation": 5040, "\u0120weekend": 5041, "odies": 5042, "\u0120sites": 5043, "\u0120animal": 5044, "\u0120Tim": 5045, "\u0120scale": 5046, "\u0120charged": 5047, "\u0120instruct": 5048, "illa": 5049, "\u0120methods": 5050, "\u0120cert": 5051, "\u0120judge": 5052, "\u0120Hel": 5053, "\u0120dollars": 5054, "\u0120standing": 5055, "\u0120Squ": 5056, "\u0120debt": 5057, "liam": 5058, "\u0120driving": 5059, "\u0120Sum": 5060, "\u0120Edition": 5061, "\u0120album": 5062, "andon": 5063, "IF": 5064, "\u0120Uk": 5065, "63": 5066, "ader": 5067, "\u0120commercial": 5068, "esh": 5069, "\u0120Government": 5070, "\u0120discovered": 5071, "\u0120output": 5072, "\u0120Hillary": 5073, "\u0120Carol": 5074, "\u01202005": 5075, "\u0120abuse": 5076, "ancing": 5077, "\u0120switch": 5078, "\u0120annual": 5079, "Tw": 5080, "\u0120stated": 5081, "agement": 5082, "inner": 5083, "\u0120democr": 5084, "\u0120residents": 5085, "\u0120allowing": 5086, "\u0120factors": 5087, "odd": 5088, "\u0120fuck": 5089, "emies": 5090, "\u0120occurred": 5091, "oti": 5092, "\u0120north": 5093, "\u0120Public": 5094, "\u0120injury": 5095, "\u0120insurance": 5096, "CL": 5097, "olly": 5098, "\u00e3\u0122": 5099, "\u0120repeated": 5100, "\u0120arms": 5101, "anged": 5102, "\u0120construction": 5103, "\u0120fle": 5104, "PU": 5105, "icians": 5106, "\u0120forms": 5107, "\u0120McC": 5108, "antic": 5109, "\u0120mental": 5110, "pire": 5111, "\u0120equipment": 5112, "\u0120fant": 5113, "\u0120discussion": 5114, "\u0120regarding": 5115, "kin": 5116, "arp": 5117, "\u0120chair": 5118, "ogue": 5119, "\u0120proceed": 5120, "\u0120Id": 5121, "Our": 5122, "\u0120murder": 5123, "Man": 5124, "\u012049": 5125, "asp": 5126, "\u0120supply": 5127, "\u0120input": 5128, "\u0120wealth": 5129, "liament": 5130, "\u0120proced": 5131, "orial": 5132, "\u0120Stat": 5133, "\u0120NFL": 5134, "hens": 5135, "\u0120Institute": 5136, "\u0120putting": 5137, "ournament": 5138, "etic": 5139, "\u0120located": 5140, "\u0120kid": 5141, "eria": 5142, "run": 5143, "\u0120princ": 5144, "\u0120!": 5145, "going": 5146, "\u0120Bet": 5147, "\u0120clot": 5148, "\u0120telling": 5149, "\u0120proposed": 5150, "iot": 5151, "orry": 5152, "\u0120funds": 5153, "gment": 5154, "\u0120Life": 5155, "\u0120baby": 5156, "\u0120Back": 5157, "\u0120spoke": 5158, "Image": 5159, "\u0120earn": 5160, "\u0120AT": 5161, "gu": 5162, "\u0120exchange": 5163, "\u0120Lin": 5164, "oving": 5165, "\u0120pair": 5166, "More": 5167, "azon": 5168, "\u0120arrested": 5169, "\u0120killing": 5170, "can": 5171, "\u0120Card": 5172, "yd": 5173, "\u0120identified": 5174, "\u0120mobile": 5175, "\u0120thanks": 5176, "onym": 5177, "\u0120Form": 5178, "\u0120hundreds": 5179, "\u0120Chris": 5180, "\u0120Cat": 5181, "\u0120trend": 5182, "hat": 5183, "\u0120Av": 5184, "oman": 5185, "\u0120electric": 5186, "\u0120Wil": 5187, "SE": 5188, "Of": 5189, "\u0120restaur": 5190, "oted": 5191, "\u0120trig": 5192, "\u0120nine": 5193, "\u0120bomb": 5194, "Why": 5195, "\u00c2\u00af": 5196, "\u0120coverage": 5197, "\u0120appeal": 5198, "\u0120Robert": 5199, "\u0120Sup": 5200, "\u0120finished": 5201, "\u0120flow": 5202, "\u0120deliver": 5203, "\u0120calcul": 5204, "\u0120photos": 5205, "\u0120phil": 5206, "\u0120pieces": 5207, "\u0120appre": 5208, "kes": 5209, "\u0120rough": 5210, "Do": 5211, "\u0120partner": 5212, "\u0120concerned": 5213, "\u012037": 5214, "\u0120Gen": 5215, "Col": 5216, "ctors": 5217, "\u0120=>": 5218, "state": 5219, "\u0120suggested": 5220, "\u0120Force": 5221, "CE": 5222, "\u0120herself": 5223, "\u0120Plan": 5224, "works": 5225, "ooth": 5226, "rency": 5227, "\u0120corner": 5228, "\u0120husband": 5229, "\u0120internet": 5230, "\u0120Aut": 5231, "ems": 5232, "osen": 5233, "\u0120Atl": 5234, "gen": 5235, "\u0120balance": 5236, "62": 5237, "\u0120sounds": 5238, "text": 5239, "\u0120arr": 5240, "oves": 5241, "\u0120millions": 5242, "\u0120radio": 5243, "\u0120satisf": 5244, "\u0120Dam": 5245, "Mr": 5246, "Go": 5247, "Spe": 5248, "\u0120combat": 5249, "rant": 5250, "\u0120Gree": 5251, "\u0120fuel": 5252, "\u0120distance": 5253, "\u0120tests": 5254, "\u0120decre": 5255, "\u0120Er": 5256, "\u0120managed": 5257, "DS": 5258, "\u0120tit": 5259, "\u0120measures": 5260, "\u0120Liber": 5261, "\u0120attend": 5262, "ashed": 5263, "\u0120Jose": 5264, "\u0120Night": 5265, "dit": 5266, "\u0120Nov": 5267, "\u0120End": 5268, "outs": 5269, "\u0120generation": 5270, "\u0120advoc": 5271, "yth": 5272, "\u0120conversation": 5273, "\u0120Sky": 5274, "active": 5275, "cel": 5276, "rier": 5277, "\u0120Frank": 5278, "\u0120gender": 5279, "\u0120concent": 5280, "\u0120carried": 5281, "anda": 5282, "\u0120Virgin": 5283, "\u0120arrived": 5284, "icide": 5285, "aded": 5286, "\u0120failure": 5287, "\u0120minimum": 5288, "lets": 5289, "\u0120worst": 5290, "\u0120keeping": 5291, "\u0120intended": 5292, "\u0120illegal": 5293, "\u0120subsc": 5294, "\u0120determined": 5295, "\u0120trip": 5296, "Yes": 5297, "\u0120raise": 5298, "\u0120~": 5299, "\u0120feels": 5300, "\u0120package": 5301, "\u0120Jo": 5302, "hi": 5303, "2016": 5304, "real": 5305, "\u0120fra": 5306, "\u0120symb": 5307, "Me": 5308, "ucky": 5309, "pret": 5310, "\u0120Kh": 5311, "\u0120Edit": 5312, "\u0120Web": 5313, "emic": 5314, "\u0120Color": 5315, "\u0120justice": 5316, "Int": 5317, "\u0120farm": 5318, "cknow": 5319, "\">": 5320, "eless": 5321, "\u0120reduced": 5322, "\u0120500": 5323, "xx": 5324, "\u0120Rad": 5325, "\u0120Wood": 5326, "\u0120clin": 5327, "\u0120hyp": 5328, "iler": 5329, "ura": 5330, "kins": 5331, "85": 5332, "61": 5333, "\u0120Their": 5334, "\u0120Mary": 5335, "\u0120san": 5336, "\u0120novel": 5337, "\u0120Who": 5338, "\u0120capacity": 5339, "\u0120impossible": 5340, "\u0120plays": 5341, "\u0120minister": 5342, "ijuana": 5343, "icate": 5344, "\u0120Set": 5345, "\u0120fram": 5346, "\u0120ing": 5347, "\u0120communities": 5348, "\u0120FBI": 5349, "ita": 5350, "\u0120bon": 5351, "\u0120strateg": 5352, "\u0120interests": 5353, "lock": 5354, "gers": 5355, "mas": 5356, "\u0120AND": 5357, "\u0120conflict": 5358, "\u0120requirements": 5359, "\u0120sac": 5360, "\u0120operating": 5361, "ini": 5362, "related": 5363, "\u0120committed": 5364, "\u0120relatively": 5365, "\u0120south": 5366, "\u00c2\u00af\u00c2\u00af": 5367, "\u0120afford": 5368, "\u0120identity": 5369, "\u0120decisions": 5370, "\u0120accused": 5371, "place": 5372, "\u0120victory": 5373, "och": 5374, "iat": 5375, "Name": 5376, "Com": 5377, "tion": 5378, "eds": 5379, "\u0120seek": 5380, "\u0120tight": 5381, "\u0120Images": 5382, "\u0120initi": 5383, "\u0120humans": 5384, "\u0120familiar": 5385, "\u0120audience": 5386, "\u0120internal": 5387, "venture": 5388, "\u0120sides": 5389, "\u0120TO": 5390, "\u0120dim": 5391, "\u0120conclud": 5392, "\u0120appoint": 5393, "\u0120enforcement": 5394, "\u0120Jim": 5395, "\u0120Association": 5396, "\u0120circumst": 5397, "\u0120Canadian": 5398, "\u0120joined": 5399, "\u0120differences": 5400, "\u0120Los": 5401, "\u0120protest": 5402, "\u0120twice": 5403, "win": 5404, "\u0120glass": 5405, "arsh": 5406, "\u0120Army": 5407, "\u0120expression": 5408, "\u0120decide": 5409, "\u0120planning": 5410, "ania": 5411, "\u0120handle": 5412, "\u0120Microsoft": 5413, "\u0120Nor": 5414, "\u0120maximum": 5415, "\u0120Rev": 5416, "\u0120sea": 5417, "\u0120eval": 5418, "\u0120helps": 5419, "ref": 5420, "\u0120bound": 5421, "\u0120mouth": 5422, "\u0120standards": 5423, "\u0120clim": 5424, "\u0120Camp": 5425, "\u0120Fox": 5426, "cles": 5427, "\u0120army": 5428, "\u0120Techn": 5429, "acking": 5430, "xy": 5431, "SS": 5432, "\u012042": 5433, "\u0120bug": 5434, "\u0120Ukrain": 5435, "\u0120Max": 5436, "\u0120Jones": 5437, "\u0120Show": 5438, "lo": 5439, "\u0120planet": 5440, "\u012075": 5441, "\u0120winning": 5442, "\u0120faster": 5443, "\u0120spect": 5444, "\u0120broken": 5445, "TR": 5446, "\u0120defined": 5447, "\u0120healthy": 5448, "\u0120competition": 5449, "https": 5450, "\u0120Island": 5451, "\u0120Fe": 5452, "\u0120announce": 5453, "\u0120Cup": 5454, "\u0120Instead": 5455, "\u0120client": 5456, "\u0120possibly": 5457, "section": 5458, "ocket": 5459, "look": 5460, "\u0120finish": 5461, "\u0120crew": 5462, "\u0120reserv": 5463, "\u0120editor": 5464, "\u0120hate": 5465, "\u0120sale": 5466, "\u0120controvers": 5467, "\u0120pages": 5468, "wing": 5469, "\u0120numer": 5470, "\u0120opposition": 5471, "\u01202004": 5472, "\u0120refuge": 5473, "\u0120flight": 5474, "\u0120apart": 5475, "\u0120Lat": 5476, "Americ": 5477, "\u0120Africa": 5478, "\u0120applications": 5479, "\u0120Palest": 5480, "\u0120Bur": 5481, "\u0120gar": 5482, "\u0120Social": 5483, "\u0120upgr": 5484, "\u0120shape": 5485, "\u0120speaking": 5486, "ansion": 5487, "ao": 5488, "\u0120Sn": 5489, "\u0120worry": 5490, "\u0120Britain": 5491, "Please": 5492, "roud": 5493, "\u0120hun": 5494, "\u0120introduced": 5495, "\u0120diet": 5496, "Ind": 5497, "\u0120Second": 5498, "\u0120functions": 5499, "uts": 5500, "\u0120Each": 5501, "\u0120Jeff": 5502, "\u0120stress": 5503, "\u0120accounts": 5504, "\u0120guarant": 5505, "\u0120Ann": 5506, "edia": 5507, "\u0120honest": 5508, "\u0120tree": 5509, "\u0120African": 5510, "\u0120Bush": 5511, "},": 5512, "\u0120sch": 5513, "\u0120Only": 5514, "\u0120fif": 5515, "igan": 5516, "\u0120exercise": 5517, "\u0120Exp": 5518, "\u0120scientists": 5519, "\u0120legislation": 5520, "\u0120Work": 5521, "\u0120Spr": 5522, "\u00c3\u0124": 5523, "\u0120Human": 5524, "\u0120\u00e8": 5525, "\u0120survey": 5526, "\u0120rich": 5527, "rip": 5528, "\u0120maintain": 5529, "\u0120flo": 5530, "\u0120leadership": 5531, "stream": 5532, "\u0120Islamic": 5533, "\u012001": 5534, "\u0120College": 5535, "\u0120magic": 5536, "\u0120Prime": 5537, "\u0120figures": 5538, "2017": 5539, "inder": 5540, "xual": 5541, "\u0120Dead": 5542, "\u0120absolutely": 5543, "\u0120fourth": 5544, "\u0120presented": 5545, "respond": 5546, "rible": 5547, "\u0120alcohol": 5548, "ato": 5549, "\u0120DE": 5550, "porary": 5551, "\u0120grab": 5552, "\u0120vari": 5553, "\u0120quant": 5554, "\u0120Photo": 5555, "\u0120plus": 5556, "rick": 5557, "arks": 5558, "\u0120alternative": 5559, "\u0120pil": 5560, "\u0120approx": 5561, "that": 5562, "\u0120objects": 5563, "\u0120Ro": 5564, "\u0120Android": 5565, "\u0120significantly": 5566, "\u0120Road": 5567, "kay": 5568, "Read": 5569, "avor": 5570, "\u0120acknow": 5571, "\u0120HD": 5572, "\u0120Sing": 5573, "Or": 5574, "\u0120Mont": 5575, "\u0120uns": 5576, "prof": 5577, "\u0120negoti": 5578, "\u0120Arch": 5579, "iki": 5580, "\u0120television": 5581, "\u0120Jewish": 5582, "\u0120committee": 5583, "\u0120motor": 5584, "\u0120appearance": 5585, "\u0120sitting": 5586, "\u0120strike": 5587, "\u0120Down": 5588, "comp": 5589, "\u0120Hist": 5590, "\u0120fold": 5591, "acement": 5592, "\u0120Louis": 5593, "\u0120belong": 5594, "\u0120\u00e2\u0122\u00a2": 5595, "\u0120mort": 5596, "\u0120prepared": 5597, "\u012064": 5598, "\u0120Master": 5599, "\u0120indeed": 5600, "\u0120Den": 5601, "\u0120rent": 5602, "TA": 5603, "ourney": 5604, "arc": 5605, "Su": 5606, "97": 5607, "\u0120advice": 5608, "\u0120changing": 5609, "\u0120listed": 5610, "\u0120launched": 5611, "isation": 5612, "\u0120Peter": 5613, "ishes": 5614, "\u0120lived": 5615, "\u0120Mel": 5616, "\u0120Supreme": 5617, "\u0120Federal": 5618, "\u0120);": 5619, "ructure": 5620, "\u0120sets": 5621, "\u0120philos": 5622, "uous": 5623, "\u0120\u00c2\u0142": 5624, "\u0120applied": 5625, "\u0120NOT": 5626, "\u0120housing": 5627, "\u0120Mount": 5628, "\u0120odd": 5629, "\u0120sust": 5630, "DA": 5631, "fficient": 5632, "\u0120?": 5633, "olved": 5634, "\u0120powers": 5635, "\u0120thr": 5636, "\u0120remaining": 5637, "\u0120Water": 5638, "LC": 5639, "\u0120causes": 5640, "\u00e3\u0123\u00ae": 5641, "\u0120manner": 5642, "ads": 5643, "\u0120suggests": 5644, "\u0120ends": 5645, "standing": 5646, "fig": 5647, "\u0120Dun": 5648, "idth": 5649, "\u0120gay": 5650, "\u0120termin": 5651, "\u0120Angeles": 5652, "MS": 5653, "\u0120scientific": 5654, "\u0120coal": 5655, "apers": 5656, "bar": 5657, "\u0120Thomas": 5658, "\u0120sym": 5659, "\u0120Run": 5660, "this": 5661, "PC": 5662, "igrants": 5663, "\u0120minute": 5664, "\u0120District": 5665, "cellent": 5666, "\u0120leaves": 5667, "\u0120completed": 5668, "amin": 5669, "\u0120focused": 5670, "\u0120monitor": 5671, "\u0120vehicles": 5672, "MA": 5673, "\u0120Mass": 5674, "\u0120Grand": 5675, "\u0120affected": 5676, "itutional": 5677, "\u0120construct": 5678, "\u0120follows": 5679, "\u0120ton": 5680, "reens": 5681, "\u0120homes": 5682, "\u0120Ext": 5683, "\u0120Level": 5684, "rast": 5685, "\u0120Ir": 5686, "\u0120elim": 5687, "\u0120largely": 5688, "\u0120Joe": 5689, "\u0120votes": 5690, "alls": 5691, "\u0120businesses": 5692, "\u0120Foundation": 5693, "\u0120Central": 5694, "\u0120yards": 5695, "\u0120materials": 5696, "ulner": 5697, "\u0120guide": 5698, "\u0120closer": 5699, "ums": 5700, "\u0120sports": 5701, "eder": 5702, "Just": 5703, "\u0120taxes": 5704, "84": 5705, "\u0120Old": 5706, "\u0120decade": 5707, "ola": 5708, "\u0120vir": 5709, "\u0120dropped": 5710, "\u0120delay": 5711, "itect": 5712, "\u0120secure": 5713, "stein": 5714, "level": 5715, "\u0120treated": 5716, "\u0120filed": 5717, "aine": 5718, "\u0120van": 5719, "\u0120mir": 5720, "\u0120column": 5721, "icted": 5722, "eper": 5723, "\u0120rot": 5724, "\u0120consult": 5725, "\u0120entry": 5726, "\u0120marijuana": 5727, "\u0120Dou": 5728, "\u0120apparently": 5729, "oking": 5730, "clusive": 5731, "\u0120increases": 5732, "ano": 5733, "\u0120specifically": 5734, "\u0120tele": 5735, "ensions": 5736, "\u0120religion": 5737, "abilities": 5738, "\u0120frame": 5739, "\u0120Note": 5740, "\u0120Lee": 5741, "\u0120helping": 5742, "\u0120edge": 5743, "oston": 5744, "\u0120organizations": 5745, "\u00c3\u0125": 5746, "\u0120Both": 5747, "hips": 5748, "\u0120bigger": 5749, "\u0120boost": 5750, "\u0120Stand": 5751, "\u0120row": 5752, "uls": 5753, "abase": 5754, "\u0120rid": 5755, "Let": 5756, "aren": 5757, "rave": 5758, "\u0120stret": 5759, "PD": 5760, "\u0120vision": 5761, "\u0120wearing": 5762, "\u0120appreci": 5763, "\u0120award": 5764, "\u0120Use": 5765, "\u0120factor": 5766, "war": 5767, "ulations": 5768, ")(": 5769, "\u0120god": 5770, "\u0120territ": 5771, "\u0120param": 5772, "asts": 5773, "87": 5774, "\u0120enemies": 5775, "\u0120Games": 5776, "FF": 5777, "\u0120accident": 5778, "Well": 5779, "\u0120Martin": 5780, "TER": 5781, "\u0120ath": 5782, "\u0120Hell": 5783, "\u0120forg": 5784, "\u0120veter": 5785, "\u0120Medic": 5786, "free": 5787, "\u0120stars": 5788, "\u0120expensive": 5789, "\u0120acad": 5790, "rawn": 5791, "\u0120Whe": 5792, "\u0120lock": 5793, "\u0120format": 5794, "\u0120soldiers": 5795, "sm": 5796, "\u0120agent": 5797, "\u0120responsibility": 5798, "ora": 5799, "\u0120Science": 5800, "\u0120rapid": 5801, "\u0120tough": 5802, "\u0120Jesus": 5803, "\u0120believes": 5804, "ML": 5805, "\u0120wear": 5806, "lete": 5807, "\u00c3\u0125\u00c3\u0124": 5808, "\u0120Dri": 5809, "\u0120commission": 5810, "\u0120Bob": 5811, "Oh": 5812, "aped": 5813, "\u0120warm": 5814, "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 5815, "\u01202003": 5816, "ortion": 5817, "\u0120hasn": 5818, "uster": 5819, "\u0120univers": 5820, "\u0120Ill": 5821, "\u0120king": 5822, "ologies": 5823, "94": 5824, "\u0120Tem": 5825, "\u0120Mos": 5826, "\u0120patient": 5827, "\u0120Mexico": 5828, "cean": 5829, "\u0120Death": 5830, "\u0120Sanders": 5831, "you": 5832, "\u0120Cast": 5833, "\u0120Company": 5834, "pty": 5835, "\u0120happening": 5836, "FP": 5837, "\u0120Battle": 5838, "\u0120bought": 5839, "Am": 5840, "Mod": 5841, "Us": 5842, "uters": 5843, "\u0120Cre": 5844, "\u0120Those": 5845, "\u012044": 5846, "iser": 5847, "\u0120soul": 5848, "\u0120Top": 5849, "\u0120Harry": 5850, "\u0120Aw": 5851, "\u0120seat": 5852, "ffee": 5853, "\u0120revolution": 5854, "\u0120(\"": 5855, "\u0120During": 5856, "ette": 5857, "\u0120ring": 5858, "\u0120offensive": 5859, "\u0120returns": 5860, "\u0120videos": 5861, "\u0120discl": 5862, "\u0120famous": 5863, "enced": 5864, "\u0120Sign": 5865, "\u0120River": 5866, "\u0120300": 5867, "PM": 5868, "\u0120Bus": 5869, "\u0120CH": 5870, "\u0120candidates": 5871, "arden": 5872, "\u0120percentage": 5873, "\u0120visual": 5874, "\u0120thank": 5875, "\u0120trouble": 5876, "nergy": 5877, "\u01202001": 5878, "\u0120prove": 5879, "ashion": 5880, "\u0120enh": 5881, "\u0120Long": 5882, "UM": 5883, "\u0120connected": 5884, "\u0120possibility": 5885, "Over": 5886, "\u0120expert": 5887, "\u0120library": 5888, "arts": 5889, "\u0120Director": 5890, "\u0120fellow": 5891, "92": 5892, "irty": 5893, "\u0120dry": 5894, "\u0120signs": 5895, "\u0120Love": 5896, "\u0120quiet": 5897, "foot": 5898, "\u0120pure": 5899, "\u0120Hun": 5900, "\u0120filled": 5901, "phas": 5902, "\u0120Elect": 5903, "endment": 5904, "\u0120Expl": 5905, "\u0120unable": 5906, "ns": 5907, "mo": 5908, "\u0120vast": 5909, "obe": 5910, "\u0120identify": 5911, "apping": 5912, "\u0120Carolina": 5913, "gress": 5914, "\u0120prote": 5915, "\u0120fish": 5916, "\u0120circumstances": 5917, "razy": 5918, "\u0120Phot": 5919, "\u0120bodies": 5920, "\u0120Mur": 5921, "\u0120developing": 5922, "\u0120AR": 5923, "\u0120experienced": 5924, "\u0120substant": 5925, "\u0120Board": 5926, "esome": 5927, "\u0120domestic": 5928, "\u0120combined": 5929, "\u0120Put": 5930, "\u0120chemical": 5931, "\u0120Child": 5932, "\u0120pool": 5933, "\u0120Cy": 5934, "\u0120egg": 5935, "cons": 5936, "sters": 5937, "\u0120hurt": 5938, "\u0120markets": 5939, "\u0120conservative": 5940, "\u0120supporters": 5941, "\u0120agencies": 5942, "idel": 5943, "Ob": 5944, "urb": 5945, "\u012043": 5946, "\u0120Defense": 5947, "ye": 5948, "\u0120Ap": 5949, "dule": 5950, "\u0120temperature": 5951, "\u0120conducted": 5952, "\u0120Chief": 5953, "\u0120pulled": 5954, "\u0120fol": 5955, "Last": 5956, "onto": 5957, "osis": 5958, "VER": 5959, "Des": 5960, "\u0120Pan": 5961, "First": 5962, "\u0120advance": 5963, "\u0120license": 5964, "rors": 5965, "\u0120Jon": 5966, "\u0120imagine": 5967, "\u0120hell": 5968, "\u0120fixed": 5969, "\u0120incor": 5970, "osite": 5971, "\u0120Log": 5972, "icken": 5973, "]:": 5974, "\u0120surprise": 5975, "hab": 5976, "\u0120craft": 5977, "olt": 5978, "\u0120Jul": 5979, "\u0120dial": 5980, "\u0120relevant": 5981, "\u0120entered": 5982, "\u0120leads": 5983, "\u0120AD": 5984, "\u0120Clean": 5985, "\u0120pictures": 5986, "essor": 5987, "\u0120alt": 5988, "\u0120paying": 5989, "Per": 5990, "\u0120Market": 5991, "\u0120updates": 5992, "amily": 5993, "\u0120Type": 5994, "\u0120Home": 5995, "\u012055": 5996, "sembly": 5997, "rome": 5998, "83": 5999, "\u0120greatest": 6000, "\u0120height": 6001, "\u0120heav": 6002, "aints": 6003, "\u0120listen": 6004, "aser": 6005, "\u0120SH": 6006, "\u0120capable": 6007, "acle": 6008, "\u0120perspect": 6009, "inating": 6010, "\u0120offering": 6011, "rypt": 6012, "\u0120Develop": 6013, "abin": 6014, "rc": 6015, "\u0120bright": 6016, "alty": 6017, "arrow": 6018, "\u0120suppl": 6019, "inding": 6020, "acked": 6021, "gypt": 6022, "\u0120Another": 6023, "pg": 6024, "\u0120Virginia": 6025, "\u0120Lu": 6026, "\u0120planned": 6027, "\u0120pit": 6028, "\u0120sweet": 6029, "Type": 6030, "\u0120Di": 6031, "\u0120typically": 6032, "\u0120Francisco": 6033, "\u0120prospect": 6034, "\u0120Dan": 6035, "\u0120teen": 6036, "rees": 6037, "\u0120sched": 6038, "\u0120hol": 6039, "\u0120scr": 6040, "\u0120lots": 6041, "life": 6042, "\u0120newsp": 6043, "\u0120forget": 6044, "\u0120None": 6045, "\u0120Middle": 6046, "\u0120Ryan": 6047, "edd": 6048, "\u0120severe": 6049, "\u0120suit": 6050, "ller": 6051, "93": 6052, "\u0120correspond": 6053, "\u0120explos": 6054, "uations": 6055, "\u0120flag": 6056, "game": 6057, "rid": 6058, "\u0120prin": 6059, "\u0120Data": 6060, "\u0120deploy": 6061, "\u0120Enter": 6062, "suit": 6063, "ghan": 6064, "\u0120Men": 6065, "\u0120thoughts": 6066, "\u0120matters": 6067, "\u0120adapt": 6068, "\u0120Ari": 6069, "\u0120fill": 6070, "\u0120forth": 6071, "\u0120sam": 6072, "\u012041": 6073, "\u0120payment": 6074, "\u0120Hor": 6075, "\u0120spring": 6076, "duc": 6077, "\u0120losing": 6078, "\u0120bringing": 6079, "FO": 6080, "ala": 6081, "\u0120distribution": 6082, "hered": 6083, "bour": 6084, "\u0120Israeli": 6085, "oma": 6086, "\u0120combination": 6087, "\u0120plenty": 6088, "VE": 6089, "Can": 6090, "\u0120Haw": 6091, "\u0120perman": 6092, "\u0120Special": 6093, "\u0120tow": 6094, "\u0120seeking": 6095, "\u0120examples": 6096, "\u0120classes": 6097, "cr": 6098, "\u0120beer": 6099, "\u0120moves": 6100, "\u0120IP": 6101, "\u0120Kn": 6102, "\u0120panel": 6103, "Even": 6104, "\u0120properly": 6105, "\u0120ris": 6106, "\u0120plug": 6107, "\u0120estimated": 6108, "Every": 6109, "\u0120defensive": 6110, "agraph": 6111, "\u0120pregn": 6112, "\u0120instit": 6113, "\u0120Vict": 6114, "\u0120volume": 6115, "\u0120positions": 6116, "\u0120links": 6117, "\u0120Program": 6118, "\u0120Week": 6119, "agues": 6120, "\u0120transform": 6121, "ker": 6122, "\u0120CEO": 6123, "\u0120cas": 6124, "\u0120opponent": 6125, "\u0120tweet": 6126, "\u0120Code": 6127, "\u0120shop": 6128, "\u0120fly": 6129, "\u0120talks": 6130, "\u0120bag": 6131, "Phone": 6132, "\u0120aid": 6133, "\u0120plants": 6134, "\u012065": 6135, "\u0120attorney": 6136, "arters": 6137, "quest": 6138, "\u0120Magic": 6139, "\u0120begins": 6140, "\u0120myster": 6141, "\u0120environmental": 6142, "\u0120storage": 6143, "NN": 6144, "\u0120marg": 6145, "\u0120ske": 6146, "\u0120metal": 6147, "elly": 6148, "\u0120ordered": 6149, "\u0120remained": 6150, "\u0120loved": 6151, "\u0120prompt": 6152, "\u0120updated": 6153, "\u0120experts": 6154, "\u0120walking": 6155, "\u0120ancient": 6156, "\u0120performed": 6157, "ATE": 6158, "\u0120neither": 6159, "iency": 6160, "\u0120manufacture": 6161, "\u0120Pak": 6162, "\u0120selected": 6163, "\u0120mine": 6164, "\u0120ultimately": 6165, "\u0120explan": 6166, "\u0120label": 6167, "\u0120Services": 6168, "ributed": 6169, "Trump": 6170, "\u0120syn": 6171, "\u0120Ult": 6172, "SC": 6173, "\u0120meat": 6174, "\u0120giant": 6175, "\u0120Wars": 6176, "\u0120ON": 6177, "\u0120adm": 6178, "\u0120interpret": 6179, "\u0120evening": 6180, "\u0120evil": 6181, "\u0120Boston": 6182, "\u0120Wild": 6183, "\u0120\u00c3": 6184, "\u0120Bitcoin": 6185, "\u0120Amazon": 6186, "Dr": 6187, "\u0120Information": 6188, "\u0120obviously": 6189, "\u0120advanced": 6190, "Photo": 6191, "olar": 6192, "\u0120weather": 6193, "\u0120symbol": 6194, "\u0120sole": 6195, "\u0120potentially": 6196, "oster": 6197, "\u0120originally": 6198, "mun": 6199, "300": 6200, "aze": 6201, "essions": 6202, "\u0120deck": 6203, "\u0120stood": 6204, "\u0120youth": 6205, "\u0120Bern": 6206, "Rep": 6207, "\u0120Test": 6208, "\u0120basically": 6209, "otic": 6210, "\u0120involve": 6211, "olit": 6212, "lyn": 6213, "See": 6214, "\u0120aircraft": 6215, "\u0120confirm": 6216, "EW": 6217, "\u0120messages": 6218, "\u0120Richard": 6219, "\u0120kit": 6220, "\u0120prohib": 6221, "\u0120vulner": 6222, "isters": 6223, "\u0120existence": 6224, "\u0120turning": 6225, "\u0120SP": 6226, "\u0120desire": 6227, "\u0120flat": 6228, "\u0120ment": 6229, "season": 6230, "anges": 6231, "\u0120neighborhood": 6232, "\u0120Lake": 6233, "ATION": 6234, "\u0120pointed": 6235, "bur": 6236, "\u0120innov": 6237, "ucks": 6238, "UL": 6239, "\u0120professor": 6240, "\u0120expressed": 6241, "AB": 6242, "icious": 6243, "\u01202002": 6244, "\u0120Dev": 6245, "\u0120session": 6246, "\u0120bare": 6247, "sen": 6248, "\u0120diss": 6249, "\u0120Cath": 6250, "\u0120Pass": 6251, "\u0120Point": 6252, "\u0120doctor": 6253, "orrow": 6254, "ailed": 6255, "\u0120Rub": 6256, "\u0120DC": 6257, "\u0120Charl": 6258, "person": 6259, "\u0120writer": 6260, "ighters": 6261, "ureau": 6262, "\u0120oblig": 6263, "\u0120recorded": 6264, "\u0120broke": 6265, "\u0120orders": 6266, "ilty": 6267, "\u0120motion": 6268, "inity": 6269, "law": 6270, "adium": 6271, "\u0120immigration": 6272, "\u0120contrast": 6273, "\u0120batt": 6274, "\u0120excellent": 6275, "\u0120technical": 6276, "ami": 6277, "\u0120tun": 6278, "\u0120cloud": 6279, "\u0120Year": 6280, "geon": 6281, "\u0120creation": 6282, "\u0120strange": 6283, "\u0120auth": 6284, "\u0120fort": 6285, "born": 6286, "\u0120extent": 6287, "\u0120Today": 6288, "\u0120Club": 6289, "\u0120rain": 6290, "\u0120sample": 6291, "\u0120accepted": 6292, "\u0120tact": 6293, "\u0120fired": 6294, "\u0120Son": 6295, "\u0120stands": 6296, "\u0120boot": 6297, "\u012047": 6298, "\u0120statements": 6299, "\u0120versions": 6300, "\u0120selling": 6301, "ounded": 6302, "\u01201990": 6303, "\u0120weren": 6304, "\u0120Watch": 6305, "\u0120experiment": 6306, "Post": 6307, "\u0120retail": 6308, "uled": 6309, "Inst": 6310, "unte": 6311, "\u00e3\u0125\u00bc": 6312, "\u0120depart": 6313, "\u0120bond": 6314, "ivery": 6315, "ompl": 6316, "\u0120reaction": 6317, "\u0120Syrian": 6318, "\u0120Pac": 6319, "apped": 6320, "aniel": 6321, "DP": 6322, "\u0120resolution": 6323, "\u0120react": 6324, "\u0120approved": 6325, "onom": 6326, "mond": 6327, "\u0120Offic": 6328, "---": 6329, "\u0120replace": 6330, "\u0120tack": 6331, "\u0120sport": 6332, "\u0120chain": 6333, "\u0120emergency": 6334, "rad": 6335, "\u0120Palestin": 6336, "\u012046": 6337, "\u0120automatically": 6338, "\u0120route": 6339, "\u0120pal": 6340, "\u0120banks": 6341, "\u0120Paris": 6342, "\u0120Media": 6343, "road": 6344, "icing": 6345, "ixt": 6346, "isted": 6347, "\u0120grew": 6348, "\u0120coord": 6349, "\u0120Where": 6350, "omin": 6351, "\u0120subs": 6352, "\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 6353, "\u0120\u00c2\u00b1": 6354, "\u0120corporate": 6355, "\u0120selection": 6356, "noon": 6357, "\u0120Report": 6358, "cs": 6359, "cluding": 6360, "orders": 6361, "anche": 6362, "\u0120Its": 6363, "\u0120slowly": 6364, "\u0120Egypt": 6365, "\u0120Acc": 6366, "\u0120colle": 6367, "iques": 6368, "EX": 6369, "\u0120attempts": 6370, "url": 6371, "\u0120Cross": 6372, "\u0120findings": 6373, "\u0120SC": 6374, "\u0120OR": 6375, "\u0120index": 6376, "ensity": 6377, "\u0120Way": 6378, "\u0120Land": 6379, "\u0120shock": 6380, "dis": 6381, "\u0120dynam": 6382, "\u0120cart": 6383, "mosp": 6384, "Since": 6385, "iest": 6386, "\u0120Boy": 6387, "\u0120storm": 6388, "\u0120Contin": 6389, "2013": 6390, "hew": 6391, "ilit": 6392, "\u0120essential": 6393, "iquid": 6394, "Other": 6395, "ivered": 6396, "\u0120reasonable": 6397, "Act": 6398, "\u0120subsequ": 6399, "\u0120Pack": 6400, "\u0120Fort": 6401, "\u0120considering": 6402, "\u0120university": 6403, "log": 6404, "\u0120married": 6405, "\u0120illust": 6406, "\u0120True": 6407, "\u00a3\u0131": 6408, "\u0120numerous": 6409, "rastructure": 6410, "\u0120seriously": 6411, "\u0120referred": 6412, "ua": 6413, "\u0120consistent": 6414, "onna": 6415, "\u0120Real": 6416, "ruption": 6417, "ciples": 6418, "\u0120facts": 6419, "91": 6420, "otes": 6421, "erg": 6422, "Then": 6423, "\u0120accompl": 6424, "Note": 6425, "\u0120revenue": 6426, "\u0120passing": 6427, "\u0120mal": 6428, "een": 6429, "\u0120Yet": 6430, "\u0120gather": 6431, "terday": 6432, "ework": 6433, "\u0120Author": 6434, "Pe": 6435, "\u0120optim": 6436, "\u0120rub": 6437, "\u0120\u00e8\u00a3\u0131": 6438, "\u0120unknown": 6439, "stone": 6440, "\u0120union": 6441, "olve": 6442, "\u0120opportunities": 6443, "\u0120browser": 6444, "\u0120Wal": 6445, "\u0120Cost": 6446, "\u0120reporting": 6447, "sts": 6448, "pet": 6449, "\u0120sand": 6450, "\u0120suddenly": 6451, "\u0120surprising": 6452, "\u0120VR": 6453, "\u0120somewhat": 6454, "\u0120Bas": 6455, "ulture": 6456, "izz": 6457, "\u0120CD": 6458, "\u0120challenges": 6459, "\u0120settings": 6460, "\u0120experiences": 6461, "\u0120Full": 6462, "\u0120cann": 6463, "\u0120receiving": 6464, "EST": 6465, "\u0120joint": 6466, "\u0120cultural": 6467, "\u0120ast": 6468, "82": 6469, "astern": 6470, "ceived": 6471, "\u0120Cru": 6472, "\u0120bull": 6473, "pired": 6474, "amm": 6475, "\u0120facing": 6476, "power": 6477, "\u0120boss": 6478, "\u0120Hol": 6479, "\u0120instr": 6480, "\u0120increasingly": 6481, "\u0120shift": 6482, "\u0120streets": 6483, "\u0120Williams": 6484, "abb": 6485, "\u0120lie": 6486, "\u0120laugh": 6487, "\u0120Ca": 6488, "PL": 6489, "\u0120adults": 6490, "\u0120customer": 6491, "\u0120obtained": 6492, "\u0120supporting": 6493, "html": 6494, "fire": 6495, "\u0120detailed": 6496, "\u0120picked": 6497, "\u0120Right": 6498, "lder": 6499, "EE": 6500, "stood": 6501, "\u0120Kim": 6502, "\u0120wire": 6503, "\u0120sight": 6504, "\u0120developers": 6505, "\u0120persons": 6506, "\u0120sad": 6507, "\u0120cup": 6508, "\u0120warning": 6509, "\u0120boys": 6510, "long": 6511, "\u0120bird": 6512, "fo": 6513, "\u0120wal": 6514, "\u0120observed": 6515, "\u0120zone": 6516, "iveness": 6517, "\u0120channel": 6518, "cript": 6519, "\u0120refused": 6520, "\u0120Again": 6521, "\u0120suc": 6522, "\u0120spokesman": 6523, "\u0120Ref": 6524, "rite": 6525, "ouston": 6526, "\u00e3\u0125\u00b3": 6527, "\u0120Sher": 6528, "\u0120acts": 6529, "\u0120Name": 6530, "\u0120struggle": 6531, "arry": 6532, "ometimes": 6533, "\u0120discrim": 6534, "HT": 6535, "\u0120category": 6536, "\u0120realize": 6537, "\u0120employee": 6538, "\u0120Afghan": 6539, "enger": 6540, "\u0120guns": 6541, "\u0120Steve": 6542, "\u0120Mot": 6543, "\u0120Ol": 6544, "oked": 6545, "\u0120thick": 6546, "\u0120fairly": 6547, "illy": 6548, "\u0120surve": 6549, "\u0120Mat": 6550, "weight": 6551, "\u00e2\u0136": 6552, "\u0120troops": 6553, "\u0120agents": 6554, "\u0120battery": 6555, "\u0120motiv": 6556, "\u00c3\u00a1": 6557, "Sec": 6558, "den": 6559, "overy": 6560, "LS": 6561, "\u0120flu": 6562, "\u0120confident": 6563, "\u0120Oper": 6564, "\u0120empty": 6565, "\u0120phen": 6566, "\u0120sector": 6567, "\u0120excited": 6568, "\u0120remote": 6569, "aph": 6570, "oen": 6571, "\u0120destroyed": 6572, "\u0120moral": 6573, "\u0120HP": 6574, "\u0120Ron": 6575, "\u0120dress": 6576, "\u0120Bat": 6577, "\u0120lit": 6578, "\u0120MS": 6579, "\u0120af": 6580, "HL": 6581, "rum": 6582, "isms": 6583, "\u0120shouldn": 6584, "\u0120sympt": 6585, "\u0120Toronto": 6586, "hetic": 6587, "\u0120carbon": 6588, "\u0120installed": 6589, "\u0120violent": 6590, "\u0120solar": 6591, "ja": 6592, "\u0120practices": 6593, "\u0120ride": 6594, "\u0120Penn": 6595, "\u0120improved": 6596, "\u0120audio": 6597, "\u0120behavi": 6598, "\u0120PS": 6599, "\u0120eating": 6600, "Data": 6601, "\u0120Review": 6602, "pass": 6603, "claim": 6604, "uated": 6605, "angers": 6606, "chen": 6607, "\u0120properties": 6608, "\u0120anywhere": 6609, "Another": 6610, "\u0120blow": 6611, "\u0120Jackson": 6612, "\u0120proud": 6613, "\u0120plane": 6614, "lines": 6615, "\u0120square": 6616, "\u0120proof": 6617, "ansas": 6618, "\u0120talked": 6619, "makers": 6620, "\u0120sister": 6621, "\u0120holds": 6622, "\u0120resident": 6623, "\u0120==": 6624, "\u0120resistance": 6625, "\u0120split": 6626, "\u0120prosecut": 6627, "\u0120confidence": 6628, "resents": 6629, "\u0120cuts": 6630, "\u0120exception": 6631, "\u0120zero": 6632, "Getty": 6633, "\u0120copyright": 6634, "\u0120totally": 6635, "ormal": 6636, "ifications": 6637, "\u0120Australian": 6638, "\u0120sick": 6639, "\u0120150": 6640, "\u0120household": 6641, "\u0120fees": 6642, "\u0120drivers": 6643, "ogen": 6644, "\u0120NY": 6645, "\u0120necessarily": 6646, "\u0120regulations": 6647, "earing": 6648, "sl": 6649, "\u0120perspective": 6650, "care": 6651, "icial": 6652, "His": 6653, "\u0120escape": 6654, "\u0120surprised": 6655, "\u0120Van": 6656, "urrent": 6657, "\u0120vac": 6658, "81": 6659, "\u0120Thus": 6660, "\u0120emphas": 6661, "\u0120Champions": 6662, "\u0120Ice": 6663, "\u0120narr": 6664, "\u0120heads": 6665, "\u0120causing": 6666, "bel": 6667, "fortunately": 6668, "\u0120Ma": 6669, "\u0120targets": 6670, "cipl": 6671, "\u0120afternoon": 6672, "\u0120adds": 6673, "\u0120Maybe": 6674, "\u0120Four": 6675, "essed": 6676, "plete": 6677, "\u0120usual": 6678, "cho": 6679, "ingu": 6680, "\u0120withd": 6681, "\u0120Energy": 6682, "\u0120Econom": 6683, "OO": 6684, "\u0120articles": 6685, "\u0120injured": 6686, "\u0120manage": 6687, "\u0120explains": 6688, "\u0120diagn": 6689, "Rec": 6690, "atures": 6691, "\u0120linked": 6692, "\u0120discussed": 6693, "\u0120explo": 6694, "\u0120occasion": 6695, "athan": 6696, "\u0120opposite": 6697, "\u0120faces": 6698, "\u0120denied": 6699, "\u0120Knight": 6700, "\u0120nut": 6701, "\u0120approximately": 6702, "\u0120disappoint": 6703, "onymous": 6704, "\u0120Best": 6705, "\u0120Lo": 6706, "\u0120Hy": 6707, "\u0120Aff": 6708, "\u0120voting": 6709, "anwhile": 6710, "\u0120III": 6711, "\u0120institutions": 6712, "agram": 6713, "\u0120Daily": 6714, "\u0120drag": 6715, "\u0120nearby": 6716, "\u0120guilty": 6717, "\u0120conver": 6718, "Pre": 6719, "ship": 6720, "\u0120reward": 6721, "\u0120philosoph": 6722, "\u0120SS": 6723, "ugh": 6724, "\u0120apps": 6725, "friend": 6726, "\u0120upper": 6727, "\u0120advert": 6728, "\u0120snow": 6729, "\u0120frust": 6730, "\u0120ourselves": 6731, "Fr": 6732, "\u0120Die": 6733, "ampion": 6734, "\u0120dismiss": 6735, "\u0120cere": 6736, "\u0120signal": 6737, "from": 6738, "\u0120).": 6739, "\u012052": 6740, "\u0120crimes": 6741, "itors": 6742, "estival": 6743, "useum": 6744, "\u0120council": 6745, "\u0120Saud": 6746, "May": 6747, "\u0120Gun": 6748, "ician": 6749, "ether": 6750, "\u0120sufficient": 6751, "\u0120Hen": 6752, "sole": 6753, "\u0120historical": 6754, "\u0120Far": 6755, "\u0120Turn": 6756, "\u0120pin": 6757, "\u0120succeed": 6758, "mat": 6759, "lymp": 6760, "\u0120tradition": 6761, "\u0120Ok": 6762, "\u0120cro": 6763, "\u0120description": 6764, "alle": 6765, "\u0120sky": 6766, "Te": 6767, "\u0120widely": 6768, "\u0120wave": 6769, "\u0120definition": 6770, "\u0120Jews": 6771, "\u0120cycle": 6772, "\u0120refere": 6773, "\u0120brings": 6774, "usal": 6775, "\u0120alive": 6776, "\u0120frequently": 6777, "\u0120intention": 6778, "\u0120Control": 6779, "lv": 6780, "ystem": 6781, "\u0120privacy": 6782, "gent": 6783, "rence": 6784, "\u0120Quest": 6785, "\u0120Christmas": 6786, "\u0120rail": 6787, "\u0120cooper": 6788, "\u0120tested": 6789, "\u0120Capt": 6790, "asks": 6791, "\u0120comfortable": 6792, "\u0120delivered": 6793, "scape": 6794, "\u0120depth": 6795, "\u0120GOP": 6796, "\u0120writes": 6797, "\u0120assets": 6798, "\u0120sav": 6799, "iments": 6800, "\u0120transition": 6801, "\u0120artist": 6802, "\u0120Look": 6803, "\u0120lob": 6804, "\u0120components": 6805, "arity": 6806, "\u0120walked": 6807, "\u0120root": 6808, "\u0120participants": 6809, "\u0120noticed": 6810, "\u0120resc": 6811, "\u0120nav": 6812, "\u0120Administ": 6813, "da": 6814, "utral": 6815, "plate": 6816, "\u0120importance": 6817, "\u0120assert": 6818, "iously": 6819, "cription": 6820, "\u0120injuries": 6821, "\u0120Check": 6822, "\u0120registered": 6823, "\u0120intent": 6824, "\u0120missed": 6825, "ographic": 6826, "\u0120sentence": 6827, "ounter": 6828, "\u0120assistance": 6829, "evin": 6830, "\u0120database": 6831, "\u0120buildings": 6832, "\u0120classic": 6833, "\u0120thinks": 6834, "\u0120Ohio": 6835, "Pr": 6836, "ugg": 6837, "\u0120fee": 6838, "pan": 6839, "\u0120effectively": 6840, "\u0120facility": 6841, "\u0120bear": 6842, "\u0120chapter": 6843, "\u0120dogs": 6844, "\u0120Columb": 6845, "\u0120latter": 6846, "itial": 6847, "\u0120admitted": 6848, "TV": 6849, "\u0120Georg": 6850, "\u0120posts": 6851, "\\\\": 6852, "\u0120lawyer": 6853, "\u0120equival": 6854, "\u0120mand": 6855, "\u0120controlled": 6856, "\u0120Walk": 6857, "\u0120Andrew": 6858, "\u0120menu": 6859, "amental": 6860, "\u0120protected": 6861, "va": 6862, "\u0120administr": 6863, "oral": 6864, "\u0120rein": 6865, "\u0120Sar": 6866, "\u0120amounts": 6867, "\u0120native": 6868, "\u0120Moon": 6869, "\u0120represents": 6870, "\u0120abandon": 6871, "\u0120carrying": 6872, "\u0120tank": 6873, "mary": 6874, "\u0120declared": 6875, "Tube": 6876, "\u0120hat": 6877, "\u0120punish": 6878, "ellect": 6879, "mes": 6880, "\u0120universe": 6881, "\u0120Rod": 6882, "phy": 6883, "\u0120infrastructure": 6884, "\u012051": 6885, "\u0120opposed": 6886, "ownt": 6887, "ca": 6888, "\u0120Make": 6889, "\u0120hardware": 6890, "\u0120coffee": 6891, "Rel": 6892, "bal": 6893, "world": 6894, "\u0120Saf": 6895, "\u0120Sea": 6896, "inals": 6897, "\u0120owned": 6898, "\u0120hall": 6899, "ersion": 6900, "\u0120describe": 6901, "\u0120Pot": 6902, "\u0120portion": 6903, "\u0120atmosp": 6904, "\u0120governments": 6905, "\u0120depending": 6906, "\u0120offense": 6907, "\u0120trick": 6908, "awa": 6909, "\u0120Line": 6910, "\u0120Vis": 6911, "\u0120Hard": 6912, "\u0120Orig": 6913, "\u0120Click": 6914, "\u0120desk": 6915, "\u0120Valley": 6916, "\u0120Sov": 6917, "\u0120movies": 6918, "\u0120remark": 6919, "\u0120mail": 6920, "\u0120conscious": 6921, "\u0120ruling": 6922, "\u0120Rights": 6923, "\u0120medic": 6924, "hent": 6925, "\u0120Women": 6926, "><": 6927, "\u0120replaced": 6928, "\u0120Prem": 6929, "\u0120Thanks": 6930, "\u0120renew": 6931, "\u0120Ball": 6932, "iform": 6933, "\u0120shots": 6934, "Comm": 6935, "\u0120armed": 6936, "\u0120constant": 6937, "\u0120taste": 6938, "\u0120realized": 6939, "\u0120buff": 6940, "\u0120mo": 6941, "\u0120efficient": 6942, "Most": 6943, "oration": 6944, "ifies": 6945, "\u0120communication": 6946, "\u0120flood": 6947, "\u0120consequences": 6948, "\u0120anyway": 6949, "igg": 6950, "\u0120GM": 6951, "\u0120Thank": 6952, "\u0120iron": 6953, "\u0120evolution": 6954, "\u0120Cop": 6955, "twitter": 6956, "\u012095": 6957, "\u0120relationships": 6958, "adel": 6959, "\u0120Young": 6960, "\u0120proposal": 6961, "ayers": 6962, "uilding": 6963, "\u0120Hot": 6964, "ORE": 6965, "cos": 6966, "\u0120collabor": 6967, "PG": 6968, "axy": 6969, "\u0120knowing": 6970, "\u0120supports": 6971, "owed": 6972, "\u0120controls": 6973, "\u0120merely": 6974, "umer": 6975, "\u0120athlet": 6976, "\u0120fashion": 6977, "path": 6978, "\u0120gift": 6979, "\u0120era": 6980, "AND": 6981, "\u0120kinds": 6982, "\u0120Korean": 6983, "\u0120legit": 6984, "ulous": 6985, "\u0120essentially": 6986, "\u0120therap": 6987, "nic": 6988, "\u0120suffered": 6989, "\u0120hur": 6990, "\u0120promise": 6991, "\u0120excess": 6992, "\u0120overw": 6993, "\u0120prime": 6994, "\u0120Houston": 6995, "erry": 6996, "\u0120Ms": 6997, "RS": 6998, "2012": 6999, "\u0120stores": 7000, "\u0120Olymp": 7001, "\u0120journey": 7002, "Although": 7003, "Sub": 7004, "\u0120Educ": 7005, "\u0120Chapter": 7006, "\u0120requests": 7007, "\u0120consumers": 7008, "\u0120tiny": 7009, "\u0120isol": 7010, "\u0120Fair": 7011, "ba": 7012, "\u0120YOU": 7013, "\u0120crash": 7014, "celer": 7015, "\u0120emotional": 7016, "\u0120goods": 7017, "\u0120elected": 7018, "\u0120moder": 7019, "\u0120Linux": 7020, "\u0120blocks": 7021, "\u0120island": 7022, "\u0120Society": 7023, "\u0120elections": 7024, "\u0120broadcast": 7025, "\u0120cheap": 7026, "\u0120nations": 7027, "\u0120seasons": 7028, "400": 7029, "\u0120waste": 7030, "\u0120Sat": 7031, "\u0120fields": 7032, "employ": 7033, "\u0120profile": 7034, "\u0120authors": 7035, "ALL": 7036, "\u0120Gra": 7037, "west": 7038, "\u0120Ty": 7039, "\u0120deaths": 7040, "\u0120vacc": 7041, "\u0120formed": 7042, "\u0120du": 7043, "\u0120ongoing": 7044, "\u0120Muslims": 7045, "elf": 7046, "igure": 7047, "\u0120assume": 7048, "\u0120Ukraine": 7049, "water": 7050, "\u0120coast": 7051, "\u0120voted": 7052, "gor": 7053, "\u0120AS": 7054, "\u0120Michigan": 7055, "aza": 7056, "\u0120Arm": 7057, "iro": 7058, "\u0120flex": 7059, "asters": 7060, "''": 7061, "\u0120welcome": 7062, "arl": 7063, "\u0120locations": 7064, "igation": 7065, "\u0120Fil": 7066, "\u0120buying": 7067, "\u0120architect": 7068, "\u0120harder": 7069, "\u0120Cub": 7070, "\u0120interface": 7071, "\u0120restaurant": 7072, "\u0120discover": 7073, "\u0120exceed": 7074, "\u0120favour": 7075, "gery": 7076, "\u0120duty": 7077, "\u0120pitch": 7078, "ador": 7079, "\u0120Mach": 7080, "boy": 7081, "\u0120responded": 7082, "\u0120extended": 7083, "hers": 7084, "Many": 7085, "raid": 7086, "ifer": 7087, "\u0120Ins": 7088, "Ser": 7089, "\u0120medium": 7090, "she": 7091, "\u0120Sports": 7092, "\u0120magazine": 7093, "utation": 7094, "\u0120limits": 7095, "\u0120Gall": 7096, "\u0120external": 7097, "razil": 7098, "\u0120younger": 7099, "tle": 7100, "\u0120remind": 7101, "\u0120CON": 7102, "\u0120immediate": 7103, "\u0120hidden": 7104, "\u0120volunte": 7105, "\u0120simpl": 7106, "odcast": 7107, "\u0120phase": 7108, "dr": 7109, "\u0120plot": 7110, "\u0120exposure": 7111, "RI": 7112, "ograp": 7113, "vin": 7114, "anish": 7115, "\u0120Acad": 7116, "\u0120Engine": 7117, "\u0120expansion": 7118, "\u0120Pay": 7119, "Your": 7120, "\u0120pushed": 7121, "\u0120Ell": 7122, "\u0120Head": 7123, "\u0120marketing": 7124, "\u0120AC": 7125, "ket": 7126, "\u0120hits": 7127, "\u0120gro": 7128, "\u0120Age": 7129, "\u0120Scot": 7130, "][": 7131, "\u0120stim": 7132, "\u0120iPhone": 7133, "\u012a\u0134": 7134, "\u0120narrow": 7135, "\u0120Getty": 7136, "\u0120Turkey": 7137, "\u0120perfectly": 7138, "\u0120enable": 7139, "utch": 7140, "\u0120precise": 7141, "\u0120regime": 7142, "\u0120shif": 7143, "\u0120compens": 7144, "gun": 7145, "div": 7146, "\u0120chosen": 7147, "\u0120Ken": 7148, "Any": 7149, "\u0120trees": 7150, "\u0120recommended": 7151, "\u0120Ren": 7152, "uable": 7153, "\u0120HT": 7154, "Follow": 7155, "EG": 7156, "\u0120Hand": 7157, "\u0120Kenn": 7158, "\u0120arguments": 7159, "\u0120exists": 7160, "\u0120bike": 7161, "\u0120Conserv": 7162, "\u0120breaking": 7163, "\u0120Gar": 7164, "\u0120crazy": 7165, "\u0120virtual": 7166, "aylor": 7167, "ixel": 7168, "\u01201980": 7169, "\u0120permission": 7170, "\u0120Series": 7171, "\u0120consumer": 7172, "\u0120closely": 7173, "called": 7174, "\u012054": 7175, "\u0120hopes": 7176, "\u0120array": 7177, "\u0120Win": 7178, "\u0120Labour": 7179, "\u0120spons": 7180, "\u0120Ire": 7181, "\u0120pow": 7182, "\u0120readers": 7183, "\u0120employment": 7184, "\u0120creature": 7185, "\u0120resulting": 7186, "\u0120accurate": 7187, "\u0120moments": 7188, "\u0120argued": 7189, "\u0120ped": 7190, "During": 7191, "\u012053": 7192, "\u0120Tal": 7193, "\u0120sought": 7194, "\u0120suffering": 7195, "\u0120icon": 7196, "lee": 7197, "\u0120($": 7198, "alian": 7199, "\u00c2\u00b0": 7200, "\u0120pra": 7201, "\u0120bonus": 7202, "(\"": 7203, "ko": 7204, "\u0120acting": 7205, "DE": 7206, "fall": 7207, "\u0120comparison": 7208, "\u0120smooth": 7209, "\u0120NAS": 7210, "upp": 7211, "\u0120Joseph": 7212, "eping": 7213, "\u0120Take": 7214, "\u0120Mid": 7215, "\u0120sending": 7216, "fast": 7217, "\u0120Fall": 7218, "\u0120dealing": 7219, "user": 7220, "\u0120Organ": 7221, "Co": 7222, "\u0120attached": 7223, "\u0120sees": 7224, "%.": 7225, "\u0120typical": 7226, "ART": 7227, "\u0120finds": 7228, "\u0120Asia": 7229, "umin": 7230, "\u0120Core": 7231, "\u0120Ent": 7232, "inent": 7233, "uce": 7234, "\u0120Blood": 7235, "\u0120Never": 7236, "\u0120emails": 7237, "\u0120highlight": 7238, "\u0120confront": 7239, "atus": 7240, "uted": 7241, "\u0120unus": 7242, "\u0120topic": 7243, "\u0120Adam": 7244, "\u0120ble": 7245, "ati": 7246, "\u0120understood": 7247, "Set": 7248, "struct": 7249, "TP": 7250, "\u0120mob": 7251, "aa": 7252, "\u0120Start": 7253, "pected": 7254, "sell": 7255, "\u0120dedicated": 7256, "\u0120CA": 7257, "uan": 7258, "\u0120songs": 7259, "escription": 7260, "\u0120tech": 7261, "\u0120rape": 7262, "\u0120aside": 7263, "\u0120grant": 7264, "\u012056": 7265, "sub": 7266, "\u0120argue": 7267, "\u0120containing": 7268, "\u0120schedule": 7269, "\u0120liberal": 7270, "\u0120publicly": 7271, "\u0120heavily": 7272, "\u0120Ut": 7273, "iner": 7274, "\u0120Section": 7275, "\u0120Care": 7276, "weet": 7277, "ls": 7278, "Dis": 7279, "\u00e2\u0136\u0122": 7280, "\u0120Follow": 7281, "Back": 7282, "\u0120IT": 7283, "\u0120bes": 7284, "ji": 7285, "\u0120Hit": 7286, "ested": 7287, "\u0120everybody": 7288, "\u0120Swed": 7289, "\u0120femin": 7290, "\u0120facilities": 7291, "\u0120conven": 7292, "Comp": 7293, "\u0120OS": 7294, "core": 7295, "\u0120anx": 7296, "\u0120division": 7297, "\u0120Cam": 7298, "\u0120Stan": 7299, "mates": 7300, "\u0120explore": 7301, "plom": 7302, "\u0120shares": 7303, "pload": 7304, "anes": 7305, "\u0120ideal": 7306, "eters": 7307, "\u0120Base": 7308, "\u0120plastic": 7309, "\u0120distinct": 7310, "\u0120Network": 7311, "\u0120Seattle": 7312, "\u0120trading": 7313, "ensus": 7314, "intend": 7315, "\u0120exhib": 7316, "\u0120initially": 7317, "\u0120Food": 7318, "\u0120thousand": 7319, "\u0120Business": 7320, "acter": 7321, "\u0120paragraph": 7322, "\u0120roughly": 7323, "\u0120www": 7324, "\u0120creative": 7325, "\u0120Conf": 7326, "\u0120consumption": 7327, "\u0120films": 7328, "agan": 7329, "\u0120obtain": 7330, "\u0120tall": 7331, "\u0120tor": 7332, "\u0120acknowled": 7333, "\u0120grown": 7334, "alo": 7335, "KE": 7336, "\u0120400": 7337, "enders": 7338, "taining": 7339, "UG": 7340, "\u0120suicide": 7341, "\u0120watched": 7342, "\u0120List": 7343, "ali": 7344, "rehens": 7345, "\u0120surrounding": 7346, "\u0120pip": 7347, "\u0120flying": 7348, "\u0120Java": 7349, "ordan": 7350, "\u0120serving": 7351, "inations": 7352, "post": 7353, "\u0120sho": 7354, "Av": 7355, "\u0120jail": 7356, "zy": 7357, "\u01201999": 7358, "\u0120>": 9609, "orous": 9610, "\u0120firms": 9611, "screen": 9612, "una": 9613, "\u0120embarrass": 9614, "ulse": 9615, "\u0120letting": 9616, "\u0120threw": 9617, "iley": 9618, "\u0120channels": 9619, "lan": 9620, "\u0120Vegas": 9621, "\u0120sear": 9622, "\u0120fantastic": 9623, "arre": 9624, "uzzle": 9625, "\u0120Der": 9626, "Those": 9627, "\u0120swing": 9628, "\u0120sheet": 9629, "index": 9630, "cover": 9631, "ogan": 9632, "\u0120variables": 9633, "\u0120Tech": 9634, "\u0120spoken": 9635, "achel": 9636, "\u0120Da": 9637, "\u0120Mountain": 9638, "\u0120loaded": 9639, "\u0120footage": 9640, "version": 9641, "\u0120unl": 9642, "\u0120Phoenix": 9643, "\u0120throwing": 9644, "\u0120firing": 9645, "\u0120tracking": 9646, "\u0120width": 9647, "\u0120struggling": 9648, "rooms": 9649, "otion": 9650, "\u0120monthly": 9651, "\u0120Server": 9652, "\u0120eggs": 9653, "open": 9654, "MC": 9655, "\u01201993": 9656, "\u0120hired": 9657, "\u0120stayed": 9658, "\u0120Allen": 9659, "\u0120stro": 9660, "\u012098": 9661, "step": 9662, "\u0120Turkish": 9663, "\u0120fabric": 9664, "isting": 9665, "\u0120Dom": 9666, "\u0120dates": 9667, "\u0120pron": 9668, "\u0120basketball": 9669, "\u0120lucky": 9670, "\u0120Arabia": 9671, "\u0120assumed": 9672, "esty": 9673, "\u0120affairs": 9674, "\u0120glad": 9675, "\u0120Indeed": 9676, "\u0120FA": 9677, "\u0120Word": 9678, "\u0120joining": 9679, "ifice": 9680, "pread": 9681, "irts": 9682, "\u0120Select": 9683, "\u0120populations": 9684, "aware": 9685, "\u0120nose": 9686, "\u0120complaints": 9687, "start": 9688, "\u0120scoring": 9689, "Thanks": 9690, "\u0120mining": 9691, "\u0120visitors": 9692, "SH": 9693, "\u0120damaged": 9694, "\u0120characteristics": 9695, "\u0120Pent": 9696, "DC": 9697, "\u012083": 9698, "\u0120Six": 9699, "rates": 9700, "\u0120flags": 9701, "\u0120Brew": 9702, "dog": 9703, "Mark": 9704, "////": 9705, "\u0120execution": 9706, "\u0120joke": 9707, "phones": 9708, "\u0120testimony": 9709, "\u0120obst": 9710, "QL": 9711, "\u0120Cut": 9712, "\u0120studied": 9713, "\u0120Nintendo": 9714, "icket": 9715, "\u0120NBC": 9716, "\u0120lad": 9717, "\u0120Bra": 9718, "\u0120Moh": 9719, "\u0120kernel": 9720, "\u0120overwhelming": 9721, "\u0120aged": 9722, "\u0120applicable": 9723, "\u0120Cond": 9724, "\u0120roads": 9725, "\u0120Block": 9726, "made": 9727, "odge": 9728, "\u0120commands": 9729, "\u0120offices": 9730, "veland": 9731, "\u0120tut": 9732, "\u0120receiver": 9733, "\u0120Fro": 9734, "\u0120shopping": 9735, "\u0120iP": 9736, "\u0120Stre": 9737, "\u0120ABC": 9738, "\u0120entertainment": 9739, "\u0120Bow": 9740, "orted": 9741, "Mc": 9742, "\u0120reads": 9743, "grad": 9744, "\u0120Collect": 9745, "\u0120\u00e2\u012a\u0134": 9746, "\u0120Capital": 9747, "ederation": 9748, "\u0120employer": 9749, "\u0120involvement": 9750, "\u0120anxiety": 9751, "alia": 9752, "\u0120roof": 9753, "\u0120Among": 9754, "\u0120Democrat": 9755, "\u0120stats": 9756, "\u0120Vill": 9757, "\u0120constitutional": 9758, "\u0120referring": 9759, "itty": 9760, "\u0120tackle": 9761, "outube": 9762, "\u0120backed": 9763, "\u0120Hong": 9764, "\u0120Broad": 9765, "\u0120ele": 9766, "\u0120Ott": 9767, "\u01201992": 9768, "hour": 9769, "achusetts": 9770, "Cal": 9771, "\u0120defeated": 9772, "\u012081": 9773, "esp": 9774, "\u0120seemingly": 9775, "was": 9776, "\u0120Jenn": 9777, "\u0120Kurd": 9778, "\u0120gene": 9779, "\u0120discount": 9780, "Ret": 9781, "ECT": 9782, "();": 9783, "\u0120clubs": 9784, "\u0120sid": 9785, "\u0120Marsh": 9786, "Check": 9787, "\u0120pp": 9788, "\u0120Eag": 9789, "idespread": 9790, "\u0120beings": 9791, "FT": 9792, "\u0120introduction": 9793, "\u0120Change": 9794, "ARD": 9795, "\u0120110": 9796, "adows": 9797, "ierce": 9798, "\u0120meal": 9799, "author": 9800, "\u0120Bang": 9801, "lahoma": 9802, "\u0120ranks": 9803, "2011": 9804, "????": 9805, "max": 9806, "\u0120collapse": 9807, "\u0120opens": 9808, "\u0120echo": 9809, "\u0120soph": 9810, "\u0120racist": 9811, "\u0120enormous": 9812, "\u0120waves": 9813, "\u0120tap": 9814, "\u0120comprehensive": 9815, ".--": 9816, "\u0120Roy": 9817, "\u0120farmers": 9818, "Related": 9819, "aired": 9820, "rones": 9821, "\u0120Crim": 9822, "\u0120proportion": 9823, "\u0120designs": 9824, "\u0120negotiations": 9825, "\u0120virtually": 9826, "\u0120Batman": 9827, "\u0120warn": 9828, "\u0120legitimate": 9829, "mate": 9830, "\u0120convention": 9831, ",,": 9832, "netic": 9833, "\u0120SD": 9834, "\u0120consistently": 9835, "\u0120compensation": 9836, "\u0120punishment": 9837, "\u0120ye": 9838, "\u0120tie": 9839, "\u0120Bureau": 9840, "irlf": 9841, "\u0120Bu": 9842, "\u0120Aren": 9843, "\u0120Philipp": 9844, "\u0120knife": 9845, "\u0120memories": 9846, "\u0120Ross": 9847, "\u0120angle": 9848, "\u012086": 9849, "\u0120Thunder": 9850, "\u0120rend": 9851, "\u0120Tour": 9852, "\u0120counts": 9853, "sung": 9854, "\u0120Imp": 9855, "\u0120educational": 9856, "\u0120accessible": 9857, "COM": 9858, "\u0120drew": 9859, "yer": 9860, "Gl": 9861, "amine": 9862, "ORT": 9863, "OB": 9864, "IB": 9865, "master": 9866, "\u0120trials": 9867, "ogy": 9868, "har": 9869, "\u0120Trust": 9870, "\u0120preferred": 9871, "irlfriend": 9872, "\u0120Nev": 9873, "\u0120bin": 9874, "\u0120cow": 9875, "Page": 9876, "\u0120signature": 9877, "\u0120BL": 9878, "700": 9879, "\u0120retired": 9880, "\u0120bytes": 9881, "\u0120neighb": 9882, "\u0120Legend": 9883, "\u0120devast": 9884, "\u0120suspected": 9885, "isons": 9886, "\u0120Pok\u00c3\u00a9mon": 9887, "scale": 9888, "\u0120capabilities": 9889, "\u0120revel": 9890, "\u0120cheese": 9891, "dy": 9892, "igrant": 9893, "\u0120failing": 9894, "bits": 9895, "\u0120Heroes": 9896, "\u0120Ghost": 9897, "\u0120Scient": 9898, "\u0120appointed": 9899, "uri": 9900, "\u0120institution": 9901, "\u0120expanded": 9902, "greg": 9903, "\u0120monitoring": 9904, "\u0120podcast": 9905, "\u0120coalition": 9906, "\u012096": 9907, "Jo": 9908, "\u0120stolen": 9909, "\u0120Sab": 9910, "\u0120stops": 9911, "\u0120holiday": 9912, "\u0120intr": 9913, "Car": 9914, "Black": 9915, "\u0120LGBT": 9916, "\u0120warming": 9917, "\u0120Anderson": 9918, "\u012089": 9919, "\u0120producer": 9920, "Med": 9921, "\u0120accuracy": 9922, "\u0120Marvel": 9923, "izabeth": 9924, "\u0120Patrick": 9925, "mony": 9926, "\u0120mini": 9927, "acles": 9928, "\u0120overt": 9929, "they": 9930, "\u0120membership": 9931, "\u0120Ven": 9932, "\u0120exch": 9933, "\u0120removal": 9934, "\u0120Dave": 9935, "TY": 9936, "mad": 9937, "\u0120Find": 9938, "\u0120adequ": 9939, "\u0120ec": 9940, "\u0120teeth": 9941, "\u0120emotion": 9942, "\u0120perm": 9943, "\u0120solely": 9944, "db": 9945, "\u0120extraord": 9946, "IGHT": 9947, "cal": 9948, "\u0120guidelines": 9949, "\u0120dying": 9950, "\u0120suspended": 9951, "\u0120Premier": 9952, "\u0120Anthony": 9953, "elve": 9954, "\u0120dad": 9955, "\u0120Eth": 9956, "\u0120Football": 9957, "\u0120abandoned": 9958, "\u0120<<": 9959, "\u0120march": 9960, "\u0120horror": 9961, "\u00e2\u0122\u00a6\"": 9962, "\u0120childhood": 9963, "\u0120campaigns": 9964, "\u0120lunch": 9965, "\u0120Albert": 9966, "block": 9967, "\u00e2\u0138\u012a\u00e2\u0138\u012a": 9968, "ounding": 9969, "\u0120bone": 9970, "organ": 9971, "aders": 9972, "\u0120Flash": 9973, "\u0120Drive": 9974, "\u0120tonight": 9975, "\u0120wars": 9976, "\u0120FL": 9977, "\u0120formation": 9978, "const": 9979, "News": 9980, "\u0120compe": 9981, "orious": 9982, "\u0120Staff": 9983, "\u0120discussions": 9984, "\u0120Protection": 9985, "\u0120Jam": 9986, "\u0120criteria": 9987, "\u0120installation": 9988, "\u0120accomplish": 9989, "izza": 9990, "\u0120publisher": 9991, "\u0120rescue": 9992, "\u0120Try": 9993, "ULL": 9994, "\u0120Som": 9995, "\u0120Hop": 9996, "oret": 9997, "ths": 9998, "ordon": 9999, "\u0120pocket": 10000, "\u0120Inv": 10001, "Download": 10002, "\u0120Crime": 10003, "\u0120bene": 10004, "\u0120Guide": 10005, "\u0120Assembly": 10006, "\u0120parameters": 10007, "IE": 10008, "\u0120Alexander": 10009, "\u0120concert": 10010, "\u0120Sche": 10011, "\u0120shoes": 10012, "\u0120visiting": 10013, "\u0120recall": 10014, "\u0120bub": 10015, "\u0120rural": 10016, "\u0120concrete": 10017, "\u0120Ros": 10018, "Next": 10019, "Russ": 10020, "\u0120loans": 10021, "\u0120Shield": 10022, "\u0120trem": 10023, "hemat": 10024, "kg": 10025, "\u0120Harris": 10026, "isition": 10027, "\u0120Move": 10028, "\u0120FC": 10029, "\u0120fate": 10030, "\u0120Cho": 10031, "\u0120tired": 10032, "\u0120principal": 10033, "hist": 10034, "iences": 10035, "athy": 10036, "\u0120sevent": 10037, "\u0120mood": 10038, "\u0120strategic": 10039, "\u0120diseases": 10040, "\u0120forum": 10041, "\u0120tempor": 10042, "\u0120headquarters": 10043, "Par": 10044, "ige": 10045, "flix": 10046, "\u0120guitar": 10047, "\u012094": 10048, "Only": 10049, "\u0120releases": 10050, "roph": 10051, "================================": 10052, "\u0120600": 10053, "\u0120Continue": 10054, "igate": 10055, "\u0120Crit": 10056, "system": 10057, "\u0120disabled": 10058, "\u0120unexpected": 10059, "ithub": 10060, "\u0120unclear": 10061, "\u0120Est": 10062, "\u0120contrad": 10063, "\u0120strategies": 10064, "ventures": 10065, "\u0120passage": 10066, "AME": 10067, "\u0120improving": 10068, "\u0120reveals": 10069, "\u0120decrease": 10070, "ova": 10071, "\u0120annoy": 10072, "\u0120Short": 10073, "\u0120Library": 10074, "\u0120cyber": 10075, "nell": 10076, "\u0120Hur": 10077, "\u0120CB": 10078, "\u0120photograp": 10079, "UI": 10080, "\u0120sed": 10081, "Ge": 10082, "\u012087": 10083, "\u0120diverse": 10084, "\u0120encouraged": 10085, "\u0120conspiracy": 10086, "\u0120birds": 10087, "\u0120operator": 10088, "\u0120handful": 10089, "\u0120classified": 10090, "?)": 10091, "\u0120dramatic": 10092, "\u0120investigators": 10093, "ito": 10094, "\u0120widespread": 10095, "\u0120Room": 10096, "----------------------------------------------------------------": 10097, "\u0120collective": 10098, "\u0120journalist": 10099, "String": 10100, "\u0120temperatures": 10101, "ila": 10102, "\u0120guid": 10103, "\u0120inspect": 10104, "\u0120missile": 10105, "\u0120Mayor": 10106, "\u0120manual": 10107, "\u0120simultane": 10108, "\u0120ratings": 10109, "\u0120suck": 10110, "\u012097": 10111, "\u0120universal": 10112, "\u0120pharm": 10113, "\u0120disrupt": 10114, "iano": 10115, "AV": 10116, "\u0120ft": 10117, "\u0120statist": 10118, "olds": 10119, "\u0120Walker": 10120, "php": 10121, "\u0120undert": 10122, "\u0120Las": 10123, "ishop": 10124, "ntil": 10125, "reshold": 10126, "\u0120Whether": 10127, "Ms": 10128, "\u0120deny": 10129, "\u0120Cloud": 10130, "\u0120provider": 10131, "\u0120surviv": 10132, "\u0120Update": 10133, "has": 10134, "\u0120mistakes": 10135, "charge": 10136, "pled": 10137, "rity": 10138, "\u0120node": 10139, "\u0120Massachusetts": 10140, "ools": 10141, "lication": 10142, "\u0120fails": 10143, "emale": 10144, "ori": 10145, "backs": 10146, "\u0120shirt": 10147, "\u0120''": 10148, "\u0120NAT": 10149, "\u0120waters": 10150, "elson": 10151, "\u0120ease": 10152, "\u0120scar": 10153, "\u0120contents": 10154, "mind": 10155, "\u0120contribution": 10156, "\u0120shr": 10157, "\u0120handed": 10158, "\u0120stability": 10159, "\u0120trave": 10160, "Em": 10161, "\u0120mirror": 10162, "123": 10163, "\u0120weigh": 10164, "\u0120fiction": 10165, "ouver": 10166, "istant": 10167, "rition": 10168, "\u0120Fed": 10169, "\u0120physically": 10170, "\u0120stake": 10171, "\u0120Article": 10172, "\u0120Arc": 10173, "\u0120Lewis": 10174, "\u0120Mind": 10175, "\u0120demonstrate": 10176, "\u0120profits": 10177, "vision": 10178, "omic": 10179, "olid": 10180, "\u0120battles": 10181, "\u0120drives": 10182, "\u0120eastern": 10183, "\u0120Sony": 10184, "!!!": 10185, "aration": 10186, "vard": 10187, "\u0120GL": 10188, "portation": 10189, "\u012092": 10190, "\u0120lawmakers": 10191, "\u0120protecting": 10192, "\u0120EPA": 10193, "\u0120yeah": 10194, "\u0120shame": 10195, "olph": 10196, "even": 10197, "xit": 10198, "\u0120attach": 10199, "\u0120representing": 10200, "\u0120obs": 10201, "\u0120Utah": 10202, "iffs": 10203, "\u0120Freedom": 10204, "\u00c3\u00b3": 10205, "AK": 10206, "\u0120incidents": 10207, "itage": 10208, "\u0120viewers": 10209, "cd": 10210, "\u0120mouse": 10211, "\u0120clar": 10212, "\u0120accordance": 10213, "\u0120bot": 10214, "cor": 10215, "\u0120Summer": 10216, "held": 10217, "\u0120innocent": 10218, "\u0120initiative": 10219, "ols": 10220, "________________________________": 10221, "\u0120spots": 10222, "pace": 10223, "\u0120conventional": 10224, "\u0120corporations": 10225, "\u0120blocked": 10226, "HD": 10227, "attered": 10228, "\u0120refers": 10229, "\u0120buck": 10230, "\u0120Digital": 10231, "120": 10232, "\u0120topics": 10233, "TF": 10234, "\u00c4\u0123": 10235, "brid": 10236, "reement": 10237, "\u0120underlying": 10238, "\u0120Member": 10239, "\u0120investigating": 10240, "\u0120pregnancy": 10241, "\u0120touchdown": 10242, "\u0120Band": 10243, "\u0120Caller": 10244, "\u0120instances": 10245, "PP": 10246, "wa": 10247, "Good": 10248, "\u01201991": 10249, "\u0120Cold": 10250, "\u0120fears": 10251, "\u0120remarks": 10252, "\u0128\u0134": 10253, "atal": 10254, "\u0120mit": 10255, "\u0120experiments": 10256, "ipt": 10257, "Color": 10258, "indu": 10259, "Update": 10260, "\u012093": 10261, "Ag": 10262, "\u0120\u00e5": 10263, "ancouver": 10264, "Both": 10265, "\u0120judges": 10266, "Object": 10267, "\u0120stere": 10268, "umbn": 10269, "\u0120participation": 10270, "\u0120Stars": 10271, "\u0120Jere": 10272, "\u0120weekly": 10273, "\u0120Ban": 10274, "\u0120conversations": 10275, "\u0120Pitt": 10276, "uz": 10277, "\u0120Indiana": 10278, "\u0120Kick": 10279, "\u0120infection": 10280, "\u0120heroes": 10281, "\u0120settled": 10282, "\u0120strip": 10283, "\u0120hal": 10284, "\u0120dump": 10285, "\u0120Sci": 10286, "\u0120les": 10287, "\u0120references": 10288, "\u0120URL": 10289, "\u0120Bridge": 10290, "\u0120wanting": 10291, "Force": 10292, "\u0120exclus": 10293, "Meanwhile": 10294, "mn": 10295, "\u0120gentle": 10296, "maker": 10297, "senal": 10298, "\u0120Gro": 10299, "ouri": 10300, "\u0120Rain": 10301, "\u0120Alliance": 10302, "\u0120lift": 10303, "ela": 10304, "SD": 10305, "\u0120Cleveland": 10306, "\u0120ranked": 10307, "\u0120stadium": 10308, "\u0120deadly": 10309, "\u00e4\u00b8": 10310, "\u0120riding": 10311, "aria": 10312, "\u0120Armor": 10313, "\u0120documentation": 10314, "\u0120Greece": 10315, "reek": 10316, "\u0120lens": 10317, "\u0120Sa": 10318, "\u0120gross": 10319, "\u0120Emer": 10320, "agers": 10321, "\u0120Dub": 10322, "\u0120Rh": 10323, "\u0120AMD": 10324, "\u0120arrival": 10325, "\u0120desert": 10326, "\u0120supplement": 10327, "\u0120Resp": 10328, "\u0120knee": 10329, "\u0120margin": 10330, "font": 10331, "ogg": 10332, "2010": 10333, "\u0120Pir": 10334, "\u0120Prom": 10335, "ivals": 10336, "\u0120intake": 10337, "\u0120differently": 10338, "ugs": 10339, "\u0120bits": 10340, "cluded": 10341, "\u0120searching": 10342, "\u0120Du": 10343, "umble": 10344, "\u0120functional": 10345, "\u0120Baltimore": 10346, "\u0120Could": 10347, "\u0120desired": 10348, "\u0120circuit": 10349, "\u0120Lyn": 10350, "\u0120GO": 10351, "\u0120False": 10352, "repre": 10353, "':": 10354, "alties": 10355, "\u0120minim": 10356, "\u0120drove": 10357, "\u0120Should": 10358, "\u0120hip": 10359, "\u0120pros": 10360, "\u0120utility": 10361, "\u0120Nature": 10362, "\u0120Mode": 10363, "President": 10364, "opp": 10365, "rat": 10366, "formance": 10367, "\u0120concentration": 10368, "\u0120font": 10369, "\u0120Bud": 10370, "\u0120amid": 10371, "\u0120revers": 10372, "\u0120ML": 10373, "Bar": 10374, "\u0120interaction": 10375, "\u0120jurisd": 10376, "\u0120spells": 10377, "dep": 10378, "fil": 10379, "\u0120civilians": 10380, "utter": 10381, "\u0120Cooper": 10382, "\u0120Below": 10383, "\u0120entrance": 10384, "\u0120convert": 10385, "\u0120controversy": 10386, "owered": 10387, "\u0120contrary": 10388, "\u0120arc": 10389, "\u0120Executive": 10390, "\u0120Officer": 10391, "\u0120packages": 10392, "\u0120progressive": 10393, "width": 10394, "\u0120reserved": 10395, "vol": 10396, "\u0120Samsung": 10397, "\u0120printed": 10398, "\u0120centers": 10399, "\u0120introduce": 10400, "\u0120Kennedy": 10401, "\u0120odds": 10402, "\u0120surely": 10403, "\u0120independence": 10404, "\u0120passengers": 10405, "reprene": 10406, "\u0120Beh": 10407, "\u0120loves": 10408, "\u0120ESPN": 10409, "\u0120facilit": 10410, "\u0120identical": 10411, "\u0120doct": 10412, "\u0120partnership": 10413, "conf": 10414, "\u0120Hide": 10415, "\u0120confused": 10416, "\u0120Cow": 10417, "Men": 10418, "\u0120wrest": 10419, "\u0120Iraqi": 10420, "\u0120holes": 10421, "\u0120Studies": 10422, "\u0120pregnant": 10423, "hard": 10424, "\u0120signals": 10425, "IX": 10426, "\u0120pulling": 10427, "\u0120graduate": 10428, "\u0120nominee": 10429, "Date": 10430, "\u0120permitted": 10431, "\u0120\u00e2\u0124\u00ac": 10432, "\u0120Oklahoma": 10433, "Start": 10434, "\u0120authorized": 10435, "\u0120alarm": 10436, "\u0120Cos": 10437, "van": 10438, "\u0120generations": 10439, "cular": 10440, "\u0120dragon": 10441, "\u0120Software": 10442, "\u0120Edward": 10443, "\u0120controller": 10444, "Sen": 10445, "gered": 10446, "\u0120Vik": 10447, "\u0120approached": 10448, "Thank": 10449, "\u0120cance": 10450, "\u0120formula": 10451, "\u0120Small": 10452, "\u0120weakness": 10453, "\u0120ramp": 10454, "itudes": 10455, "jud": 10456, "\u0120brilliant": 10457, "\u0120accus": 10458, "source": 10459, "\u0120800": 10460, "\u0120Evil": 10461, "Sw": 10462, "\u0120homeless": 10463, "week": 10464, "iens": 10465, "rics": 10466, "\u0120Third": 10467, "TO": 10468, "\u0120organic": 10469, "\u0120presentation": 10470, "agh": 10471, "\u0120Download": 10472, "vation": 10473, "\u0120assembly": 10474, "orable": 10475, "holders": 10476, "\u0120Bernie": 10477, "\u0120Help": 10478, "\u0120tong": 10479, "\u0120Fight": 10480, "\u0120beach": 10481, "Book": 10482, "\u0120Lic": 10483, "\u0120rush": 10484, "\u0120Round": 10485, "oup": 10486, "\u0120Marx": 10487, "\u0120calculated": 10488, "\u0120Devil": 10489, "\u0120Sarah": 10490, "\u0120occasionally": 10491, "\u0120bullet": 10492, "Available": 10493, "gate": 10494, "\u012091": 10495, "\u0120hosp": 10496, "\u0120promises": 10497, "\u0120HIV": 10498, "\u0120Stadium": 10499, "\u0120Stock": 10500, "\u0120Corporation": 10501, "gage": 10502, "NG": 10503, "\u0120Credit": 10504, "\u0120sne": 10505, "ibl": 10506, "\u0120accum": 10507, "such": 10508, "\u0120terrorists": 10509, "\u0120consciousness": 10510, "\u0120Zh": 10511, "\u0120drama": 10512, "oola": 10513, "piration": 10514, "\u0120labour": 10515, "\u0120Nin": 10516, "\u0120utter": 10517, "\u0120democratic": 10518, "\u0120assass": 10519, "ilation": 10520, "\u0120gest": 10521, "\u0120abroad": 10522, "\u0120metab": 10523, "\u0120sorts": 10524, "\u0120flav": 10525, "UB": 10526, "\u0120mg": 10527, "\u0120Nothing": 10528, "\u0120Od": 10529, "\u0120musical": 10530, "2009": 10531, "\u0120drops": 10532, "ocated": 10533, "ateral": 10534, "000000": 10535, "\u0120gre": 10536, "\u0120equality": 10537, "\u0120burden": 10538, "\u0120vig": 10539, "\u0120Leader": 10540, "------------": 10541, "\u0120ceremony": 10542, "\u0120fighter": 10543, "\u0120actors": 10544, "\u0120\u00e6": 10545, "aman": 10546, "Fi": 10547, "\u0120align": 10548, "puter": 10549, "\u0120elder": 10550, "\u0120NSA": 10551, "\u0120representation": 10552, "\u0120Ontario": 10553, "ITH": 10554, "usalem": 10555, "\u0120harassment": 10556, "itzer": 10557, "\u0120symp": 10558, "\u0120boxes": 10559, "\u0120DR": 10560, "\u0120manifest": 10561, "atre": 10562, "\u0120^": 10563, "\u0120dies": 10564, "leton": 10565, "\u0120missions": 10566, "ethe": 10567, "\u0120resolve": 10568, "\u0120followers": 10569, "\u0120asc": 10570, "\u0120km": 10571, "lord": 10572, "ammed": 10573, "\u0120silent": 10574, "\u0120Associated": 10575, "\u0120timing": 10576, "\u0120prisoners": 10577, "\u0120Kings": 10578, "\u0120Five": 10579, "\u0120tower": 10580, "\u0120approaches": 10581, "\u0120precisely": 10582, "\u0120bureau": 10583, "\u0120Mother": 10584, "\u0120Iss": 10585, "\u0120keyboard": 10586, "itual": 10587, "\u0120funded": 10588, "\u0120staying": 10589, "\u0120psychological": 10590, "\u0120mile": 10591, "\u0120Leon": 10592, "\u0120Barb": 10593, "will": 10594, "\u0120wider": 10595, "\u0120Atlantic": 10596, "\u0120till": 10597, "\u0120Rome": 10598, "rot": 10599, "\u0120accompan": 10600, "\u0120flour": 10601, "aco": 10602, "World": 10603, "\u0120Express": 10604, "\u0120Yu": 10605, "Cor": 10606, "\u0120pleased": 10607, "party": 10608, "\u0120pointing": 10609, "\u0120inflation": 10610, "\u0120roy": 10611, "\u0120),": 10612, "ainer": 10613, "\u0120wedding": 10614, "ormon": 10615, "\u0120requiring": 10616, "\u0120qualified": 10617, "\u0120segment": 10618, "END": 10619, "\u0120sizes": 10620, "eals": 10621, "\u0120corrupt": 10622, "assador": 10623, "\u0120celeb": 10624, "\u0120dreams": 10625, "\u0120Mess": 10626, "\u0120checking": 10627, "\u0120Version": 10628, "\u0120preparing": 10629, "\u0120actively": 10630, "\u0120Diff": 10631, "\u0120lux": 10632, "\u0120Winter": 10633, "acteria": 10634, "\u0120NE": 10635, "\u0120deputy": 10636, "\u0120transgender": 10637, "\u0120summary": 10638, "\u0120inher": 10639, "eries": 10640, "char": 10641, "\u0120Yan": 10642, "\u0120knock": 10643, "\u0120Path": 10644, "\u0120lip": 10645, "roller": 10646, "\u0120impression": 10647, "\u0120celebrate": 10648, "\u0120slide": 10649, "\u0120guests": 10650, "\u0120clip": 10651, "FS": 10652, "\u0120savings": 10653, "\u0120captain": 10654, "\u0120legacy": 10655, "\u0120Denver": 10656, "\u0120wounded": 10657, "taboola": 10658, "ACT": 10659, "\u0120pursue": 10660, "\u0120oxy": 10661, "\u0120q": 10662, "\u0120semi": 10663, "\u0120Need": 10664, "\u0120Affairs": 10665, "\u0120obsc": 10666, "\u0120checked": 10667, "\u0120dual": 10668, "Code": 10669, "\u0120MD": 10670, "lem": 10671, "ulty": 10672, "\u0120\u00c2\u00a9": 10673, "\u0120Elizabeth": 10674, "\u0120centuries": 10675, "arded": 10676, "src": 10677, "\u0120evident": 10678, "ennis": 10679, "atin": 10680, "\u0120unemployment": 10681, "\u0120Mario": 10682, "\u0120intim": 10683, "Christ": 10684, "\u0120biological": 10685, "\u0120soldier": 10686, "\u0120Added": 10687, "\u0120math": 10688, "\u0120Gil": 10689, "\u0120bias": 10690, "\u0120dating": 10691, "\u0120Ocean": 10692, "\u0120mice": 10693, "Mus": 10694, "hire": 10695, "\u0120Tes": 10696, "Server": 10697, "limited": 10698, "Size": 10699, "\u0120meters": 10700, "\u0120rocket": 10701, "essee": 10702, "\u0120certificate": 10703, "\u0120Iranian": 10704, "ASS": 10705, "\u0120grid": 10706, "Dec": 10707, "\u0120rolling": 10708, "commun": 10709, "\u0120Sweden": 10710, "bury": 10711, "\u0120tissue": 10712, "\u0120racism": 10713, "\u0120Local": 10714, "\u0120mystery": 10715, "\u0120examine": 10716, "\u0120stem": 10717, "\u0120sits": 10718, "\u0120hoped": 10719, "oting": 10720, "\u0120dialogue": 10721, "\u0120persu": 10722, "Watch": 10723, "lay": 10724, "MAN": 10725, "\u0120chronic": 10726, "\u0120Portland": 10727, "market": 10728, "\u0120SEC": 10729, "\u0120parallel": 10730, "\u0120scandal": 10731, "\u0120carries": 10732, "\u0120phenomenon": 10733, "human": 10734, "acker": 10735, "\u0120Ox": 10736, "\u0120retirement": 10737, "tainment": 10738, "ovie": 10739, "\u0120Gear": 10740, "\u0120duties": 10741, "\u0120dose": 10742, "\u0120scroll": 10743, "MB": 10744, "inf": 10745, "\u0120sauce": 10746, "\u0120landscape": 10747, "reddit": 10748, "\u0120Championship": 10749, "\u0120Reddit": 10750, "alid": 10751, "\u0120coin": 10752, "\u0120overs": 10753, "\u0120posting": 10754, "about": 10755, "\u0120fel": 10756, "andy": 10757, "\u0120bold": 10758, "\u0120focusing": 10759, "effect": 10760, "GR": 10761, "\u0120deemed": 10762, "\u0120recommendations": 10763, "\u0120stepped": 10764, "\u0120voter": 10765, "\u0120Deep": 10766, "\u0120Instagram": 10767, "\u0120moderate": 10768, "\u0120Maryland": 10769, "\u0120restricted": 10770, "\u0120MB": 10771, "\u0120Chall": 10772, "\u0120tob": 10773, "\u0120cir": 10774, "\u0120Occ": 10775, "\u0120Ever": 10776, "\u0120collaps": 10777, "INFO": 10778, "=-": 10779, "\u0120Pict": 10780, "\u0120Account": 10781, "nc": 10782, "\u0120ought": 10783, "\u0120export": 10784, "\u0120drunk": 10785, "('": 10786, "\u0120wise": 10787, "\u0120Mort": 10788, "necess": 10789, "\u0120ancest": 10790, "\u0120Incre": 10791, "\u0120frequent": 10792, "mir": 10793, "\u0120interpretation": 10794, "\u0120dependent": 10795, "\u0120coins": 10796, "\u0120Bol": 10797, "Video": 10798, "\u0120Justin": 10799, "\u0120fatal": 10800, "\u0120cooking": 10801, "\u0120confusion": 10802, "ipher": 10803, "\u0120custody": 10804, "\u0120Morgan": 10805, "omach": 10806, "\u0120Governor": 10807, "\u0120restaurants": 10808, "eling": 10809, "\u0120acknowledged": 10810, "\u0120ther": 10811, "\u0120genes": 10812, "ching": 10813, "Hey": 10814, "\u0120tactics": 10815, "\u0120Mexican": 10816, "\u0120vend": 10817, "\u0120hes": 10818, "quer": 10819, "\u0120noting": 10820, "\u0120Cameron": 10821, "\u0120targeting": 10822, "rock": 10823, "\u0120credits": 10824, "\u0120emotions": 10825, "\u0120representatives": 10826, "news": 10827, "\u0120legislative": 10828, "\u0120removing": 10829, "\u0120tweeted": 10830, "\u0120Carter": 10831, "\u0120Fixed": 10832, "\u0120forcing": 10833, "\u0120speaker": 10834, "\u0120males": 10835, "\u0120Vietnam": 10836, "lined": 10837, "\u0120concepts": 10838, "\u0120voices": 10839, "oir": 10840, "\u0120Trib": 10841, "Whe": 10842, "\u0120Jerusalem": 10843, "\u0120Sant": 10844, "\u0120cul": 10845, "\u0120lady": 10846, "\u0120Hawai": 10847, "\u0120arts": 10848, "\u0120Inn": 10849, "\u0120Machine": 10850, "\u0120Emperor": 10851, "\u0120slot": 10852, "gly": 10853, "\u0120Process": 10854, "III": 10855, "\u0120athletes": 10856, "\u0120Temple": 10857, "\u0120Represent": 10858, "\u0120presc": 10859, "\u0120tons": 10860, "\u0120golden": 10861, "\u0120punch": 10862, "\u0120GR": 10863, "iverpool": 10864, "\u0120enact": 10865, "\u0120lobby": 10866, "\u0120mos": 10867, "\u0120picking": 10868, "\u0120lifetime": 10869, "\u0120cognitive": 10870, "Each": 10871, "zo": 10872, "\u0120dub": 10873, "\u0120consists": 10874, "oln": 10875, "\u0120festival": 10876, "amous": 10877, "\u0120intellig": 10878, "words": 10879, "\u0120Smart": 10880, "\u0120dele": 10881, "\u0120lapt": 10882, "\u0120magical": 10883, "\u0120Sin": 10884, "bus": 10885, "urities": 10886, "ighth": 10887, "\u0120Ruby": 10888, "\u0120Sure": 10889, "olving": 10890, "\u0120jun": 10891, "OST": 10892, "\u0120imposed": 10893, "\u0120astron": 10894, "\u0120correl": 10895, "\u0120NS": 10896, "\u0120Kit": 10897, "\u0120Future": 10898, "burn": 10899, "\u0120immune": 10900, "ocus": 10901, "\u0120courses": 10902, "\u0120String": 10903, "\u0120lean": 10904, "\u0120ghost": 10905, "\u0120outcomes": 10906, "\u0120expense": 10907, "\u0120everyday": 10908, "\u0120acceptable": 10909, "Ah": 10910, "\u0120equipped": 10911, "\u0120orange": 10912, "FR": 10913, "\u0120Dutch": 10914, "Though": 10915, "\u0120Rank": 10916, "QU": 10917, "\u0120Roberts": 10918, "what": 10919, "rend": 10920, "\u0120disappear": 10921, "\u0120spawn": 10922, "\u0120Lam": 10923, "ois": 10924, "\u0120deserve": 10925, "\u0120minimal": 10926, "\u0120nervous": 10927, "\u0120Would": 10928, "\u0120rook": 10929, "\u0120Vancouver": 10930, "\u0120resign": 10931, "shire": 10932, "\u0120Works": 10933, "\u0120Build": 10934, "\u0120affordable": 10935, "\u0120Gary": 10936, "\u0120Arena": 10937, "\u0120hanging": 10938, "\u0120implications": 10939, "\u0120Song": 10940, "\u0120maintaining": 10941, "\u0120guards": 10942, "CON": 10943, "\u0120derived": 10944, "\u0120executed": 10945, "\u0120theories": 10946, "\u0120quoted": 10947, "\u0120Andre": 10948, "oga": 10949, "seless": 10950, "info": 10951, "\u0120Belg": 10952, "\u0120tears": 10953, "\u0120Surv": 10954, "\u0120birthday": 10955, "igious": 10956, "immer": 10957, "\u0120spectrum": 10958, "\u0120architecture": 10959, "\u0120recruit": 10960, "arma": 10961, "Table": 10962, "\u0120monsters": 10963, "\u0120Gov": 10964, "\u0120destination": 10965, "\u0120attractive": 10966, "\u0120foss": 10967, "\u0120Moreover": 10968, "\u0120presents": 10969, "THE": 10970, "\u0120reply": 10971, "pton": 10972, "\u0120cum": 10973, "\u0120delight": 10974, "\u0120affects": 10975, "\u0120donations": 10976, "\u0120Toy": 10977, "\u0120Him": 10978, "MENT": 10979, "\u0120overcome": 10980, "itched": 10981, "\u0120Fantasy": 10982, "\u0120Hat": 10983, "\u0120Beast": 10984, "bott": 10985, "\u0120investigations": 10986, "Run": 10987, "\u0120hunting": 10988, "di": 10989, "fund": 10990, "\u0120sessions": 10991, "estyle": 10992, "\u0120portray": 10993, "oids": 10994, "Yeah": 10995, "\u0120communicate": 10996, "\u0120comedy": 10997, "\u0120Yang": 10998, "\u0120belt": 10999, "\u0120Marine": 11000, "\u0120predicted": 11001, "Play": 11002, "\u0120importantly": 11003, "\u0120remarkable": 11004, "\u0120eliminate": 11005, "David": 11006, "\u0120bind": 11007, "VID": 11008, "\u0120advocates": 11009, "\u0120Gaza": 11010, "imp": 11011, "DB": 11012, "\u0120Na": 11013, "\u0120Similar": 11014, "IES": 11015, "\u0120charity": 11016, "vas": 11017, "math": 11018, "\u0120\u00e2\u0138": 11019, "oker": 11020, "ndum": 11021, "\u0120caps": 11022, "\u0120Hal": 11023, "2000": 11024, "ean": 11025, "\u0120fleet": 11026, "\u0120recre": 11027, "Right": 11028, "\u0120sleeping": 11029, "ijing": 11030, "kind": 11031, "\u0120designated": 11032, "\u00c3\u00a4": 11033, "\u0120animation": 11034, "kee": 11035, "\u0120Introdu": 11036, "\u0120/>": 11037, "\u0120delayed": 11038, "\u0120tremend": 11039, "\u0120curious": 11040, "Use": 11041, "\u0120lect": 11042, "dam": 11043, "\u0120innovation": 11044, "\u0120Points": 11045, "\u0120loading": 11046, "\u0120dispute": 11047, "ctic": 11048, "irds": 11049, "\u0120BY": 11050, "\u0120nurs": 11051, "\u0120Value": 11052, "IONS": 11053, "\u0120Hum": 11054, "\u0120template": 11055, "mers": 11056, "\u0120appearances": 11057, "\u0120Entertainment": 11058, "\u0120translation": 11059, "\u0120sake": 11060, "\u0120beneath": 11061, "\u0120inhib": 11062, "\u0120euro": 11063, "abetes": 11064, "\u0120studying": 11065, "\u0120Mas": 11066, "\u0120perceived": 11067, "\u0120examined": 11068, "\u0120eager": 11069, "\u0120coaches": 11070, "\u0120imper": 11071, "chi": 11072, "\u0120produces": 11073, "\").": 11074, "\u0120Everyone": 11075, "\u0120municip": 11076, "\u0120girlfriend": 11077, "\u0120hire": 11078, "\u0120Vice": 11079, "\u0120suitable": 11080, "opy": 11081, "\u0120inequ": 11082, "\u0120Duke": 11083, "fish": 11084, "first": 11085, "\u0120Obs": 11086, "\u0120interior": 11087, "\u0120Bruce": 11088, "\u0120Ry": 11089, "\u0120analys": 11090, "\u0120considerable": 11091, "\u0120forecast": 11092, "\u0120fert": 11093, "orship": 11094, "\u0120Drug": 11095, "\u0120ALL": 11096, ":\"": 11097, "thur": 11098, "\u0120Mail": 11099, "\u0120ballot": 11100, "\u0120instantly": 11101, "\u0120Channel": 11102, "\u0120picks": 11103, "\u01201989": 11104, "\u0120tent": 11105, "oli": 11106, "\u0120civilian": 11107, "bling": 11108, "ello": 11109, "bu": 11110, "\u0120inch": 11111, "\u0120logo": 11112, "\u0120cooperation": 11113, "\u0120walks": 11114, "\u0120investments": 11115, "\u0120imprison": 11116, "\u0120Festival": 11117, "\u0120Ky": 11118, "\u0120legally": 11119, "\u0120gri": 11120, "charg": 11121, "Sl": 11122, "\u0120threatening": 11123, "duction": 11124, "flow": 11125, "\u0120dismissed": 11126, "ibraries": 11127, "cap": 11128, "ele": 11129, "\u0120McG": 11130, "\u0120Harvard": 11131, "\u0120Conservative": 11132, "\u0120CBS": 11133, "png": 11134, "\u0120roots": 11135, "\u0120Having": 11136, "umbled": 11137, "\u0120Fun": 11138, "\\/": 11139, "\u0120Search": 11140, "plex": 11141, "\u0120discussing": 11142, "\u0120continu": 11143, "\u0120Tai": 11144, "\u0120Wik": 11145, "Free": 11146, "fit": 11147, "\u0120refuse": 11148, "\u0120managing": 11149, "\u0120synd": 11150, "ipedia": 11151, "walk": 11152, "\u0120professionals": 11153, "\u0120guidance": 11154, "\u0120universities": 11155, "\u0120assemb": 11156, "untu": 11157, "Finally": 11158, "ASE": 11159, "\u0120Auto": 11160, "\u0120Had": 11161, "\u0120anniversary": 11162, "LD": 11163, "\u0120Dur": 11164, "\u0120Ultimate": 11165, "ihad": 11166, "product": 11167, "\u0120transit": 11168, "\u0120restore": 11169, "\u0120explaining": 11170, "\u0120asset": 11171, "\u0120transferred": 11172, "\u0120burst": 11173, "apolis": 11174, "\u0120Magazine": 11175, "\u0120Cra": 11176, "\u0120BR": 11177, "gged": 11178, "\u0120HE": 11179, "Mich": 11180, "bet": 11181, "\u0120Lady": 11182, "ylum": 11183, "erves": 11184, "\u0120meets": 11185, "white": 11186, "Log": 11187, "\u0120corresponding": 11188, "\u0120insisted": 11189, "GG": 11190, "\u0120surrounded": 11191, "\u0120tens": 11192, "\u0120lane": 11193, "\u0120coinc": 11194, "home": 11195, "\u0120existed": 11196, "ected": 11197, "\u0120Double": 11198, "lamm": 11199, "\u0120skept": 11200, "exp": 11201, "\u0120perception": 11202, "iev": 11203, "\u0120Being": 11204, "oft": 11205, "\u0120adopt": 11206, ".:": 11207, "];": 11208, "Windows": 11209, "\u0120satellite": 11210, "ASH": 11211, "\u0120infant": 11212, "description": 11213, "\u0120Meanwhile": 11214, "cm": 11215, "oca": 11216, "\u0120Treat": 11217, "actor": 11218, "\u0120tobacco": 11219, "\u0120Norm": 11220, "emption": 11221, "\u0120flesh": 11222, "\u0120je": 11223, "oop": 11224, "\u0120Heaven": 11225, "\u0120beating": 11226, "anim": 11227, "\u0120gathering": 11228, "\u0120cultiv": 11229, "GO": 11230, "abe": 11231, "\u0120Jonathan": 11232, "\u0120Safety": 11233, "\u0120badly": 11234, "prot": 11235, "\u0120choosing": 11236, "\u0120contacted": 11237, "\u0120quit": 11238, "\u0120distur": 11239, "\u0120stir": 11240, "\u0120token": 11241, "Det": 11242, "\u0120Pa": 11243, "\u0120functionality": 11244, "003": 11245, "some": 11246, "\u0120limitations": 11247, "\u0120meth": 11248, "build": 11249, "config": 11250, "NT": 11251, "rell": 11252, "blem": 11253, "\u0120Mom": 11254, "\u0120veterans": 11255, "\u0120Hu": 11256, "\u0120trends": 11257, "arer": 11258, "\u0120Given": 11259, "\u0120Caption": 11260, "may": 11261, "AST": 11262, "\u0120wondering": 11263, "\u0120Clark": 11264, "normal": 11265, "\u0120separated": 11266, "\u0120desp": 11267, "stic": 11268, "brew": 11269, "\u0120relating": 11270, "\u0120Nik": 11271, "\u0120Farm": 11272, "\u0120enthusi": 11273, "good": 11274, "deb": 11275, "\u0120activist": 11276, "\u0120mart": 11277, "\u0120explosion": 11278, "\u0120Economic": 11279, "Link": 11280, "\u0120insight": 11281, "\u0120convenient": 11282, "\u0120counterpart": 11283, "support": 11284, "\u0120Virt": 11285, "agen": 11286, "\u0120Tennessee": 11287, "\u0120Simon": 11288, "\u0120Award": 11289, "OCK": 11290, "\u0120Figure": 11291, "\u0120overseas": 11292, "\u0120pride": 11293, "\u0120Cas": 11294, "note": 11295, "mg": 11296, "Current": 11297, "\u0120displays": 11298, "content": 11299, "\u0120traveling": 11300, "\u0120hospitals": 11301, "\u0120Financial": 11302, "\u0120Past": 11303, "\u0120defendant": 11304, "\u0120streaming": 11305, "mble": 11306, "\u0120Berlin": 11307, "uki": 11308, "\u0120distribut": 11309, "\u0120antib": 11310, "\u0120chocolate": 11311, "\u0120Castle": 11312, "\u0120interrupt": 11313, "\u0120Row": 11314, "\u0120conversion": 11315, "\u0120bugs": 11316, "\u0120Rather": 11317, "liest": 11318, "LY": 11319, "\u0120Jean": 11320, "common": 11321, "akh": 11322, "\u0120130": 11323, "otton": 11324, "\u0120Dean": 11325, "\u0120amendment": 11326, "\u0120gameplay": 11327, "\u0120Warren": 11328, "oda": 11329, "\u0120highlights": 11330, "\u0120irre": 11331, "\u0120NATO": 11332, "\u0120balls": 11333, "\u0120demanding": 11334, "URE": 11335, "\u0120Luke": 11336, "Figure": 11337, "stop": 11338, "onia": 11339, "zone": 11340, "izers": 11341, "\u0120WR": 11342, "\u0120awarded": 11343, "\u0120regulatory": 11344, "\u0120Hart": 11345, "\u0120SN": 11346, "pling": 11347, "\u0120sour": 11348, "\u0120Pixel": 11349, "usive": 11350, "\u0120fet": 11351, "\u0120Sent": 11352, "\u0120automatic": 11353, "\u0120fer": 11354, "vernment": 11355, "\u0120Khan": 11356, "TON": 11357, "father": 11358, "\u0120extraordinary": 11359, "throp": 11360, "\u0120Python": 11361, "\u0120GPU": 11362, "\u0120sexually": 11363, "\u0120desktop": 11364, "itivity": 11365, "\u0120Antonio": 11366, "\u0120orient": 11367, "\u0120ears": 11368, "obby": 11369, "ouses": 11370, "vertisements": 11371, "\u0120manufacturers": 11372, "icient": 11373, "minute": 11374, "\u0120conviction": 11375, "\u0120garden": 11376, "public": 11377, "\u0120satisfied": 11378, "fold": 11379, "OK": 11380, "\u0120inhab": 11381, "\u0120Think": 11382, "\u0120programme": 11383, "\u0120stomach": 11384, "\u0120coordin": 11385, "\u0120holy": 11386, "\u0120threshold": 11387, "\u0120rhet": 11388, "\u0120serial": 11389, "\u0120employers": 11390, "\u0120Everything": 11391, "rah": 11392, "\u0120bother": 11393, "\u0120brands": 11394, "Value": 11395, "\u0120Ted": 11396, "\u0120Planet": 11397, "\u0120pink": 11398, "\u0120Furthermore": 11399, "sa": 11400, "PE": 11401, "reck": 11402, "\u0120USD": 11403, "otte": 11404, "\u0120&&": 11405, "\u0120landed": 11406, "gets": 11407, "\u0120producers": 11408, "\u0120healthcare": 11409, "\u0120dominant": 11410, "\u0120destro": 11411, "\u0120amended": 11412, "chron": 11413, "\u0120fits": 11414, "\u0120Syd": 11415, "\u0120Authority": 11416, "ATCH": 11417, "\u0120fights": 11418, "\u0120LLC": 11419, "\u0120---": 11420, "\u0120Corp": 11421, "\u0120toxic": 11422, "specific": 11423, "\u0120Corn": 11424, "\u0120Chel": 11425, "\u0120telephone": 11426, "\u0120Pant": 11427, "\u0120mysterious": 11428, "aunch": 11429, "odox": 11430, "media": 11431, "\u0120witnesses": 11432, "agu": 11433, "\u0120questioned": 11434, "\u0120Brexit": 11435, "\u0120Remember": 11436, "enez": 11437, "\u0120endorse": 11438, "iatric": 11439, "\u0120Ident": 11440, "\u0120ridiculous": 11441, "110": 11442, "\u0120prayer": 11443, "\u0120scientist": 11444, "\u01201950": 11445, "\u0120Aqu": 11446, "\u0120underground": 11447, "\u0120UFC": 11448, "mare": 11449, "\u0120Later": 11450, "wich": 11451, "\u0120subscrib": 11452, "\u0120hosts": 11453, "\u0120err": 11454, "\u0120grants": 11455, "antom": 11456, "\u0120summon": 11457, "early": 11458, "\u0120Clear": 11459, "\u0120Prim": 11460, "\u0120suspension": 11461, "\u0120guaranteed": 11462, "apper": 11463, "\u0120rice": 11464, "\u0120Sean": 11465, "\u0120Shin": 11466, "\u0120referendum": 11467, "\u0120fled": 11468, "rust": 11469, "\u0120360": 11470, "tery": 11471, "\u0120shocked": 11472, "BR": 11473, "\u0120Oil": 11474, "\u0120Allah": 11475, "\u0120partly": 11476, "\u0120ignor": 11477, "\u0120transmission": 11478, "\u0120homosexual": 11479, "iversal": 11480, "\u0120hopefully": 11481, "\u00e3\u0124\u00a4": 11482, "\u0120lesson": 11483, "Leg": 11484, "\u0120..": 11485, "Yet": 11486, "table": 11487, "appropri": 11488, "rett": 11489, "\u0120boards": 11490, "\u0120incorrect": 11491, "\u0120bacteria": 11492, "aru": 11493, "amac": 11494, "\u0120snap": 11495, ".'\"": 11496, "\u0120parad": 11497, "tem": 11498, "heart": 11499, "\u0120availability": 11500, "\u0120wisdom": 11501, "\u0120(+": 11502, "\u0120priest": 11503, "\u0120\u00c2\u0142\u0120\u00c2\u0142": 11504, "Open": 11505, "\u0120span": 11506, "\u0120parameter": 11507, "\u0120convince": 11508, "\u0120(%)": 11509, "rac": 11510, "\u0120fo": 11511, "\u0120safely": 11512, "\u0120converted": 11513, "\u0120Olympic": 11514, "\u0120reserve": 11515, "\u0120healing": 11516, "\u0120Mine": 11517, "Max": 11518, "\u0120inherent": 11519, "\u0120Graham": 11520, "\u0120integrated": 11521, "Dem": 11522, "\u0120pipeline": 11523, "\u0120applying": 11524, "\u0120embed": 11525, "\u0120Charlie": 11526, "\u0120cave": 11527, "2008": 11528, "\u0120consensus": 11529, "\u0120rewards": 11530, "Pal": 11531, "\u0120HTML": 11532, "\u0120popularity": 11533, "looking": 11534, "\u0120Sword": 11535, "\u0120Arts": 11536, "')": 11537, "\u0120electron": 11538, "clusions": 11539, "\u0120integrity": 11540, "\u0120exclusively": 11541, "\u0120grace": 11542, "\u0120torture": 11543, "\u0120burned": 11544, "two": 11545, "\u0120180": 11546, "Produ": 11547, "\u0120entreprene": 11548, "raphics": 11549, "\u0120gym": 11550, "ricane": 11551, "\u0120Tam": 11552, "\u0120administrative": 11553, "\u0120manufacturer": 11554, "\u0120vel": 11555, "\u0120Ni": 11556, "\u0120isolated": 11557, "\u0120Medicine": 11558, "\u0120backup": 11559, "\u0120promoting": 11560, "\u0120commander": 11561, "\u0120flee": 11562, "\u0120Russell": 11563, "\u0120forgotten": 11564, "\u0120Missouri": 11565, "\u0120residence": 11566, "mons": 11567, "\u0120resemb": 11568, "\u0120wand": 11569, "\u0120meaningful": 11570, "PT": 11571, "\u0120bol": 11572, "\u0120helic": 11573, "\u0120wealthy": 11574, "\u0120rifle": 11575, "strong": 11576, "rowing": 11577, "plan": 11578, "asury": 11579, "\u00e2\u0122\u00a6.": 11580, "\u0120expanding": 11581, "\u0120Hamilton": 11582, "\u0120receives": 11583, "SI": 11584, "eatures": 11585, "\u0120Anim": 11586, "REE": 11587, "Put": 11588, "\u0120briefly": 11589, "rive": 11590, "\u0120stimul": 11591, "\u0120``(": 11592, "\u0120__": 11593, "\u0120chip": 11594, "\u0120haz": 11595, "\u0120prize": 11596, "\u0120Things": 11597, "ACE": 11598, "ulin": 11599, "dict": 11600, "oku": 11601, "\u0120associate": 11602, "ockets": 11603, "youtube": 11604, "Story": 11605, "ategory": 11606, "\u0120mild": 11607, "ailing": 11608, "\u0120Ye": 11609, "Orig": 11610, "\u0120Ka": 11611, "orig": 11612, "\u0120propaganda": 11613, "\u0120anonymous": 11614, "\u0120struggled": 11615, "\u0120outrage": 11616, "ATED": 11617, "\u0120Beijing": 11618, "rary": 11619, "\u0120leather": 11620, "\u0120worlds": 11621, "\u0120broader": 11622, "125": 11623, "idal": 11624, "\u0120Better": 11625, "\u0120tear": 11626, "Ext": 11627, "\u0120proposals": 11628, "\u0120iter": 11629, "\u0120Squad": 11630, "\u0120volunt": 11631, "mi": 11632, "Did": 11633, "\u0120Pu": 11634, "pin": 11635, "\u0120speakers": 11636, "\u0120borders": 11637, "\u0120figured": 11638, "='": 11639, "\u0120simultaneously": 11640, "aeda": 11641, "\u0120charging": 11642, "\u0120urged": 11643, "\u0120conj": 11644, "256": 11645, "\u0120Gordon": 11646, "merce": 11647, "\u0120documentary": 11648, "Share": 11649, "itol": 11650, "ONE": 11651, "\u0120Garden": 11652, "hatt": 11653, "\u0120Thompson": 11654, "aneous": 11655, "apore": 11656, "\u0120tanks": 11657, "\u0120lessons": 11658, "track": 11659, "\u0120outstanding": 11660, "\u0120volunteers": 11661, "\u0120spray": 11662, "\u0120managers": 11663, "large": 11664, "\u0120camps": 11665, "\u0120artificial": 11666, "\u0120Ru": 11667, "\u0120bags": 11668, "thal": 11669, "\u0120compatible": 11670, "\u0120Blade": 11671, "\u0120fed": 11672, "\u0120argues": 11673, "FI": 11674, "\u0120unfair": 11675, "\u0120corn": 11676, "\u0120offset": 11677, "\u0120directions": 11678, "\u0120disappointed": 11679, "\u0120Convention": 11680, "\u0120viewing": 11681, "ME": 11682, "ocity": 11683, "\u0120towns": 11684, "\u0120layers": 11685, "\u0120rolled": 11686, "\u0120jumped": 11687, "\u0120attribute": 11688, "\u0120unnecess": 11689, "incoln": 11690, "\u0120suppose": 11691, "\u0120Nether": 11692, "cha": 11693, "\u0120buried": 11694, "\u0120sixth": 11695, "Ben": 11696, "ressing": 11697, "OUR": 11698, "\u0120wound": 11699, "\u0120cycl": 11700, "\u0120mechanisms": 11701, "\u0120congressional": 11702, "\u0120Element": 11703, "\u0120agreements": 11704, "\u0120decor": 11705, "\u0120closest": 11706, "\u0120Mit": 11707, "Google": 11708, "}}": 11709, "\u0120mixture": 11710, "\u0120fluid": 11711, "Sign": 11712, "\u0120Scholar": 11713, "\u0120pist": 11714, "asket": 11715, "abling": 11716, "\u0120racing": 11717, "hero": 11718, "riel": 11719, "assy": 11720, "\u0120cheaper": 11721, "ben": 11722, "\u0120vertical": 11723, "amacare": 11724, "\u0120Reading": 11725, "gments": 11726, "\u0120helicop": 11727, "\u0120sacrifice": 11728, "aya": 11729, "paren": 11730, "VA": 11731, "\u0120Les": 11732, "\u0120Studio": 11733, "\u0120violations": 11734, "\u0120Anna": 11735, "acer": 11736, "\u00e9\u00be": 11737, "\u0120Rat": 11738, "\u0120Beck": 11739, "\u0120Dick": 11740, "\u0120ACT": 11741, "\u0120composition": 11742, "\u0120texture": 11743, "\u0120Own": 11744, "\u0120smartphone": 11745, "\u0120NA": 11746, "\u0120forb": 11747, "import": 11748, "\u0120defending": 11749, "ilst": 11750, "rer": 11751, "\u0120oh": 11752, "\u0120Jeremy": 11753, "\u0120banking": 11754, "ceptions": 11755, "\u0120respective": 11756, "/.": 11757, "\u0120drinks": 11758, "\u0120Wi": 11759, "\u0120bands": 11760, "\u0120Liverpool": 11761, "\u0120grip": 11762, "\u0120Buy": 11763, "\u0120openly": 11764, "\u0120reviewed": 11765, "pert": 11766, "\u0120verify": 11767, "\u0120Cole": 11768, "\u0120Wales": 11769, "MO": 11770, "\u0120unpre": 11771, "\u0120shelter": 11772, "\u0120Imperial": 11773, "\u0120gui": 11774, "\u0120Dak": 11775, "\u0120suggestions": 11776, "\u0120explicitly": 11777, "\u0120slave": 11778, "\u0120blockchain": 11779, "\u0120competing": 11780, "\u0120promising": 11781, "SON": 11782, "\u0120soccer": 11783, "\u0120constitution": 11784, "429": 11785, "\u0120distract": 11786, "\u0120User": 11787, "esides": 11788, "\u0120Method": 11789, "\u0120Tokyo": 11790, "\u0120accompanied": 11791, "Client": 11792, "sur": 11793, "alog": 11794, "\u0120identification": 11795, "\u0120invasion": 11796, "asma": 11797, "\u0120industries": 11798, "ppers": 11799, "\u0120subtle": 11800, "\u0120Unit": 11801, "natural": 11802, "\u0120survived": 11803, "\u0120flaw": 11804, "\u013a\u0127": 11805, "\u0120Holl": 11806, "\u0120deficit": 11807, "\u0120tutorial": 11808, "\u0120Chance": 11809, "\u0120arguing": 11810, "\u0120contemporary": 11811, "\u0120integration": 11812, "forward": 11813, "\u0120tum": 11814, "itis": 11815, "\u0120hiding": 11816, "\u0120Domin": 11817, "\u0120Tan": 11818, "\u0120Building": 11819, "\u0120Vin": 11820, "\u0120spokesperson": 11821, "\u0120Notes": 11822, "\u0120emerging": 11823, "\u0120preparation": 11824, "\u0120prost": 11825, "\u0120suspects": 11826, "\u0120autonom": 11827, "Description": 11828, "\u0120dealt": 11829, "\u0120Pear": 11830, "\u0120steady": 11831, "\u0120decreased": 11832, "\u0120sovere": 11833, "\u0120Clin": 11834, "\u0120gradually": 11835, "orses": 11836, "\u0120WAR": 11837, "Serv": 11838, "\u00e3\u0124\u00a2": 11839, "hr": 11840, "\u0120dirty": 11841, "\u0120Barn": 11842, "\u0120BC": 11843, "\u0120dil": 11844, "\u0120calendar": 11845, "\u0120compliance": 11846, "\u0120chamber": 11847, "bb": 11848, "\u0120passenger": 11849, "ateful": 11850, "\u0120Title": 11851, "\u0120Sydney": 11852, "\u0120Got": 11853, "\u0120darkness": 11854, "\u0120defect": 11855, "\u0120packed": 11856, "assion": 11857, "\u0120gods": 11858, "\u0120harsh": 11859, "ICK": 11860, "leans": 11861, "\u0120algorithm": 11862, "\u0120oxygen": 11863, "\u0120visits": 11864, "\u0120blade": 11865, "\u0120kilomet": 11866, "\u0120Kentucky": 11867, "\u0120killer": 11868, "Pack": 11869, "enny": 11870, "\u0120divine": 11871, "\u0120nomination": 11872, "being": 11873, "\u0120engines": 11874, "\u0120cats": 11875, "\u0120buffer": 11876, "\u0120Phill": 11877, "\u0120traff": 11878, "AGE": 11879, "\u0120tongue": 11880, "\u0120radiation": 11881, "erer": 11882, "mem": 11883, "\u0120Explicit": 11884, "\u00e9\u00be\u012f": 11885, "\u0120couples": 11886, "\u0120physics": 11887, "\u0120McK": 11888, "\u0120politically": 11889, "awks": 11890, "\u0120Bloom": 11891, "\u0120worship": 11892, "eger": 11893, "uter": 11894, "\u0120FO": 11895, "\u0120mathemat": 11896, "\u0120sentenced": 11897, "\u0120disk": 11898, "\u0120Marg": 11899, "\u0120/*": 11900, "PI": 11901, "\u0120optional": 11902, "\u0120babies": 11903, "\u0120seeds": 11904, "\u0120Scottish": 11905, "\u0120thy": 11906, "]]": 11907, "\u0120Hitler": 11908, "PH": 11909, "ngth": 11910, "\u0120recovered": 11911, "inge": 11912, "\u0120powder": 11913, "\u0120lips": 11914, "\u0120designer": 11915, "\u0120disorders": 11916, "\u0120courage": 11917, "\u0120chaos": 11918, "\"},{\"": 11919, "\u0120carrier": 11920, "bably": 11921, "High": 11922, "\u0120RT": 11923, "esity": 11924, "len": 11925, "\u0120routes": 11926, "uating": 11927, "Fil": 11928, "NOT": 11929, "wall": 11930, "sburgh": 11931, "\u0120engaging": 11932, "\u0120JavaScript": 11933, "orer": 11934, "lihood": 11935, "\u0120unions": 11936, "\u0120Federation": 11937, "\u0120Tesla": 11938, "\u0120completion": 11939, "\u0120Ta": 11940, "\u0120privilege": 11941, "\u0120Orange": 11942, "\u0120neur": 11943, "parency": 11944, "\u0120bones": 11945, "\u0120titled": 11946, "\u0120prosecutors": 11947, "\u0120ME": 11948, "\u0120engineer": 11949, "\u0120Universe": 11950, "\u0120Hig": 11951, "nie": 11952, "oard": 11953, "\u0120hearts": 11954, "\u0120Gre": 11955, "ussion": 11956, "\u0120ministry": 11957, "\u0120penet": 11958, "\u0120Nut": 11959, "\u0120Ow": 11960, "\u0120XP": 11961, "instein": 11962, "\u0120bulk": 11963, "System": 11964, "icism": 11965, "\u0120Marketable": 11966, "\u0120preval": 11967, "\u0120poster": 11968, "\u0120attending": 11969, "urable": 11970, "\u0120licensed": 11971, "\u0120Gh": 11972, "etry": 11973, "\u0120Tradable": 11974, "\u0120blast": 11975, "\u00e0\u00a4": 11976, "\u0120Titan": 11977, "elled": 11978, "die": 11979, "Have": 11980, "\u0120Flame": 11981, "\u0120profound": 11982, "\u0120participating": 11983, "\u0120anime": 11984, "\u0120Ess": 11985, "\u0120specify": 11986, "\u0120regarded": 11987, "\u0120Spell": 11988, "\u0120sons": 11989, "owned": 11990, "\u0120merc": 11991, "\u0120experimental": 11992, "lando": 11993, "hs": 11994, "\u0120Dungeon": 11995, "inos": 11996, "\u0120comply": 11997, "\u0120Systems": 11998, "arth": 11999, "\u0120seized": 12000, "local": 12001, "\u0120Girls": 12002, "udo": 12003, "oned": 12004, "\u0120Fle": 12005, "\u0120constructed": 12006, "\u0120hosted": 12007, "\u0120scared": 12008, "actic": 12009, "\u0120Islands": 12010, "\u0120MORE": 12011, "\u0120bless": 12012, "\u0120blocking": 12013, "\u0120chips": 12014, "\u0120evac": 12015, "Ps": 12016, "\u0120corporation": 12017, "\u0120ox": 12018, "\u0120lighting": 12019, "\u0120neighbors": 12020, "\u0120Ub": 12021, "aro": 12022, "\u0120beef": 12023, "\u0120Uber": 12024, "Facebook": 12025, "armed": 12026, "itate": 12027, "\u0120Rating": 12028, "\u0120Quick": 12029, "\u0120occupied": 12030, "\u0120aims": 12031, "\u0120Additionally": 12032, "\u0120Interest": 12033, "\u0120dramatically": 12034, "\u0120heal": 12035, "\u0120painting": 12036, "\u0120engineers": 12037, "MM": 12038, "\u0120Must": 12039, "\u0120quantity": 12040, "Paul": 12041, "\u0120earnings": 12042, "\u0120Posts": 12043, "stra": 12044, "\u00e3\u0125\u00bc\u00e3\u0125": 12045, "\u0120stance": 12046, "\u0120dropping": 12047, "script": 12048, "\u0120dressed": 12049, "Make": 12050, "\u0120justify": 12051, "\u0120Ltd": 12052, "\u0120prompted": 12053, "\u0120scrut": 12054, "\u0120speeds": 12055, "\u0120Giants": 12056, "omer": 12057, "\u0120Editor": 12058, "\u0120describing": 12059, "\u0120Lie": 12060, "mented": 12061, "\u0120nowhere": 12062, "ocaly": 12063, "\u0120instruction": 12064, "fortable": 12065, "\u0120entities": 12066, "\u0120cm": 12067, "\u0120Natural": 12068, "\u0120inquiry": 12069, "\u0120pressed": 12070, "izont": 12071, "forced": 12072, "\u0120raises": 12073, "\u0120Netflix": 12074, "\u0120Side": 12075, "\u0120outer": 12076, "\u0120amongst": 12077, "ims": 12078, "owski": 12079, "\u0120climb": 12080, "never": 12081, "\u0120combine": 12082, "ding": 12083, "\u0120compr": 12084, "\u0120significance": 12085, "\u0120remembered": 12086, "\u0120Nevada": 12087, "\u0120Tel": 12088, "\u0120Scar": 12089, "\u0120Warriors": 12090, "\u0120Jane": 12091, "\u0120coup": 12092, "bas": 12093, "\u0120terminal": 12094, ",-": 12095, "OH": 12096, "\u0120tension": 12097, "\u0120wings": 12098, "\u0120Myster": 12099, "\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 12100, "\u0120Unlike": 12101, "valid": 12102, "vironments": 12103, "\u0120Ali": 12104, "\u0120naked": 12105, "books": 12106, "\u0120Mun": 12107, "\u0120Gulf": 12108, "\u0120density": 12109, "\u0120dimin": 12110, "\u0120desperate": 12111, "\u0120presidency": 12112, "\u01201986": 12113, "hy": 12114, "IND": 12115, "\u0120unlock": 12116, "imens": 12117, "\u0120handled": 12118, "\u0120Eb": 12119, "\u0120disappeared": 12120, "\u0120genre": 12121, "\u01201988": 12122, "\u0120determination": 12123, "Stream": 12124, "iko": 12125, "apters": 12126, "\u0120acknowledge": 12127, "Jan": 12128, "\u0120capitalism": 12129, "Pat": 12130, "\u01202020": 12131, "\u0120painful": 12132, "\u0120curve": 12133, "\u0120bombs": 12134, "storm": 12135, "\u0120Metal": 12136, "encer": 12137, "\u0120Fig": 12138, "\u0120Aaron": 12139, "anches": 12140, "\u0120inspiration": 12141, "\u0120exhaust": 12142, "tains": 12143, "ashi": 12144, "\u0120descript": 12145, "\u0120ritual": 12146, "\u0120Chelsea": 12147, "\u0120promotion": 12148, "\u0120Hung": 12149, "\u0120Ward": 12150, "iva": 12151, "\u0120ET": 12152, "\u0120toss": 12153, "allow": 12154, "\u0120Francis": 12155, "Dep": 12156, "\u0120happiness": 12157, "\u0120Glass": 12158, "\u0120beta": 12159, "\u0120strengthen": 12160, "NE": 12161, "oa": 12162, "\u0120buttons": 12163, "\u0120Murray": 12164, "\u0120kicked": 12165, "Quest": 12166, "\u0120Talk": 12167, "\u0120Several": 12168, "\u0120Zero": 12169, "\u0120drone": 12170, "ulk": 12171, "\u0120cam": 12172, "\u0120Mobile": 12173, "\u0120preventing": 12174, "\u0120retro": 12175, "\u0120Ax": 12176, "\u0120cruel": 12177, "\u0120float": 12178, ".),": 12179, "\u0120filing": 12180, "\u0120Grant": 12181, "\u0120Bor": 12182, "\u0120rib": 12183, "\u0120championship": 12184, "\u0120Merc": 12185, "\u0120styles": 12186, "\u0120cake": 12187, "\u0120builds": 12188, "\u0120Self": 12189, "iox": 12190, "\u0120epic": 12191, "oyd": 12192, "Bel": 12193, "\u0120Stew": 12194, ".(": 12195, "ahu": 12196, "\u0120Beyond": 12197, "\u0120outs": 12198, "\u0120solo": 12199, "\u0120Tree": 12200, "\u0120preserve": 12201, "\u0120tub": 12202, "ARE": 12203, "roc": 12204, "\u0120Impro": 12205, "\u0120Wright": 12206, "\u0120bund": 12207, "\u0120traged": 12208, "\u0120occasional": 12209, "bian": 12210, "Second": 12211, "rons": 12212, "\u0120interactions": 12213, "formed": 12214, "sing": 12215, "\u0120owns": 12216, "\u0120hockey": 12217, "General": 12218, "\u0120logical": 12219, "\u0120expend": 12220, "\u0120escal": 12221, "\u0120Griff": 12222, "\u0120Crown": 12223, "\u0120Reserve": 12224, "\u0120stopping": 12225, "\u0120excuse": 12226, "second": 12227, "\u0120operated": 12228, "\u0120reaches": 12229, "\u0120Malays": 12230, "\u0120pollution": 12231, "\u0120Brooklyn": 12232, "\u0120delete": 12233, "\u0120hash": 12234, "Block": 12235, "aha": 12236, "\u00e2\u0122\u00b3": 12237, "\u0120shorter": 12238, "piece": 12239, ">>>": 13163, "\u0120Mormon": 13164, "tor": 13165, "\u0120particles": 13166, "\u0120Bart": 13167, "ryption": 13168, "\u0120admin": 13169, "\u0120squee": 13170, "VIDIA": 13171, "\u0120creator": 13172, "iameter": 13173, "icular": 13174, "NBC": 13175, "\u0120grabbed": 13176, "\u0120nodd": 13177, "\u0120rated": 13178, "\u0120rotation": 13179, "\u0120grasp": 13180, "\u0120excessive": 13181, "\u0120EC": 13182, "\u0120Whit": 13183, "\u0120inventory": 13184, "aults": 13185, "\u0120FB": 13186, "\u0120ecosystem": 13187, "\u0120billions": 13188, "\u0120venture": 13189, "named": 13190, "\u0120defender": 13191, "oute": 13192, "Instead": 13193, "irable": 13194, "War": 13195, "\u0120assumption": 13196, "\u0120bite": 13197, "\u0120earthqu": 13198, "tail": 13199, "space": 13200, "\u0120gifts": 13201, "boys": 13202, "\u0120inevitable": 13203, "\u0120structural": 13204, "\u0120beneficial": 13205, "\u0120compelling": 13206, "hole": 13207, "ervation": 13208, "\u0120coat": 13209, "oj": 13210, "incarn": 13211, "\u0120Years": 13212, "\u0120determining": 13213, "\u0120rhetoric": 13214, "\u0120boundaries": 13215, "\u0120whites": 13216, "Ant": 13217, "addy": 13218, ")-": 13219, "raham": 13220, "etermin": 13221, "\u0120harvest": 13222, "\u0120Conc": 13223, "\u0120laptop": 13224, "\u0120Match": 13225, "\u0120enjoying": 13226, "cca": 13227, "ollar": 13228, "\u0120trips": 13229, "\u0120addiction": 13230, "\u0120Sak": 13231, "\u0120powered": 13232, "\u0120cous": 13233, "\u0120Russians": 13234, "iere": 13235, "\u0120retrie": 13236, "quality": 13237, "\u0120differ": 13238, "\u0120kingdom": 13239, "\u0120Laur": 13240, "\u0120Capitol": 13241, "\u0120conclusions": 13242, "\u0120Altern": 13243, "\u0120Nav": 13244, "\u0120transparent": 13245, "BER": 13246, "Group": 13247, "\u0120Complete": 13248, "\u0120infer": 13249, "\u0120intrig": 13250, "\u0120insane": 13251, "RO": 13252, "ophob": 13253, "isen": 13254, "qual": 13255, "Michael": 13256, "\u0120museum": 13257, "\u0120Pope": 13258, "\u0120reset": 13259, "rative": 13260, "five": 13261, "\u0120aggreg": 13262, "ittees": 13263, "ository": 13264, "\u0120carb": 13265, "\u0120Record": 13266, "\u0120decides": 13267, "\u0120Fix": 13268, "\u0120exceptions": 13269, "\u0120Commissioner": 13270, "uns": 13271, "\u0120Environmental": 13272, "\u0120legendary": 13273, "istence": 13274, "\u0120tunnel": 13275, "km": 13276, "\u0120insult": 13277, "\u0120troll": 13278, "\u0120shake": 13279, "\u0120detention": 13280, "ques": 13281, "\u0120Chrome": 13282, "\u0120Files": 13283, "\u0120subt": 13284, "\u0120prospects": 13285, "\u0120prol": 13286, "render": 13287, "proof": 13288, "\u0120performances": 13289, "Str": 13290, "\u0120href": 13291, "ername": 13292, "\u0120achievement": 13293, "\u0120fut": 13294, "Full": 13295, "\u0120Leban": 13296, "google": 13297, "\u00e3\u0125\u012a": 13298, "ampa": 13299, "Maybe": 13300, "\u0120projected": 13301, "\u0120Emb": 13302, "\u0120colleg": 13303, "\u0120awards": 13304, "\u0120\u00e2\u0136": 13305, "Gold": 13306, "\u0120Blake": 13307, "\u0120Raj": 13308, "ifting": 13309, "\u0120pending": 13310, "\u0120instinct": 13311, "\u0120developments": 13312, "Connect": 13313, "\u0120Mand": 13314, "\u0120WITH": 13315, "\u0120Philippines": 13316, "profile": 13317, "\u0120altogether": 13318, "\u0120Bund": 13319, "\u0120TD": 13320, "oooo": 13321, "amped": 13322, "iph": 13323, "\u0120steam": 13324, "\u0120oldest": 13325, "\u0120detection": 13326, "ulpt": 13327, "\u0120\u00e7": 13328, "\u0120Wayne": 13329, "2006": 13330, "fa": 13331, "\u0120circles": 13332, "\u0120Fu": 13333, "\u0120donors": 13334, "appropriate": 13335, "\u0120Dakota": 13336, "jamin": 13337, "\u0120motivated": 13338, "\u0120purchases": 13339, "\u0120Louisiana": 13340, "\u0120Spl": 13341, "\u0120globe": 13342, "\u0120105": 13343, "zip": 13344, "call": 13345, "\u0120departments": 13346, "\u0120sustainable": 13347, "105": 13348, "\u0120OP": 13349, "ifiers": 13350, "\u0120prevented": 13351, "\u0120incomp": 13352, "\u0120Commander": 13353, "\u0120dominated": 13354, "\u0120\u00c2\u00bb": 13355, "\u0120invested": 13356, "\u0120complexity": 13357, "\u0120incl": 13358, "\u0120ensuring": 13359, "\u0120realm": 13360, "ync": 13361, "\u0120Independent": 13362, "rained": 13363, "\u0120Jen": 13364, "\u0120Flight": 13365, "\u0120athe": 13366, "\u0120speculation": 13367, "\u0120TE": 13368, "ocate": 13369, "tic": 13370, "\u0120plaint": 13371, "herry": 13372, "\u0120toy": 13373, "\u0120111": 13374, "\u0120plates": 13375, "status": 13376, "\u0120Isa": 13377, "\u0120devoted": 13378, "Cop": 13379, "\u0120ES": 13380, "255": 13381, "urrency": 13382, "Main": 13383, "\u0120slaves": 13384, "\u0120pepper": 13385, "\u0120quotes": 13386, "\u0120ceiling": 13387, "\u0120Fish": 13388, "\u0120transformation": 13389, "\u0120fraction": 13390, "\u0120advantages": 13391, "\u0120toile": 13392, "\u0120stunning": 13393, "\u0120moist": 13394, "breaking": 13395, "si": 13396, "\u0120Location": 13397, "\u0120Medium": 13398, "\u0120texts": 13399, "\u0120ugly": 13400, "\u0120bio": 13401, ".\u00e2\u0122\u0136": 13402, "\u0120Based": 13403, "\u0120trains": 13404, "\u0120Wing": 13405, "\u0120Ancient": 13406, "\u0120Records": 13407, "\u0120Hope": 13408, "Special": 13409, "adesh": 13410, "obi": 13411, "[/": 13412, "\u0120temporarily": 13413, "Ver": 13414, "hu": 13415, "oser": 13416, "\u0120overnight": 13417, "\u0120mamm": 13418, "\u0120Treasury": 13419, "\u0120Venezuel": 13420, "\u0120Mega": 13421, "\u0120tar": 13422, "\u0120expects": 13423, "black": 13424, "orph": 13425, "\\\\\\\\": 13426, "\u0120acceptance": 13427, "\u0120radar": 13428, "sis": 13429, "\u0120junior": 13430, "\u0120frames": 13431, "\u0120observation": 13432, "acies": 13433, "Power": 13434, "\u0120Advanced": 13435, "Mag": 13436, "ologically": 13437, "\u0120Mechan": 13438, "\u0120sentences": 13439, "\u0120analysts": 13440, "aughters": 13441, "forcement": 13442, "\u0120vague": 13443, "\u0120clause": 13444, "\u0120directors": 13445, "\u0120evaluate": 13446, "\u0120cabinet": 13447, "Matt": 13448, "\u0120Classic": 13449, "Ang": 13450, "\u0120cler": 13451, "\u0120Buck": 13452, "\u0120researcher": 13453, "\u0120160": 13454, "\u0120poorly": 13455, "\u0120experiencing": 13456, "\u0120Ped": 13457, "\u0120Manhattan": 13458, "\u0120freed": 13459, "\u0120themes": 13460, "advant": 13461, "\u0120nin": 13462, "\u0120praise": 13463, "104": 13464, "\u0120Libya": 13465, "best": 13466, "\u0120trusted": 13467, "\u0120cease": 13468, "\u0120dign": 13469, "Direct": 13470, "\u0120bombing": 13471, "\u0120migration": 13472, "\u0120Sciences": 13473, "\u0120municipal": 13474, "\u0120Average": 13475, "\u0120glory": 13476, "\u0120revealing": 13477, "\u0120arena": 13478, "\u0120uncertainty": 13479, "\u0120battlefield": 13480, "iao": 13481, "God": 13482, "\u0120cinem": 13483, "rape": 13484, "elle": 13485, "apons": 13486, "\u0120listing": 13487, "\u0120waited": 13488, "\u0120spotted": 13489, "keley": 13490, "\u0120Audio": 13491, "eor": 13492, "arding": 13493, "idding": 13494, "igma": 13495, "\u0120Neg": 13496, "\u0120lone": 13497, "\u0120----": 13498, "exe": 13499, "deg": 13500, "\u0120transf": 13501, "\u0120wash": 13502, "\u0120slavery": 13503, "\u0120exploring": 13504, "\u0120WW": 13505, "atson": 13506, "\u0120encl": 13507, "lies": 13508, "\u0120Creek": 13509, "\u0120wooden": 13510, "Manager": 13511, "\u0120Brand": 13512, "ummy": 13513, "\u0120Arthur": 13514, "\u0120bureaucr": 13515, "\u0120blend": 13516, "arians": 13517, "Further": 13518, "\u0120supposedly": 13519, "\u0120winds": 13520, "\u01201979": 13521, "\u0120gravity": 13522, "\u0120analyses": 13523, "\u0120Travel": 13524, "\u0120Veter": 13525, "\u0120dumb": 13526, "\u0120alternate": 13527, "gal": 13528, "\u0120consumed": 13529, "\u0120effectiveness": 13530, ".''": 13531, "\u0120paths": 13532, "onda": 13533, "LA": 13534, "\u0120Strong": 13535, "\u0120enables": 13536, "\u0120escaped": 13537, "\u0120\"\"": 13538, "\u0120112": 13539, "\u01201983": 13540, "\u0120smiled": 13541, "\u0120tendency": 13542, "Fire": 13543, "\u0120pars": 13544, "\u0120Roc": 13545, "\u0120lake": 13546, "\u0120fitness": 13547, "\u0120Ath": 13548, "\u0120Horn": 13549, "\u0120hier": 13550, "\u0120impose": 13551, "mother": 13552, "\u0120pension": 13553, "icut": 13554, "borne": 13555, "iciary": 13556, "._": 13557, "\u0120SU": 13558, "\u0120polar": 13559, "isy": 13560, "engu": 13561, "itialized": 13562, "ATA": 13563, "write": 13564, "\u0120exercises": 13565, "\u0120Diamond": 13566, "otypes": 13567, "\u0120harmful": 13568, "onz": 13569, "\u0120printing": 13570, "story": 13571, "\u0120expertise": 13572, "\u0120Ger": 13573, "\u0120tragedy": 13574, "\u0120Fly": 13575, "\u0120divid": 13576, "ampire": 13577, "stock": 13578, "Mem": 13579, "\u0120reign": 13580, "\u0120unve": 13581, "\u0120amend": 13582, "\u0120Prophet": 13583, "\u0120mutual": 13584, "\u0120Fac": 13585, "\u0120replacing": 13586, "Har": 13587, "\u0120Circuit": 13588, "\u0120throat": 13589, "\u0120Shot": 13590, "\u0120batteries": 13591, "\u0120toll": 13592, "\u0120addressing": 13593, "\u0120Medicaid": 13594, "\u0120pupp": 13595, "\u0120Nar": 13596, "olk": 13597, "\u0120equity": 13598, "MR": 13599, "\u0120Hispan": 13600, "\u0120Large": 13601, "mid": 13602, "Dev": 13603, "\u0120exped": 13604, "\u0120demo": 13605, "\u0120Marshall": 13606, "ergus": 13607, "\u0120fiber": 13608, "\u0120divorce": 13609, "\u0120Create": 13610, "\u0120slower": 13611, "\u0120Parker": 13612, "\u0120Student": 13613, "\u0120Training": 13614, "Return": 13615, "\u0120Tru": 13616, "\u0120cub": 13617, "\u0120Reached": 13618, "\u0120panic": 13619, "\u0120quarters": 13620, "\u0120rect": 13621, "\u0120treating": 13622, "\u0120rats": 13623, "\u0120Christianity": 13624, "oler": 13625, "\u0120sacred": 13626, "\u0120declare": 13627, "ulative": 13628, "eting": 13629, "\u0120delivering": 13630, "estone": 13631, "\u0120tel": 13632, "\u0120Larry": 13633, "\u0120meta": 13634, "accept": 13635, "artz": 13636, "\u0120Roger": 13637, "handed": 13638, "\u0120header": 13639, "\u0120trapped": 13640, "\u0120Century": 13641, "\u0120knocked": 13642, "\u0120Oxford": 13643, "\u0120survivors": 13644, "bot": 13645, "\u0120demonstration": 13646, "\u0120dirt": 13647, "\u0120assists": 13648, "OME": 13649, "\u0120Draft": 13650, "ortunate": 13651, "folio": 13652, "pered": 13653, "usters": 13654, "gt": 13655, "\u0120Lock": 13656, "\u0120judicial": 13657, "verted": 13658, "\u0120secured": 13659, "outing": 13660, "\u0120Books": 13661, "\u0120hosting": 13662, "\u0120lifted": 13663, "length": 13664, "\u0120jer": 13665, "\u0120wheels": 13666, "\u0120Range": 13667, "umbnails": 13668, "\u0120diagnosis": 13669, "tech": 13670, "\u0120Stewart": 13671, "\u0120Pract": 13672, "\u0120nationwide": 13673, "\u0120dear": 13674, "\u0120obligations": 13675, "\u0120grows": 13676, "\u0120mandatory": 13677, "\u0120suspicious": 13678, "!'": 13679, "Apr": 13680, "Great": 13681, "\u0120mortgage": 13682, "\u0120prosecutor": 13683, "\u0120editorial": 13684, "\u0120Kr": 13685, "\u0120processed": 13686, "ungle": 13687, "\u0120flexibility": 13688, "Earlier": 13689, "\u0120Cart": 13690, "\u0120Sug": 13691, "\u0120focuses": 13692, "\u0120startup": 13693, "\u0120breach": 13694, "\u0120Tob": 13695, "cycle": 13696, "\u00e3\u0122\u012e": 13697, "rose": 13698, "\u0120bizarre": 13699, "\u00e3\u0122\u012f": 13700, "\u0120vegetables": 13701, "$$": 13702, "\u0120retreat": 13703, "oshi": 13704, "\u0120Shop": 13705, "\u0120Ground": 13706, "\u0120Stop": 13707, "\u0120Hawaii": 13708, "\u0120Ay": 13709, "Perhaps": 13710, "\u0120Beaut": 13711, "uffer": 13712, "enna": 13713, "\u0120productivity": 13714, "Fixed": 13715, "control": 13716, "\u0120absent": 13717, "\u0120Campaign": 13718, "Green": 13719, "\u0120identifying": 13720, "\u0120regret": 13721, "\u0120promoted": 13722, "\u0120Seven": 13723, "\u0120eru": 13724, "neath": 13725, "aughed": 13726, "\u0120Pin": 13727, "\u0120Living": 13728, "Cost": 13729, "omatic": 13730, "mega": 13731, "\u0120Nig": 13732, "ocy": 13733, "\u0120inbox": 13734, "\u0120empire": 13735, "\u0120horizont": 13736, "\u0120branches": 13737, "\u0120metaph": 13738, "Active": 13739, "edi": 13740, "\u0120Film": 13741, "\u0120Something": 13742, "\u0120mods": 13743, "incial": 13744, "\u0120Original": 13745, "Gen": 13746, "\u0120spirits": 13747, "\u0120earning": 13748, "Hist": 13749, "\u0120riders": 13750, "\u0120sacrific": 13751, "MT": 13752, "\u0120VA": 13753, "\u0120Salt": 13754, "\u0120occupation": 13755, "\u0120Mi": 13756, "\u0120disg": 13757, "lict": 13758, "\u0120nit": 13759, "\u0120nodes": 13760, "eem": 13761, "\u0120Pier": 13762, "\u0120hatred": 13763, "psy": 13764, "\u00e3\u0125\u012b": 13765, "\u0120theater": 13766, "\u0120sophisticated": 13767, "\u0120defended": 13768, "\u0120besides": 13769, "\u0120thoroughly": 13770, "\u0120Medicare": 13771, "\u0120blamed": 13772, "arently": 13773, "\u0120crying": 13774, "FOR": 13775, "priv": 13776, "\u0120singing": 13777, "\u0120Il": 13778, "\u0120cute": 13779, "oided": 13780, "olitical": 13781, "\u0120Neuro": 13782, "\u00e5\u00a4": 13783, "\u0120donation": 13784, "\u0120Eagles": 13785, "\u0120Give": 13786, "Tom": 13787, "\u0120substantially": 13788, "\u0120License": 13789, "\u0120Ja": 13790, "\u0120grey": 13791, "\u0120Animal": 13792, "\u0120ER": 13793, "\u0120Und": 13794, "\u0120keen": 13795, "\u0120conclude": 13796, "\u0120Mississippi": 13797, "Engine": 13798, "\u0120Studios": 13799, "Press": 13800, "overs": 13801, "llers": 13802, "\u0120350": 13803, "\u0120Rangers": 13804, "\u0120rou": 13805, "erto": 13806, "Ep": 13807, "issa": 13808, "ivan": 13809, "\u0120seal": 13810, "\u0120Regist": 13811, "display": 13812, "\u0120weaken": 13813, "uum": 13814, "\u0120Commons": 13815, "\u0120Say": 13816, "\u0120cultures": 13817, "\u0120laughed": 13818, "\u0120slip": 13819, "\u0120treatments": 13820, "izable": 13821, "mart": 13822, "\u0120Rice": 13823, "\u0120beast": 13824, "\u0120obesity": 13825, "\u0120Laure": 13826, "iga": 13827, "Which": 13828, "holder": 13829, "\u0120elderly": 13830, "\u0120pays": 13831, "\u0120complained": 13832, "\u0120crop": 13833, "\u0120proc": 13834, "\u0120explosive": 13835, "\u0120Fan": 13836, "\u0120Arsenal": 13837, "Author": 13838, "eful": 13839, "\u0120meals": 13840, "\u0120(-": 13841, "idays": 13842, "\u0120imagination": 13843, "\u0120annually": 13844, "\u0120ms": 13845, "asures": 13846, "Head": 13847, "ikh": 13848, "matic": 13849, "\u0120boyfriend": 13850, "\u0120Computer": 13851, "\u0120bump": 13852, "\u0120surge": 13853, "\u0120Craig": 13854, "\u0120Kirk": 13855, "Del": 13856, "mediate": 13857, "\u0120scenarios": 13858, "\u0120Mut": 13859, "\u0120Stream": 13860, "\u0120competitors": 13861, "\u00d9\u0126": 13862, "\u0120Stanford": 13863, "\u0120Resources": 13864, "azed": 13865, "bage": 13866, "\u0120organis": 13867, "\u0120Release": 13868, "\u0120separately": 13869, "\u0120habits": 13870, "\u0120measurements": 13871, "\u0120Close": 13872, "\u0120accompany": 13873, "\u0120gly": 13874, "\u0120tang": 13875, "\u0120Rou": 13876, "\u0120plugin": 13877, "\u0120convey": 13878, "\u0120Challenge": 13879, "oots": 13880, "jan": 13881, "\u0120curs": 13882, "\u0120Relations": 13883, "keeper": 13884, "\u0120approaching": 13885, "ping": 13886, "Speaking": 13887, "\u0120arrangement": 13888, "\u0120VI": 13889, "arettes": 13890, "\u0120affecting": 13891, "\u0120permits": 13892, "because": 13893, "\u0120useless": 13894, "\u0120Hus": 13895, "!!!!": 13896, "\u0120destroying": 13897, "Unfortunately": 13898, "\u0120fascinating": 13899, "Sem": 13900, "\u0120electoral": 13901, "\u0120transparency": 13902, "\u0120Chaos": 13903, "\u0120volunteer": 13904, "\u0120statistical": 13905, "\u0120activated": 13906, "rox": 13907, "Web": 13908, "HE": 13909, "\u0120Hampshire": 13910, "isive": 13911, "Map": 13912, "\u0120trash": 13913, "\u0120Lawrence": 13914, "stick": 13915, "Cr": 13916, "\u0120rings": 13917, "EXT": 13918, "\u0120operational": 13919, "opes": 13920, "Does": 13921, "\u0120Evans": 13922, "\u0120witnessed": 13923, "Port": 13924, "\u0120launching": 13925, "econom": 13926, "wear": 13927, "\u0120Particip": 13928, "umm": 13929, "cules": 13930, "\u0120RAM": 13931, "\u0120Tun": 13932, "\u0120assured": 13933, "\u0120binary": 13934, "\u0120betray": 13935, "\u0120exploration": 13936, "\u0120Fel": 13937, "\u0120admission": 13938, "itated": 13939, "Sy": 13940, "\u0120avoided": 13941, "\u0120Simulator": 13942, "\u0120celebrated": 13943, "\u0120Electric": 13944, "\u00a5\u0140": 13945, "\u0120cluster": 13946, "itzerland": 13947, "health": 13948, "Line": 13949, "\u0120Nash": 13950, "aton": 13951, "\u0120spare": 13952, "\u0120enterprise": 13953, "\u0120DIS": 13954, "cludes": 13955, "\u0120flights": 13956, "\u0120regards": 13957, "\u0120\u00c3\u0139": 13958, "half": 13959, "\u0120trucks": 13960, "\u0120contacts": 13961, "\u0120uncons": 13962, "\u0120Climate": 13963, "\u0120immense": 13964, "NEW": 13965, "occ": 13966, "ective": 13967, "\u0120embod": 13968, "\u0120patrol": 13969, "\u0120beside": 13970, "\u0120viable": 13971, "\u0120creep": 13972, "\u0120triggered": 13973, "verning": 13974, "\u0120comparable": 13975, "ql": 13976, "\u0120gaining": 13977, "asses": 13978, "\u0120();": 13979, "\u0120Grey": 13980, "\u0120MLS": 13981, "sized": 13982, "\u0120prosper": 13983, "\"?": 13984, "\u0120polling": 13985, "\u0120shar": 13986, "\u0120RC": 13987, "\u0120firearm": 13988, "orient": 13989, "\u0120fence": 13990, "\u0120variations": 13991, "giving": 13992, "\u0120Pi": 13993, "ospel": 13994, "\u0120pledge": 13995, "\u0120cure": 13996, "\u0120spy": 13997, "\u0120violated": 13998, "\u0120rushed": 13999, "\u0120stroke": 14000, "\u0120Blog": 14001, "sels": 14002, "\u0120Ec": 14003, ",''": 14004, "\u0120pale": 14005, "\u0120Collins": 14006, "terror": 14007, "\u0120Canadians": 14008, "\u0120tune": 14009, "\u0120laboratory": 14010, "\u0120nons": 14011, "tarian": 14012, "\u0120disability": 14013, "\u0120Gam": 14014, "\u0120singer": 14015, "alg": 14016, "\u0120Senior": 14017, "\u0120traded": 14018, "\u0120Warrior": 14019, "\u0120infring": 14020, "\u0120Franklin": 14021, "\u0120strain": 14022, "\u0120Swedish": 14023, "\u0120seventh": 14024, "\u0120Benn": 14025, "\u0120Tell": 14026, "\u0120syndrome": 14027, "\u0120wondered": 14028, "iden": 14029, "++++": 14030, "igo": 14031, "\u0120purple": 14032, "\u0120journalism": 14033, "\u0120rebel": 14034, "\u0120fu": 14035, "blog": 14036, "\u0120invite": 14037, "rencies": 14038, "\u0120Contact": 14039, "Israel": 14040, "\u0120Content": 14041, "\u0120cheer": 14042, "\u0120bedroom": 14043, "\u0120Engineering": 14044, "\u0120Queens": 14045, "\u0120dwell": 14046, "\u0120PlayStation": 14047, "\u0120Dim": 14048, "\u0120Colon": 14049, "lr": 14050, "\u0120operates": 14051, "\u0120motivation": 14052, "USA": 14053, "astered": 14054, "Core": 14055, "\u0120Truth": 14056, "olo": 14057, "OSE": 14058, "\u0120Memory": 14059, "\u0120predec": 14060, "\u0120anarch": 14061, "\u01201920": 14062, "\u0120Yam": 14063, "\u00c3\u00a8": 14064, "bid": 14065, "\u0120grateful": 14066, "\u0120excitement": 14067, "\u0120treasure": 14068, "\u0120longest": 14069, "ctive": 14070, "\u0120deserves": 14071, "\u0120reserves": 14072, "\u0120cops": 14073, "\u0120Ottawa": 14074, "\u0120Egyptian": 14075, "anked": 14076, "\u0120artif": 14077, "\u0120hypothesis": 14078, ":/": 14079, "\u0120purchasing": 14080, "\u0120lovely": 14081, "HP": 14082, "\u0120divide": 14083, "\u0120strictly": 14084, "\u0120questioning": 14085, "\u0120taxpayers": 14086, "\u0120Joy": 14087, "\u0120rolls": 14088, "\u0120Heavy": 14089, "\u0120ports": 14090, "\u0120magnetic": 14091, "\u0120inflamm": 14092, "\u0120brush": 14093, "tics": 14094, "\u00e2\u012a\u0134": 14095, "\u0120bottles": 14096, "ppy": 14097, "\u0120padd": 14098, "\u00e3\u0124\u00af": 14099, "million": 14100, "\u0120devastating": 14101, "\u0120compiled": 14102, "\u0120medication": 14103, "\u0120twelve": 14104, "\u0120Perry": 14105, "Space": 14106, "imb": 14107, "your": 14108, "\u0120leaked": 14109, "\u0120Tar": 14110, "\u0120unity": 14111, "\u0120infected": 14112, "\u0120traveled": 14113, "IDE": 14114, "\u0120McDonald": 14115, "txt": 14116, "\u0120Princ": 14117, "\u0120interven": 14118, "\u0120Taiwan": 14119, "\u0120Pow": 14120, "\u0120bearing": 14121, "\u0120Thread": 14122, "\u0120zones": 14123, "izards": 14124, "unks": 14125, "Chapter": 14126, "llor": 14127, "\u0120\u00c2\u00b7": 14128, "\u0120wounds": 14129, "\u0120discretion": 14130, "\u0120succeeded": 14131, "iking": 14132, "\u0120iconic": 14133, "Call": 14134, "\u0120screening": 14135, "\u0120Mis": 14136, "icts": 14137, "\u0120ministers": 14138, "\u0120separation": 14139, "Player": 14140, "\u0120bip": 14141, "\u0120beloved": 14142, "\u0120counting": 14143, "\u0120Eye": 14144, "around": 14145, "inging": 14146, "\u0120tablet": 14147, "\u0120offence": 14148, "inance": 14149, "have": 14150, "\u0120Info": 14151, "\u0120Ninja": 14152, "\u0120protective": 14153, "\u0120Cass": 14154, "Mac": 14155, "\u0120Quality": 14156, "North": 14157, "\u0120ic": 14158, "\u0120Cuba": 14159, "\u0120Chronicle": 14160, "\u0120Property": 14161, "\u0120fastest": 14162, "otos": 14163, "\u0120Germ": 14164, "OWN": 14165, "\u0120boom": 14166, "\u0120Stanley": 14167, "erguson": 14168, "\u0120clever": 14169, "\u0120enters": 14170, "mode": 14171, "terior": 14172, "\u0120Sens": 14173, "\u0120linear": 14174, "ARK": 14175, "\u0120comparing": 14176, "\u0120purely": 14177, "\u0120safer": 14178, "\u0120Potter": 14179, "\u0120cups": 14180, "RT": 14181, "\u0120gluc": 14182, "\u0120attributed": 14183, "\u0120dupl": 14184, "\u0120Pap": 14185, "\u0120precious": 14186, "\u0120pa": 14187, "ictionary": 14188, "\u0120Tig": 14189, "\u0120Too": 14190, "olutions": 14191, "stan": 14192, "\u0120robots": 14193, "\u0120lobb": 14194, "\u0120statute": 14195, "\u0120prevention": 14196, "western": 14197, "160": 14198, "\u0120Active": 14199, "\u0120Maria": 14200, "hal": 14201, "None": 14202, "ellar": 14203, "\u0120KB": 14204, "\u0120Partners": 14205, "\u0120Single": 14206, "\u0120Following": 14207, "ango": 14208, "acious": 14209, "\u0120thou": 14210, "\u0120kg": 14211, "\u0120influential": 14212, "\u0120Friends": 14213, "Sur": 14214, "ainted": 14215, "\u0120forums": 14216, "\u0120starter": 14217, "\u0120citizenship": 14218, "\u0120Election": 14219, "onge": 14220, "otation": 14221, "osph": 14222, ";;;;": 14223, "utical": 14224, "pur": 14225, "eren": 14226, "\u0120accusations": 14227, "bitious": 14228, "abbit": 14229, "\u0120Ord": 14230, "Posted": 14231, "irk": 14232, "\u0120sensitivity": 14233, "iche": 14234, "\u0120Amy": 14235, "\u0120Fab": 14236, "\u0120summit": 14237, "\u0120pedest": 14238, "\u0120rubber": 14239, "\u0120agricultural": 14240, "\u0120cancel": 14241, "AE": 14242, "\u0120inaug": 14243, "\u0120contam": 14244, "\u0120firmly": 14245, "iw": 14246, "stage": 14247, "\u0120Kan": 14248, "\u0120tier": 14249, "\u0120invention": 14250, "\u0120translated": 14251, "\u0120Rules": 14252, "Box": 14253, "Twitter": 14254, "IDS": 14255, "\u0120pizza": 14256, "\u0120debug": 14257, "\u0120Drop": 14258, "vs": 14259, "\u0120horses": 14260, "big": 14261, "\u0120boring": 14262, "\u0120hood": 14263, "\u0120McCain": 14264, "atched": 14265, "\u0120Bros": 14266, "\u0120skip": 14267, "\u0120essay": 14268, "stat": 14269, "\u0120Legends": 14270, "\u0120ammunition": 14271, "auc": 14272, "\u0120shooter": 14273, "\u0120unh": 14274, "\u0120supplied": 14275, "\u0120generic": 14276, "\u0120SK": 14277, "iban": 14278, "yrics": 14279, "\u0120255": 14280, "\u0120climbing": 14281, "Former": 14282, "\u0120flip": 14283, "\u0120jumping": 14284, "\u0120frustration": 14285, "\u0120Terry": 14286, "\u0120neighborhoods": 14287, "\u0120median": 14288, "bean": 14289, "\u0120brains": 14290, "Following": 14291, "\u0120shaped": 14292, "\u0120draws": 14293, "\u0120altered": 14294, "Jack": 14295, "\u0120recipes": 14296, "\u0120skilled": 14297, "wealth": 14298, "achi": 14299, "election": 14300, "\u0120behaviors": 14301, "deals": 14302, "\u0120Until": 14303, "Fe": 14304, "\u0120declaration": 14305, "marks": 14306, "\u0120Between": 14307, "celona": 14308, "\u0120reson": 14309, "\u0120bubble": 14310, "Among": 14311, "\u0120imperial": 14312, "GS": 14313, "\u0120feminist": 14314, "2005": 14315, "\u0120Kyle": 14316, "\u0120accounting": 14317, "\u0120Tele": 14318, "\u0120Tyr": 14319, "\u0120connecting": 14320, "\u0120rehab": 14321, "\u0120Pred": 14322, "sim": 14323, "\u0120meantime": 14324, "\u0120physician": 14325, "MW": 14326, "\u0120Campbell": 14327, "\u0120Brandon": 14328, "\u0120contributing": 14329, "\u0120Rule": 14330, "\u0120Weight": 14331, "\u0120Nap": 14332, "\u0120interactive": 14333, "\u0120vag": 14334, "\u0120helmet": 14335, "\u0120Comb": 14336, "four": 14337, "\u0120shipped": 14338, "\u0120completing": 14339, "\u0120PD": 14340, "PDATE": 14341, "\u0120spreading": 14342, "\u0120scary": 14343, "erving": 14344, "\u0120Gas": 14345, "\u0120frank": 14346, "school": 14347, "\u0120romantic": 14348, "\u0120stabil": 14349, "Rob": 14350, "\u0120accurately": 14351, "\u0120acute": 14352, "\u0120Hann": 14353, "\u0120symbols": 14354, "\u0120civilization": 14355, "\u0120AW": 14356, "\u0120lightning": 14357, "\u0120considers": 14358, "\u0120venue": 14359, "\u0120\u00d7": 14360, "\u0120oven": 14361, "\u0120SF": 14362, "his": 14363, "\u0120nu": 14364, "\u0120Learn": 14365, "\u0120peoples": 14366, "\u0120std": 14367, "\u0120slee": 14368, "\u0120slic": 14369, "\u0120Statistics": 14370, "\u0120corners": 14371, "\u0120Baker": 14372, "\u0120:)": 14373, "mentation": 14374, "olver": 14375, "\u0120laughing": 14376, "\u0120Todd": 14377, "onde": 14378, "\u0120Hills": 14379, "\u0120nuts": 14380, "\u0120Woman": 14381, "plane": 14382, "\u0120liver": 14383, "\u0120Inside": 14384, "Sorry": 14385, "\u0120agrees": 14386, "\u0120fundament": 14387, "\u0120Fisher": 14388, "\u0120auction": 14389, "\u0120threads": 14390, "glas": 14391, "\u0120Basic": 14392, "\u0120Nat": 14393, "\u0120lacking": 14394, "\u0120celebration": 14395, "ju": 14396, "\u0120silly": 14397, "Euro": 14398, "\u0120tatt": 14399, "ighty": 14400, "controlled": 14401, "Test": 14402, "\u0120Singh": 14403, "\u0120rage": 14404, "\u0120rhyth": 14405, "offic": 14406, "\u0120Phantom": 14407, "\u0120headlines": 14408, "\u0120responding": 14409, "\u0120Morning": 14410, "\u0120vitamin": 14411, "\u0120boots": 14412, "\u0120Site": 14413, "alin": 14414, "pi": 14415, "\u0120viral": 14416, "\u0120UC": 14417, "DER": 14418, "\u0120Sex": 14419, "\u0120stocks": 14420, "current": 14421, "\u0120churches": 14422, "\u0120Rare": 14423, "\u0120Murphy": 14424, "\u0120denial": 14425, "\u0120Gaming": 14426, "\u0120toug": 14427, "\u0120nick": 14428, "\u0120makers": 14429, "\u0120Ronald": 14430, "\u0120generous": 14431, "\u0120Doc": 14432, "\u0120Morris": 14433, "\u0120transformed": 14434, "\u0120Normal": 14435, "\u0120104": 14436, "\u0120Kickstarter": 14437, "\u0120Upon": 14438, "Online": 14439, "\u0120IRS": 14440, "\u0120wrap": 14441, "\u0120loving": 14442, "\u0120arrives": 14443, "\u0120Due": 14444, "\u0120heter": 14445, "\u0120Made": 14446, "\u0120rental": 14447, "\u0120belongs": 14448, "\u0120attorneys": 14449, "\u0120crops": 14450, "\u0120matched": 14451, "ulum": 14452, "oline": 14453, "109": 14454, "\u0120dispar": 14455, "\u0120buyers": 14456, "\u0120Cambridge": 14457, "\u0120ethics": 14458, "roups": 14459, "\u0120justified": 14460, "\u0120marginal": 14461, "\u0120respected": 14462, "winning": 14463, "\u0120nodded": 14464, "\u0120Serge": 14465, "\u0120Former": 14466, "Craft": 14467, "################": 14468, "\u0120Warner": 14469, "\u0120dash": 14470, "ete": 14471, "\u0120entert": 14472, "\u0120Escape": 14473, "outheast": 14474, "\u0120knees": 14475, "\u0120Bomb": 14476, "\u0120rug": 14477, "Pass": 14478, "\u0120attitudes": 14479, "government": 14480, "\u0120Prior": 14481, "\u0120qualities": 14482, "\u0120notification": 14483, "\u0120Phone": 14484, "lie": 14485, "\u0120anticipated": 14486, "\u0120Combat": 14487, "\u0120Barry": 14488, "\u01201982": 14489, "Users": 14490, "oner": 14491, "\u0120computing": 14492, "\u0120Connecticut": 14493, "\u0120lesser": 14494, "\u0120peers": 14495, "\u0120Cu": 14496, "\u0120technically": 14497, "\u0120submission": 14498, "\u0120Universal": 14499, "\u0120manually": 14500, "ourge": 14501, "\u0120respondents": 14502, "\u0120BTC": 14503, "\u0120Host": 14504, "\u0120fare": 14505, "\u0120Bird": 14506, "\u0120receipt": 14507, "also": 14508, "\u0120jack": 14509, "\u0120agriculture": 14510, "\u0120skull": 14511, "\u0120!=": 14512, "\u0120passive": 14513, "\u0120CI": 14514, "\u0120societies": 14515, "\u0120reminded": 14516, "\u0120interference": 14517, "Buy": 14518, "\u0120\u00e2\u013e": 14519, "gon": 14520, "\u0120scrutiny": 14521, "\u0120Witch": 14522, "\u0120conducting": 14523, "\u0120\u00e3\u0125": 14524, "\u0120exchanges": 14525, "\u0120Mitchell": 14526, "\u0120inhabit": 14527, "\u0120twist": 14528, "BD": 14529, "\u0120wherever": 14530, "groupon": 14531, "\u0120jokes": 14532, "\u0120Benjamin": 14533, "\u0120Random": 14534, "frame": 14535, "\u0120Lions": 14536, "\u0120highlighted": 14537, "\u0120Arkansas": 14538, "Ent": 14539, "\u0120pile": 14540, "\u0120prelim": 14541, "gs": 14542, "minded": 14543, "\u0120felony": 14544, "\u0120GA": 14545, "\u0120Luck": 14546, "\u0120practically": 14547, "\u0120Bos": 14548, "\u0120actress": 14549, "Dam": 14550, "\u0120Bou": 14551, "\u0120visa": 14552, "\u0120embedded": 14553, "\u0120hybrid": 14554, "\u0120earliest": 14555, "\u0120sooner": 14556, "social": 14557, "\u0120HA": 14558, "\u0120steep": 14559, "\u0120disadvant": 14560, "\u0120exploit": 14561, "\u0120Egg": 14562, "\u0120Ultra": 14563, "\u0120necessity": 14564, "Local": 14565, "iege": 14566, "\u0120dated": 14567, "\u0120masses": 14568, "\u0120subscription": 14569, "pless": 14570, "\u0120anonym": 14571, "\u0120presumably": 14572, "Blue": 14573, "Their": 14574, "asketball": 14575, "\u0120Philip": 14576, "\u0120comed": 14577, "loaded": 14578, "rane": 14579, "\u0120reflection": 14580, "China": 14581, "\u0120extends": 14582, "\u0120forming": 14583, "\u0120unders": 14584, "2001": 14585, "\u0120grat": 14586, "\u0120concentrations": 14587, "\u0120insulin": 14588, "\u0120secular": 14589, "\u0120whilst": 14590, "\u0120winners": 14591, "Advertisements": 14592, "\u0120deliberately": 14593, "\u0120Working": 14594, "\u0120sink": 14595, "etics": 14596, "dale": 14597, "\u0120mandate": 14598, "\u0120gram": 14599, "\u0120vacation": 14600, "\u0120warnings": 14601, "ripp": 14602, "\u0120THAT": 14603, "\u0120commentary": 14604, "\u0120intu": 14605, "\u0120aest": 14606, "\u0120reasoning": 14607, "\u0120breakdown": 14608, "\u0120Zombie": 14609, "\u0120-->": 14610, "\u0120Political": 14611, "cott": 14612, "\u0120thrust": 14613, "\u0120technological": 14614, "\u0120deciding": 14615, "\u0120trafficking": 14616, "Long": 14617, "Welcome": 14618, "prising": 14619, "\u0120Communications": 14620, "\u0120endors": 14621, "\u0120swift": 14622, "\u0120metabol": 14623, "coins": 14624, "resa": 14625, "\u0120HTTP": 14626, "\u0120enroll": 14627, "\u0120Happy": 14628, "usr": 14629, "intage": 14630, "\u0120[\"": 14631, "uably": 14632, "\u0120Material": 14633, "\u0120repeal": 14634, "Sept": 14635, "kh": 14636, "\u0120Modi": 14637, "\u0120underneath": 14638, "\u0120IL": 14639, "shore": 14640, "\u0120diagnosed": 14641, "aceutical": 14642, "\u0120shower": 14643, "aux": 14644, "\u0120Switch": 14645, "\u0120Strength": 14646, "\u0120jihad": 14647, "national": 14648, "\u0120trauma": 14649, "ussy": 14650, "oni": 14651, "\u0120consolid": 14652, "\u0120calories": 14653, "\u0120Flynn": 14654, "agged": 14655, "168": 14656, "\u0120Pink": 14657, "\u0120fulfill": 14658, "\u0120chains": 14659, "\u0120notably": 14660, "\u0120AV": 14661, "Life": 14662, "\u0120Chuck": 14663, "mus": 14664, "\u0120Urban": 14665, "\u0120Hend": 14666, "\u0120deposit": 14667, "\u0120Sad": 14668, "\u0120affair": 14669, "ORK": 14670, "ieval": 14671, "\u0120FDA": 14672, "\u0120trop": 14673, "\u0120Overall": 14674, "\u0120virtue": 14675, "\u0120satisfaction": 14676, "aund": 14677, "\u0120lun": 14678, "\u0120Switzerland": 14679, "\u0120Operation": 14680, "process": 14681, "\u0120shook": 14682, "\u0120counties": 14683, "leased": 14684, "\u0120Charlotte": 14685, "112": 14686, "\u0120transcript": 14687, "\u0120redd": 14688, "push": 14689, "\u0120Hey": 14690, "\u0120Analysis": 14691, "[\"": 14692, "\u0120alternatives": 14693, "ardless": 14694, "\u0120eleph": 14695, "\u0120prejud": 14696, "\u0120Leaf": 14697, "Having": 14698, "\u0120Hub": 14699, "\u0120expressions": 14700, "\u0120Volume": 14701, "\u0120shocking": 14702, "\u0120Reds": 14703, "\u0120readily": 14704, "\u0120planets": 14705, "adata": 14706, "\u0120collapsed": 14707, "\u0120Madrid": 14708, "\u0120irrit": 14709, "ipper": 14710, "\u0120Enc": 14711, "\u0120Wire": 14712, "\u0120buzz": 14713, "\u0120GP": 14714, "asha": 14715, "\u0120accidentally": 14716, "uru": 14717, "\u0120frustrated": 14718, "\u0120SA": 14719, "\u0120hungry": 14720, "\u0120Huff": 14721, "\u0120labels": 14722, "anto": 14723, "\u0120EP": 14724, "\u0120barriers": 14725, ")|": 14726, "\u0120Berkeley": 14727, "\u0120Jets": 14728, "\u0120pairs": 14729, "\u0120Lan": 14730, "James": 14731, "\u0120Bear": 14732, "\u0120humor": 14733, "\u0120Liberty": 14734, "\u0120magnitude": 14735, "\u0120aging": 14736, "\u0120Mason": 14737, "\u0120friendship": 14738, "umbling": 14739, "\u0120emerge": 14740, "\u0120newspapers": 14741, "\u0120ambitious": 14742, "\u0120Richards": 14743, "aternal": 14744, "\u01201981": 14745, "\u0120cookies": 14746, "\u0120sculpt": 14747, "\u0120pursuit": 14748, "Location": 14749, "\u0120scripts": 14750, "pc": 14751, "\u0120arrangements": 14752, "\u0120diameter": 14753, "\u0120loses": 14754, "amation": 14755, "\u0120liqu": 14756, "\u0120Jake": 14757, "arette": 14758, "\u0120understands": 14759, "\u0120Zen": 14760, "vm": 14761, "\u0120approve": 14762, "\u0120wip": 14763, "\u0120ultra": 14764, "\u0120intend": 14765, "\u0120DI": 14766, "ascular": 14767, "\u0120stays": 14768, "\u0120Kor": 14769, "\u0120Kl": 14770, "\u0120investing": 14771, "La": 14772, "\u0120believing": 14773, "bad": 14774, "mouth": 14775, "\u0120taxpayer": 14776, "\u00e3\u0125\u0125": 14777, "\u0120Quebec": 14778, "\u0120lap": 14779, "\u0120Swiss": 14780, "drop": 14781, "\u0120drain": 14782, "iri": 14783, "etc": 14784, "ften": 14785, "\u0120Nex": 14786, "\u0120straw": 14787, "\u0120screaming": 14788, "\u0120counted": 14789, "\u0120damaging": 14790, "\u0120ambassador": 14791, "century": 14792, "\u0120prox": 14793, "\u0120arrests": 14794, "uv": 14795, "ilateral": 14796, "\u0120Charg": 14797, "\u0120prescribed": 14798, "\u0120independently": 14799, "\u0120fierce": 14800, "\u0120Baby": 14801, "\u0120brave": 14802, "\u0120suits": 14803, "=>": 14804, "\u0120baseline": 14805, "\u0120Rate": 14806, "\u0120islands": 14807, "\u0120((": 14808, "green": 14809, "ixels": 14810, "\u0120namely": 14811, "\u0120Village": 14812, "than": 14813, "amy": 14814, "Version": 14815, "gmail": 14816, "entials": 14817, "\u0120Sud": 14818, "\u0120Melbourne": 14819, "\u0120arriving": 14820, "\u0120quantum": 14821, "eff": 14822, "ropolitan": 14823, "Tri": 14824, "\u0120funeral": 14825, "\u0120IR": 14826, "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 14827, "\u0120Cob": 14828, "itably": 14829, "\u0120turb": 14830, "\u0120combo": 14831, "Review": 14832, "\u0120deployment": 14833, "uity": 14834, "\u0120Bott": 14835, "\u0120invisible": 14836, "\u0120rendering": 14837, "\u0120unlocked": 14838, "\u0120aqu": 14839, "\u0120Vladimir": 14840, "\u0120pad": 14841, "\u0120Brain": 14842, "\u0120Legacy": 14843, "dragon": 14844, "\u0120Kurdish": 14845, "\u0120sounded": 14846, "\u0120detained": 14847, "\u0120DM": 14848, "gary": 14849, "\u0120daughters": 14850, "\u0120disturbing": 14851, "uka": 14852, "\u0120Parad": 14853, "\u0120tast": 14854, "\u0120unfortunate": 14855, "\u0120ul": 14856, "emin": 14857, "\u0120attendance": 14858, "trl": 14859, "\u0120parks": 14860, "\u0120Memorial": 14861, "\u0120Alice": 14862, "othy": 14863, "guard": 14864, "\u0120Dise": 14865, "\u0120Shan": 14866, "\u0120Forum": 14867, "Rich": 14868, "\u0120shifted": 14869, "uez": 14870, "\u0120lighter": 14871, "\u0120Magn": 14872, "\u0120cod": 14873, "Sch": 14874, "hammad": 14875, "Pub": 14876, "350": 14877, "\u0120Pokemon": 14878, "\u0120prototype": 14879, "\u0120unre": 14880, "Base": 14881, "\u0120Students": 14882, "\u0120Reply": 14883, "\u0120Communist": 14884, "\u0120gau": 14885, "\u0120Tyler": 14886, "IZ": 14887, "\u0120participated": 14888, "\u0120suprem": 14889, "\u0120Details": 14890, "\u0120vessels": 14891, "rod": 14892, "\u0120tribe": 14893, "keep": 14894, "\u0120assumptions": 14895, "\u0120pound": 14896, "\u0120crude": 14897, "\u0120Available": 14898, "\u0120swimming": 14899, "\u0120inclusion": 14900, "\u0120advances": 14901, "culation": 14902, "\u0120conservation": 14903, "\u0120overd": 14904, "\u0120Buffalo": 14905, "Article": 14906, "edge": 14907, "\u0120awa": 14908, "\u0120Madison": 14909, "\u0120sidew": 14910, "\u0120catast": 14911, "\u0120Krist": 14912, "ucle": 14913, "\u0120Highway": 14914, "\u0120Terror": 14915, "\u0120activation": 14916, "\u0120unconscious": 14917, "\u0120Satan": 14918, "\u0120Susan": 14919, "illery": 14920, "\u0120arranged": 14921, "iop": 14922, "\u0120rumors": 14923, "urring": 14924, "think": 14925, "\u0120Keith": 14926, "\u0120Kind": 14927, "\u0120avoiding": 14928, "byn": 14929, "nut": 14930, "\u0120Speaker": 14931, "rus": 14932, "names": 14933, "\u0120guilt": 14934, "\u0120Olympics": 14935, "\u0120sail": 14936, "\u0120Mes": 14937, "levant": 14938, "\u0120Columbus": 14939, "aft": 14940, "City": 14941, "South": 14942, "\u0120Harvey": 14943, "\u0120Pun": 14944, "Several": 14945, "\u0120mentally": 14946, "\u0120impress": 14947, "mount": 14948, "\u0120Ubuntu": 14949, "\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136": 14950, "\u0120Superman": 14951, "\u0120MPs": 14952, "\u0120intentions": 14953, "\u0120Racing": 14954, "\u0120likelihood": 14955, "\u0120240": 14956, "Total": 14957, "\u0120toys": 14958, "\u0120Watson": 14959, "\u0120urge": 14960, "Lear": 14961, "\u0120Paper": 14962, "\u0120occurring": 14963, "\u0120Beng": 14964, "\u0120Cert": 14965, "\u0120stones": 14966, "Tim": 14967, "\u0120Twin": 14968, "zb": 14969, "\u0120Dynam": 14970, "\u0120politician": 14971, "kens": 14972, "\u0120Enterprise": 14973, "UTERS": 14974, "\u0120abol": 14975, "\u0120refresh": 14976, "\u0120arbitrary": 14977, "pection": 14978, "\u0120troubles": 14979, "\u0120});": 14980, "tv": 14981, "\u0120pilots": 14982, "\u0120distribute": 14983, "\u0120audit": 14984, "\u0120pause": 14985, "original": 14986, "\u0120rivals": 14987, "\u00c2\u00a3": 14988, "Fig": 14989, "TL": 14990, "abil": 14991, "rying": 14992, "Lin": 14993, "ioned": 14994, "lon": 14995, "\u0120fancy": 14996, "\u0120crashed": 14997, "\u0120tract": 14998, "\u0120shed": 14999, "\u0120consume": 15000, "Based": 15001, "download": 15002, "init": 15003, "\u0120voltage": 15004, "Introdu": 15005, "\u0120condemned": 15006, "\u0120Finance": 15007, "respect": 15008, "\u0120excluded": 15009, "\u0120establishing": 15010, "heric": 15011, "\u0120heritage": 15012, "\u0120spectacular": 15013, "\u0120unst": 15014, "\u0120Snowden": 15015, "\u0120Lane": 15016, "San": 15017, "\u0120protections": 15018, "struction": 15019, "incinn": 15020, "\u0120macro": 15021, "Custom": 15022, "iosity": 15023, "\u0120esp": 15024, "\u0120functioning": 15025, "\u0120mush": 15026, "\u0120puzzle": 15027, "\u0120ethical": 15028, "Mal": 15029, "\u0120governing": 15030, "\u0120Ferguson": 15031, "\u0120restored": 15032, "\u0120stressed": 15033, "\u0120Counter": 15034, "\u0120Kas": 15035, "clip": 15036, "ANS": 15037, "\u0120seiz": 15038, "UK": 15039, "byss": 15040, "oldown": 15041, "api": 15042, "\u0120permanently": 15043, "ounters": 15044, "West": 15045, "Through": 15046, "Light": 15047, "atoes": 15048, "\u0120neat": 15049, "\u0120cord": 15050, "urer": 15051, "\u0120severely": 15052, "\u0120Aven": 15053, "\u0120interrog": 15054, "\u0120triple": 15055, "Given": 15056, "Number": 15057, "\u0120arise": 15058, "\u0120sher": 15059, "plant": 15060, "\u0120flower": 15061, "\u0120Cou": 15062, "\u0120ate": 15063, "\u0120newer": 15064, "bul": 15065, "\u0120meanwhile": 15066, "\u0120Lair": 15067, "\u0120adjustment": 15068, "\u0120Copyright": 15069, "\u0120divers": 15070, "iological": 15071, "\u0120gamers": 15072, "oat": 15073, "\u0120historically": 15074, "\u0120analog": 15075, "\u0120longtime": 15076, "\u0120prescription": 15077, "\u0120Mist": 15078, "\u0120Hyper": 15079, "\u0120Maine": 15080, "\u0120Deity": 15081, "\u0120multipl": 15082, "\u0120Reincarn": 15083, "\u0120Hyd": 15084, "\u0120Pic": 15085, "Sil": 15086, "rants": 15087, "\u0120Cris": 15088, ".;": 15089, "({": 15090, "ependence": 15091, "\u0120recy": 15092, "ateur": 15093, "\u0120quad": 15094, "\u0120glob": 15095, "\u0120conced": 15096, "team": 15097, "\u0120capitalist": 15098, "\u0120Lot": 15099, "\u0120royal": 15100, "\u0120Cyber": 15101, "\u0120blacks": 15102, "metic": 15103, "riv": 15104, "\u0120Danny": 15105, "\u0120spo": 15106, "\u0120RO": 15107, "\u0120animated": 15108, "rypted": 15109, "\u0120Deputy": 15110, "\u0120rendered": 15111, "FE": 15112, "\u0120streak": 15113, "\u0120clouds": 15114, "\u0120Doug": 15115, "~~~~~~~~": 15116, "\u0120discour": 15117, "\u0120Veh": 15118, "\u0120psychology": 15119, "\u0120Journey": 15120, "\u0120crystal": 15121, "\u0120Frost": 15122, "\u0120suspicion": 15123, "\u0120relate": 15124, "orus": 15125, "\u0120Crypt": 15126, "\u0120NVIDIA": 15127, "comed": 15128, "uting": 15129, "incinnati": 15130, "\u0120vulnerability": 15131, "ostic": 15132, "\u0120isolation": 15133, "\u0120cooling": 15134, "\u0120Coalition": 15135, "\u0120119": 15136, "Four": 15137, "\u0120Deal": 15138, "\u0120\u00e2\u012b": 15139, "semble": 15140, "rament": 15141, "\u0120Barcelona": 15142, "\u0120102": 15143, "\u0120cocaine": 15144, "ocalypse": 15145, "Feb": 15146, "ogenic": 15147, "\u0120mutation": 15148, "\u0120cryptoc": 15149, "\u0120Kel": 15150, "\u0120Git": 15151, "ais": 15152, "\u0120sisters": 15153, "ANK": 15154, "\u0120activate": 15155, "Ter": 15156, "\u0120dread": 15157, "ylon": 15158, "\u0120propri": 15159, "Aust": 15160, "\u0120Default": 15161, "\u0120outdoor": 15162, "\u0120sheer": 15163, "ceive": 15164, "\u0120gently": 15165, "\u00d0\u00be": 15166, "Program": 15167, "\u0120\u00e2\u0128\u0134": 15168, "\u0120vegan": 15169, "\u0120Crus": 15170, "\u0120responsibilities": 15171, "\u0120HR": 15172, "OLD": 15173, "\u0120prevents": 15174, "\u0120stiff": 15175, "\u0120Were": 15176, "\u0120athletic": 15177, "\u0120Score": 15178, "\u0120):": 15179, "\u0120columns": 15180, "\u0120Loc": 15181, "available": 15182, "\u0120Fram": 15183, "\u0120Sessions": 15184, "\u0120companion": 15185, "\u0120packs": 15186, "140": 15187, "\u0120Knights": 15188, "\u0120fart": 15189, "\u0120streams": 15190, "\u0120shore": 15191, "\u0120appeals": 15192, "\u0120Performance": 15193, "haul": 15194, "\u0120Stra": 15195, "\u0120Nag": 15196, "103": 15197, "\u0120Transportation": 15198, "BB": 15199, "Ev": 15200, "zan": 15201, "Public": 15202, "\u0120twin": 15203, "ulsion": 15204, "Mult": 15205, "\u0120electro": 15206, "\u0120statue": 15207, "ationally": 15208, "\u0120Nort": 15209, "\u0120inspection": 15210, "/*": 15211, "igue": 15212, "\u0120compassion": 15213, "\u0120Tales": 15214, "\u0120Stein": 15215, "\u0120Screen": 15216, "\u0120Bug": 15217, "\u0120Lion": 15218, "girl": 15219, "\u0120withdrawal": 15220, "\u0120objectives": 15221, "\u0120bloody": 15222, "\u0120preliminary": 15223, "\u0120jacket": 15224, "\u0120dimensions": 15225, "\u0120Cool": 15226, "\u0120Occup": 15227, "\u0120wreck": 15228, "\u0120doubled": 15229, "anking": 15230, "\u01201975": 15231, "\u0120glasses": 15232, "\u0120Wang": 15233, "prov": 15234, "Path": 15235, "connected": 15236, "\u0120Multi": 15237, "\u0120Norway": 15238, "agonist": 15239, "\u0120feared": 15240, "\u0120touching": 15241, "\u0120arguably": 15242, "\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af": 15243, "\u0120NCAA": 15244, "chem": 15245, "\u0120spat": 15246, "\u0120WWE": 15247, "\u0120Cel": 15248, "igger": 15249, "\u0120attacker": 15250, "\u0120Join": 15251, "object": 15252, "etta": 15253, "\u0120eliminated": 15254, "det": 15255, "\u0120destruct": 15256, "\u0120Lucas": 15257, "ctuary": 15258, "180": 15259, "\u0120Brady": 15260, "\u0120Blues": 15261, "Bay": 15262, "aukee": 15263, "\u0120timeline": 15264, "\u0120delegates": 15265, "written": 15266, "ufficient": 15267, "\u0120shapes": 15268, "Copyright": 15269, "ouble": 15270, "service": 15271, "\u0120pione": 15272, "\u0120colleges": 15273, "\u0120rows": 15274, "\u0120spite": 15275, "\u0120assessed": 15276, "360": 15277, "\u0120lease": 15278, "\u0120confidential": 15279, "cker": 15280, "\u0120Manning": 15281, "\u0120Voice": 15282, "\u0120sealed": 15283, "\u0120calculate": 15284, "NO": 15285, "\u0120Assistant": 15286, "\u0120teenager": 15287, "ulent": 15288, "atherine": 15289, "\u0120mock": 15290, "\u0120diamond": 15291, "\u0120fest": 15292, "\u0120switched": 15293, "\u0120resume": 15294, "\u0120Puerto": 15295, "\u0120lanes": 15296, "iration": 15297, "\u0120Similarly": 15298, "\u0120rod": 15299, "\u0120Sel": 15300, "\u0120Palace": 15301, "\u0120Limited": 15302, "eous": 15303, "\u0120variant": 15304, "\u0120ward": 15305, "\u0120))": 15306, "Show": 15307, "OOK": 15308, "Alex": 15309, "\u0120Nep": 15310, "bris": 15311, "\u0120Wikipedia": 15312, "\u0120exceptional": 15313, "\u0120manages": 15314, "\u0120Draw": 15315, "Again": 15316, "\u0120copper": 15317, "utt": 15318, "\u0120exports": 15319, "\u0120portfolio": 15320, "\u0120elevated": 15321, "Rated": 15322, "\u0120Otherwise": 15323, "\u0120Tact": 15324, "\u0120Shel": 15325, "\u0120TX": 15326, "\"\u00e2\u0122\u0136": 15327, "\u0120resur": 15328, "\u0120Wa": 15329, "venant": 15330, "\u0120monetary": 15331, "people": 15332, "Email": 15333, "\u0120fifty": 15334, "\u0120Sweet": 15335, "\u0120Malaysia": 15336, "\u0120confusing": 15337, "\u0120Rio": 15338, "uda": 15339, "utenant": 15340, "\");": 15341, "\u0120praised": 15342, "\u0120volumes": 15343, "turn": 15344, "\u0120mature": 15345, "\u0120nonprofit": 15346, "\u0120passionate": 15347, "\u0120Private": 15348, "\u0120103": 15349, "\u0120descend": 15350, "\u00e7\u00a5\u0140": 15351, "uffy": 15352, "headed": 15353, "Whether": 15354, "rien": 15355, "zech": 15356, "beit": 15357, "\u0120chrom": 15358, "\u0120McM": 15359, "\u0120dancing": 15360, "\u0120eleg": 15361, "\u0120Noticed": 15362, "115": 15363, "\u0120advocacy": 15364, "ENTS": 15365, "ambling": 15366, "\u0120Minor": 15367, "\u0120Finn": 15368, "\u0120priorities": 15369, "\u0120thereof": 15370, "\u0120Stage": 15371, "\u0120Rogers": 15372, "\u0120substitute": 15373, "\u0120Jar": 15374, "\u0120Jefferson": 15375, "\u0120lightly": 15376, "102": 15377, "\u0120Lisa": 15378, "uits": 15379, "ysical": 15380, "\u0120shifts": 15381, "\u0120drones": 15382, "\u0120workplace": 15383, "\u0120resid": 15384, "ensed": 15385, "ahn": 15386, "\u0120preferences": 15387, "server": 15388, "\u0120debates": 15389, "doc": 15390, "\u0120Gods": 15391, "\u0120helicopter": 15392, "\u0120honour": 15393, "\u0120considerably": 15394, "eded": 15395, "\u0120Female": 15396, "\u0120Anne": 15397, "\u0120reun": 15398, "\u0120Face": 15399, "\u0120Hallow": 15400, "\u0120Budget": 15401, "\u0120condemn": 15402, "\u0120tender": 15403, "Prof": 15404, "ocratic": 15405, "\u0120Turner": 15406, "\u0120Agric": 15407, "\u01201976": 15408, "\u0120apt": 15409, "disc": 15410, "\u0120Fighter": 15411, "\u0120Aur": 15412, "\u0120garbage": 15413, "input": 15414, "\u0120Karl": 15415, "\u0120Oliver": 15416, "\u0120Language": 15417, "kn": 15418, "Non": 15419, "\u0120Clar": 15420, "\u0120traditions": 15421, "\u0120advertisement": 15422, "\u0120Sor": 15423, "\u0120archive": 15424, "\u0120villages": 15425, "750": 15426, "\u0120implementing": 15427, "waukee": 15428, "\u0120dietary": 15429, "\u0120switching": 15430, "Republic": 15431, "\u0120velocity": 15432, "\u0120cit": 15433, "\u0120Awards": 15434, "\u0120financing": 15435, "\u0120lasted": 15436, ")]": 15437, "\u0120reminder": 15438, "Person": 15439, "\u0120precision": 15440, "\u0120designers": 15441, "\u0120Fried": 15442, "\u0120Border": 15443, "\u0120tragic": 15444, "\u0120wield": 15445, "\u0120initiatives": 15446, "\u0120Tank": 15447, "wer": 15448, "\u0120joins": 15449, "Ro": 15450, "inery": 15451, "\u0120arrow": 15452, "\u0120generating": 15453, "founder": 15454, "\u0120searches": 15455, "\u0120randomly": 15456, "Access": 15457, "\u0120batch": 15458, "\u0120posed": 15459, "lat": 15460, "\u0120pursuing": 15461, "asa": 15462, "\u0120testified": 15463, "forming": 15464, "\u0120Shar": 15465, "wiki": 15466, "\u0120Either": 15467, "Sometimes": 15468, "\u0120senators": 15469, "\u0120Johnny": 15470, "\u0120Taliban": 15471, "\u0120GPS": 15472, "\":\"/": 15473, "\u00e3\u0123\u00ae\u00e5": 15474, "\u0120analyzed": 15475, "\u0120Rubio": 15476, "\u0120Movement": 15477, "opard": 15478, "iii": 15479, "Stand": 15480, "fight": 15481, "\u0120ignoring": 15482, "iang": 15483, "\u0120GN": 15484, "soever": 15485, "\u0120STAT": 15486, "\u0120refusing": 15487, "\u0120sweat": 15488, "\u0120bay": 15489, "PORT": 15490, "irmed": 15491, "aky": 15492, "\u0120dispro": 15493, "\u0120labeled": 15494, "\u0120108": 15495, "Hello": 15496, "\u0120pleasant": 15497, "aba": 15498, "\u0120triumph": 15499, "\u0120aboard": 15500, "\u0120incom": 15501, "\u0120Crow": 15502, "lett": 15503, "\u0120folk": 15504, "\u0120chase": 15505, "``": 15506, "\u0120Brus": 15507, "\u0120teens": 15508, "cue": 15509, "\u0120terrain": 15510, "hyd": 15511, "ilight": 15512, "ORY": 15513, "Support": 15514, "ews": 15515, "lli": 15516, "raints": 15517, "\u0120Cand": 15518, "\u0120abused": 15519, "achment": 15520, "larg": 15521, "Bas": 15522, "\u0120Cancer": 15523, "\u01201978": 15524, "\u0120supporter": 15525, "access": 15526, "\u0120Termin": 15527, "\u0120Tampa": 15528, "\u0120ANY": 15529, "\u0120newest": 15530, "\u0120Criminal": 15531, "edu": 15532, "\u01201930": 15533, "\u0120admits": 15534, "\u0120ende": 15535, "\u0120failures": 15536, "urate": 15537, "fulness": 15538, "cycl": 15539, "\u0120Subject": 15540, "\u0120infinite": 15541, "three": 15542, "WA": 15543, "pit": 15544, "\u0120Install": 15545, "Rad": 15546, "iliation": 15547, "GM": 15548, "\u0120continent": 15549, "\u0120accommodate": 15550, "\u0120Clay": 15551, "\u0120pup": 15552, "\u0120Function": 15553, "\u0120hammer": 15554, "\u0120Alberta": 15555, "\u0120revised": 15556, "\u0120minorities": 15557, "\u0120measurement": 15558, "Connell": 15559, "\u0120disable": 15560, "\u0120Mix": 15561, "Incre": 15562, "\u0120fork": 15563, "\u0120Rosen": 15564, "\u0120implies": 15565, "umblr": 15566, "ANG": 15567, "\u0120proteins": 15568, "\u0120aggression": 15569, "\u0120facilitate": 15570, "SN": 15571, "\u0120illegally": 15572, "uer": 15573, "\u0120academ": 15574, "\u0120puzz": 15575, "\u0120Shift": 15576, "pay": 15577, "ollo": 15578, "\u0120audiences": 15579, "Build": 15580, "\u0120noble": 15581, "\u0120syntax": 15582, "\u00e2\u013a\u0127": 15583, "\u0120beam": 15584, "\u0120Bed": 15585, "\u0120Ald": 15586, "\u0120origins": 15587, "video": 15588, "\u01201977": 15589, "\u0120Assault": 15590, "\u0120garage": 15591, "Team": 15592, "\u0120verdict": 15593, "\u0120dwar": 15594, "\u0120Virtual": 15595, "event": 15596, "Keep": 15597, "\u0120sentiment": 15598, "\u0120wildlife": 15599, "shirt": 15600, "\u0120burg": 15601, "\u0120recommendation": 15602, "represent": 15603, "\u0120gallery": 15604, "owners": 15605, "\u0120scholar": 15606, "\u0120convenience": 15607, "\u0120Swift": 15608, "\u0120convinc": 15609, "Cap": 15610, "\u0120warfare": 15611, "\u0120Visual": 15612, "\u0120constitute": 15613, "\u0120abort": 15614, "\u0120Weather": 15615, "\u0120Looking": 15616, "\u0120Hem": 15617, "\u0120martial": 15618, "\u0120incoming": 15619, "etition": 15620, "\u0120tolerance": 15621, "\u0120Created": 15622, "\u0120flows": 15623, "\u0120Elder": 15624, "\u0120souls": 15625, "\u0120foul": 15626, "\u0120Pain": 15627, "\u0120CAN": 15628, "\u0120220": 15629, "bc": 15630, "hend": 15631, "\u0120genius": 15632, "Real": 15633, "\u0120Wr": 15634, "ometer": 15635, "pad": 15636, "\u0120limiting": 15637, "\u0120Si": 15638, "\u0120Lore": 15639, "\u0120Adventures": 15640, "\u0120varied": 15641, "Disc": 15642, "fin": 15643, "\u0120Personal": 15644, "Chris": 15645, "\u0120invented": 15646, "\u0120dive": 15647, "\u0120Rise": 15648, "\u0120oz": 15649, "\u0120Comics": 15650, "\u0120expose": 15651, "\u0120Reb": 15652, "letters": 15653, "site": 15654, "imated": 15655, "\u0120hacking": 15656, "\u0120educated": 15657, "\u0120Nobody": 15658, "\u0120depri": 15659, "\u0120incentive": 15660, "\u00e3\u0124\u00b7": 15661, "\u0120oversight": 15662, "\u0120tribes": 15663, "\u0120Belgium": 15664, "\u0120licensing": 15665, "ourt": 15666, "Product": 15667, "ahl": 15668, "\u0120Gem": 15669, "\u0120specialist": 15670, "\u0120cra": 15671, "anners": 15672, "\u0120Corbyn": 15673, "\u01201973": 15674, "READ": 15675, "\u0120summar": 15676, "\u0120overlook": 15677, "\u0120Application": 15678, "\u0120inappropriate": 15679, "\u0120downloaded": 15680, "Que": 15681, "\u0120Bears": 15682, "\u0120thumb": 15683, "\u0120Character": 15684, "\u0120Reincarnated": 15685, "\u0120Sid": 15686, "\u0120demonstrates": 15687, "sky": 15688, "\u0120Bloomberg": 15689, "\u0120Array": 15690, "\u0120Results": 15691, "\u0120Fourth": 15692, "\u0120EDT": 15693, "\u0120Oscar": 15694, "cend": 15695, "\u0120106": 15696, "\u0120NULL": 15697, "\u0120HERE": 15698, "match": 15699, "\u0120Brun": 15700, "\u0120glucose": 15701, "ieg": 15702, "egu": 15703, "\u0120certified": 15704, "\u0120relie": 15705, "\u0120humanitarian": 15706, "\u0120prayers": 15707, "King": 15708, "\u0120nan": 15709, "hou": 15710, "108": 15711, "ulu": 15712, "\u0120renewable": 15713, "\u0120distinguish": 15714, "\u0120dense": 15715, "\u0120Vent": 15716, "\u0120Package": 15717, "\u0120Boss": 15718, "\u0120editors": 15719, "\u0120migr": 15720, "Tra": 15721, "\u0120Peters": 15722, "\u0120Arctic": 15723, "2004": 15724, "\u0120Cape": 15725, "\u0120locally": 15726, "\u0120lasting": 15727, "\u0120handy": 15728, ".).": 15729, "Pan": 15730, "\u0120RES": 15731, "Index": 15732, "\u0120tensions": 15733, "\u0120formerly": 15734, "\u0120ideological": 15735, "\u0120sensors": 15736, "\u0120dealers": 15737, "\u0120defines": 15738, "Sk": 15739, "\u0120proceeds": 15740, "\u0120proxy": 15741, "azines": 15742, "\u0120Bash": 15743, "\u0120Pad": 15744, "\u0120Craft": 15745, "ealous": 15746, "\u0120sheets": 15747, "ometry": 15748, "June": 15749, "clock": 15750, "TT": 15751, "\u0120Theatre": 15752, "\u0120Buzz": 15753, "\u0120chapters": 15754, "\u0120millenn": 15755, "\u0120dough": 15756, "\u0120Congressional": 15757, "\u0120imagined": 15758, "avior": 15759, "\u0120clinic": 15760, "\u01201945": 15761, "\u0120holder": 15762, "root": 15763, "olester": 15764, "\u0120restart": 15765, "BN": 15766, "\u0120Hamas": 15767, "\u0120Job": 15768, "\u0120orb": 15769, "\u0120ram": 15770, "\u0120disclose": 15771, "\u0120translate": 15772, "\u0120immigrant": 15773, "\u0120annoying": 15774, "\u0120treaty": 15775, "anium": 15776, "\u0120Tea": 15777, "\u0120Legion": 15778, "\u0120crowds": 15779, "\u0120Bec": 15780, "\u0120Aer": 15781, "ohyd": 15782, "Bro": 15783, "Looking": 15784, "\u0120lbs": 15785, "\u0120aggress": 15786, "\u0120seam": 15787, "\u0120intercept": 15788, "\u0120MI": 15789, "mercial": 15790, "activ": 15791, "\u0120Cit": 15792, "\u0120dimension": 15793, "\u0120consistency": 15794, "\u0120rushing": 15795, "\u0120Douglas": 15796, "\u0120trim": 15797, "Install": 15798, "icker": 15799, "\u0120shy": 15800, "106": 15801, "\u0120mentions": 15802, "pelled": 15803, "\u0120Tak": 15804, "cost": 15805, "\u0120classroom": 15806, "\u0120fortune": 15807, "driven": 15808, "\u0120unle": 15809, "\u0120Wheel": 15810, "\u0120investor": 15811, "\u0120Masters": 15812, "kit": 15813, "\u0120associations": 15814, "\u0120Evolution": 15815, "oping": 15816, "uscript": 15817, "\u0120provincial": 15818, "\u0120Walter": 15819, "avi": 15820, "SO": 15821, "\u0120unlimited": 15822, "English": 15823, "\u0120Cards": 15824, "\u0120Ebola": 15825, "nered": 15826, "\u0120revenge": 15827, "\u0120outright": 15828, "umper": 15829, "\u0120fitting": 15830, "\u0120Solid": 15831, "\u0120formally": 15832, "\u0120problematic": 15833, "\u0120hazard": 15834, "\u0120encryption": 15835, "\u0120straightforward": 15836, "\u0120AK": 15837, "\u0120pse": 15838, "\u0120Orb": 15839, "\u0120Chamber": 15840, "\u0120Mak": 15841, "Contents": 15842, "\u0120loyalty": 15843, "\u0120lyrics": 15844, "\u0120Sym": 15845, "\u0120welcomed": 15846, "\u0120cooked": 15847, "\u0120monop": 15848, "\u0120nurse": 15849, "\u0120misleading": 15850, "\u0120eternal": 15851, "\u0120shifting": 15852, "\u0120+=": 15853, "Vis": 15854, "\u0120institutional": 15855, "illary": 15856, "\u0120pant": 15857, "VERT": 15858, "\u0120ACC": 15859, "\u0120Enh": 15860, "\u0120incon": 15861, "\u0120REUTERS": 15862, "\u0120donated": 15863, "\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6": 15864, "Intern": 15865, "\u0120exhibit": 15866, "\u0120tire": 15867, "\u0120Ric": 15868, "\u0120Champion": 15869, "\u0120Muhammad": 15870, "NING": 15871, "\u0120Soccer": 15872, "\u0120mobility": 15873, "\u0120varying": 15874, "\u0120Movie": 15875, "\u0120lord": 15876, "oak": 15877, "Field": 15878, "\u0120vector": 15879, "usions": 15880, "\u0120scrap": 15881, "\u0120enabling": 15882, "make": 15883, "Tor": 15884, ".*": 15885, "||": 15886, "\u0120Website": 15887, "\u0120NPC": 15888, "\u0120socialist": 15889, "\u0120Billy": 15890, "\u0120Additional": 15891, "\u0120cargo": 15892, "\u0120farms": 15893, "\u0120Soon": 15894, "\u0120Prize": 15895, "\u0120midnight": 15896, "\u0120900": 15897, "seen": 15898, "\u0120Spot": 15899, "\u0120sheep": 15900, "\u0120sponsored": 15901, "\u0120Hi": 15902, "\u0120Jump": 15903, "\u01201967": 15904, "Microsoft": 15905, "\u0120Agent": 15906, "\u0120charts": 15907, "dir": 15908, "\u0120adjacent": 15909, "\u0120tricks": 15910, "\u0120manga": 15911, "\u0120exagger": 15912, "/>": 15913, "football": 15914, "\u0120FCC": 15915, "GC": 15916, "\u0120Tier": 15917, "andra": 15918, "OUND": 15919, "%),": 15920, "\u0120fruits": 15921, "VC": 15922, "\u0120AA": 15923, "Rober": 15924, "\u0120midst": 15925, "\u00e2\u0139": 15926, "anka": 15927, "\u0120legislature": 15928, "\u0120Neil": 15929, "\u0120tourists": 15930, "\"\"": 15931, "\u0120Warning": 15932, "\u0120Nevertheless": 15933, "\u0120Official": 15934, "\u0120Whatever": 15935, "\u0120mold": 15936, "\u0120drafted": 15937, "\u0120substances": 15938, "\u0120breed": 15939, "\u0120tags": 15940, "\u0120Task": 15941, "\u0120verb": 15942, "\u0120manufactured": 15943, "comments": 15944, "\u0120Polish": 15945, "Prov": 15946, "\u0120determines": 15947, "Obama": 15948, "kers": 15949, "\u0120utterly": 15950, "\u0120sect": 15951, "sche": 15952, "\u0120Gates": 15953, "\u0120Chap": 15954, "\u0120aluminum": 15955, "\u0120zombie": 15956, "\u0120Touch": 15957, "\u0120UP": 15958, "\u0120satisfy": 15959, "\u0120predomin": 15960, "ascript": 15961, "\u0120elaborate": 15962, "\u01201968": 15963, "\u0120measuring": 15964, "\u0120Vari": 15965, "anyahu": 15966, "\u0120sir": 15967, "ulates": 15968, "idges": 15969, "ickets": 15970, "\u0120Spencer": 15971, "TM": 15972, "oubted": 15973, "\u0120prey": 15974, "\u0120installing": 15975, "\u0120Cab": 15976, "reed": 15977, "reated": 15978, "Supp": 15979, "\u0120wrist": 15980, "\u0120Kerry": 15981, "107": 15982, "\u0120Kle": 15983, "\u0120Rachel": 15984, "\u0120cotton": 15985, "\u0120ARE": 15986, "\u0120Ele": 15987, "Control": 15988, "\u0120loads": 15989, "\u0120Dod": 15990, "anas": 15991, "bone": 15992, "\u0120classical": 15993, "\u0120Regional": 15994, "\u0120Integ": 15995, "VM": 15996, "\u0120desires": 15997, "\u0120autism": 15998, "supported": 15999, "\u0120Message": 16000, "\u0120compact": 16001, "writer": 16002, "\u0120109": 16003, "\u0120Hurricane": 16004, "cision": 16005, "\u0120cycles": 16006, "\u0120drill": 16007, "\u0120colleague": 16008, "\u0120maker": 16009, "German": 16010, "\u0120mistaken": 16011, "Sun": 16012, "\u0120Gay": 16013, "\u0120whatsoever": 16014, "\u0120sells": 16015, "\u0120Airl": 16016, "liv": 16017, "\u0120Option": 16018, "\u0120solved": 16019, "\u0120sectors": 16020, "\u0120horizontal": 16021, "\u0120equation": 16022, "\u0120Skill": 16023, "\u0120Bio": 16024, "gement": 16025, "\u0120Snap": 16026, "\u0120Legal": 16027, "\u0120trademark": 16028, "\u0120makeup": 16029, "\u0120assembled": 16030, "\u0120saves": 16031, "\u0120Halloween": 16032, "\u0120Vermont": 16033, "\u0120FROM": 16034, "\u0120farming": 16035, "\u0120Podcast": 16036, "acceptable": 16037, "\u0120Higher": 16038, "\u0120asleep": 16039, "ullivan": 16040, "\u0120referen": 16041, "\u0120Lev": 16042, "\u0120bullets": 16043, "oko": 16044, "HC": 16045, "\u0120stairs": 16046, "\u0120maintains": 16047, "\u0120Lower": 16048, "\u0120Vi": 16049, "\u0120marine": 16050, "\u0120acres": 16051, "\u0120coordinator": 16052, "\u0120Joh": 16053, "\u0120counterparts": 16054, "\u0120Brothers": 16055, "\u0120indict": 16056, "bra": 16057, "\u0120chunk": 16058, "\u0120cents": 16059, "Home": 16060, "\u0120Month": 16061, "\u0120accordingly": 16062, "ifles": 16063, "\u0120Germans": 16064, "\u0120Syn": 16065, "Hub": 16066, "\u0120eyeb": 16067, "\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122": 16068, "\u0120ranges": 16069, "\u0120Holland": 16070, "\u0120Robot": 16071, "fc": 16072, "Mike": 16073, "\u0120plasma": 16074, "\u0120swap": 16075, "\u0120athlete": 16076, "\u0120Rams": 16077, ",'\"": 16078, "\u0120infections": 16079, "\u0120corrid": 16080, "\u0120vib": 16081, "\u0120patches": 16082, "\u0120traditionally": 16083, "\u0120revelation": 16084, "\u0120sweep": 16085, "\u0120glance": 16086, "\u0120inex": 16087, "2003": 16088, "\u0120Raw": 16089, "working": 16090, "osures": 16091, "\u0120Dat": 16092, "\u0120Lynch": 16093, "\u0120leverage": 16094, "\u0120Reid": 16095, "\u0120correlation": 16096, "iances": 16097, "avascript": 16098, "\u0120repository": 16099, "retty": 16100, "\u01201972": 16101, "240": 16102, "\u0120oun": 16103, "pol": 16104, "\u0120Reed": 16105, "\u0120tactical": 16106, "isite": 16107, "Apple": 16108, "\u0120Quinn": 16109, "\u0120raped": 16110, "illo": 16111, "Europe": 16112, "\u0120algorithms": 16113, "\u0120Rodrig": 16114, "iu": 16115, "\u0120illum": 16116, "\u0120fame": 16117, "\u0120introducing": 16118, "\u0120delays": 16119, "\u0120Raiders": 16120, "\u0120whistle": 16121, "\u0120novels": 16122, "\u0120Really": 16123, "\u0120deriv": 16124, "\u0120publications": 16125, "\u0120Neither": 16126, "\u0120Commerce": 16127, "\u0120aston": 16128, "language": 16129, "Notes": 16130, "\u0120Roth": 16131, "\u0120Fear": 16132, "\u0120mate": 16133, "\u0120parade": 16134, "\u0120QB": 16135, "\u0120maneu": 16136, "\u0120Cincinnati": 16137, "mitting": 16138, "\u0120waist": 16139, "\u0120Rew": 16140, "\u0120discont": 16141, "\u00d0\u00b0": 16142, "\u0120staring": 16143, "\u0120alias": 16144, "\u0120securities": 16145, "\u0120toilet": 16146, "\u0120Jedi": 16147, "\u0120unlaw": 16148, "vised": 16149, "////////": 16150, "](": 16151, "\u0120Weiss": 16152, "\u0120prest": 16153, "\u0120Compan": 16154, "\u0120memo": 16155, "\u0120Grace": 16156, "July": 16157, "\u0120Elite": 16158, "center": 16159, "\u0120Stay": 16160, "\u0120galaxy": 16161, "\u0120tooth": 16162, "\u0120Settings": 16163, "\u0120subjected": 16164, "\u00e3\u0124\u00a6": 16165, "\u0120lineback": 16166, "\u0120retailers": 16167, "\u0120Want": 16168, "\u0120dangers": 16169, "Air": 16170, "\u0120voluntary": 16171, "eway": 16172, "\u0120interpreted": 16173, "otine": 16174, "\u00c3\u00a7": 16175, "\u0120pel": 16176, "Service": 16177, "\u0120Eventually": 16178, "\u0120careers": 16179, "\u0120threaten": 16180, "\u0120memor": 16181, "\u0120Bradley": 16182, "ancies": 16183, "sn": 16184, "\u0120Unknown": 16185, "National": 16186, "\u0120shadows": 16187, "ailand": 16188, "\u0120Dash": 16189, "Everyone": 16190, "izzard": 16191, "March": 16192, "=(": 16193, "\u0120pulls": 16194, "\u0120stranger": 16195, "\u0120backwards": 16196, "\u0120Bernard": 16197, "imensional": 16198, "\u0120chron": 16199, "\u0120theoretical": 16200, "ktop": 16201, "\u0120ware": 16202, "\u0120Investig": 16203, "\u0120Initi": 16204, "\u0120Operations": 16205, "oven": 16206, "ocide": 16207, "*/": 16208, "\u0120flames": 16209, "\u0120Cash": 16210, "shit": 16211, "\u0120cab": 16212, "\u0120Analy": 16213, "\u0120Seah": 16214, "\u0120defining": 16215, "\u0120ordering": 16216, "\u0120immun": 16217, "\u0120persistent": 16218, "ACH": 16219, "Russian": 16220, "mans": 16221, "\u0120hind": 16222, "\u0120photography": 16223, "\u00c2\u00a9": 16224, "\u0120hug": 16225, "\u0120107": 16226, "\u0120Hence": 16227, "iots": 16228, "udeau": 16229, "\u0120subsidies": 16230, "\u0120routinely": 16231, "\u0120Device": 16232, "itic": 16233, "\u0120disgust": 16234, "lander": 16235, "\u01201940": 16236, "\u0120assignment": 16237, "\u0120Besides": 16238, "wick": 16239, "\u0120Dust": 16240, "usc": 16241, "structed": 16242, "111": 16243, "develop": 16244, "\u0120fond": 16245, "\u0120intersection": 16246, "\u0120dignity": 16247, "\u0120commissioner": 16248, "Without": 16249, "reach": 16250, "\u0120cartoon": 16251, "\u0120scales": 16252, "\u00e3\u0125\u0143": 16253, "FIG": 16254, "\u0120surveys": 16255, "\u0120Indonesia": 16256, "\u0120artwork": 16257, "\u0120unch": 16258, "\u0120cycling": 16259, "unct": 16260, "auer": 16261, "orate": 16262, "\u0120Obviously": 16263, "\u0120characterized": 16264, "feld": 16265, "\u0120affirm": 16266, "\u0120innings": 16267, "\u0120\u00e9": 16268, "\u0120aliens": 16269, "\u0120cloth": 16270, "etooth": 16271, "\u0120Certain": 16272, "\u00c2\u00a7": 16273, "\u0120digest": 16274, "know": 16275, "\u0120XL": 16276, "\u0120predictions": 16277, "\u0120din": 16278, "WAR": 16279, "\u0120aftermath": 16280, "Example": 16281, "\u0120Success": 16282, "\u0120Thr": 16283, "IGN": 16284, "\u0120miner": 16285, "Bus": 16286, "\u0120clarity": 16287, "heimer": 16288, "\u0120OUT": 16289, "\u0120Send": 16290, "\u0120Circle": 16291, "\u0120Diet": 16292, "\u0120pronounced": 16293, "\u0120creators": 16294, "\u0120earthquake": 16295, "attery": 16296, "geons": 16297, "\u0120od": 16298, "\u0120laying": 16299, "orp": 16300, "Ult": 16301, "project": 16302, "\u0120undermin": 16303, "\u0120sequel": 16304, "Sam": 16305, "\u0120Darkness": 16306, "\u0120reception": 16307, "bull": 16308, "YS": 16309, "\u0120Vir": 16310, "\u0120sequences": 16311, "\u0120Coin": 16312, "\u0120outfit": 16313, "\u0120Wait": 16314, "119": 16315, "\u0120delivers": 16316, "......": 16317, "\u0120blown": 16318, "\u0120Esc": 16319, "\u0120Math": 16320, "perm": 16321, "\u0120Ul": 16322, "\u0120glim": 16323, "\u0120facial": 16324, "\u0120greenhouse": 16325, "\u0120tokens": 16326, "/-": 16327, "\u0120Annual": 16328, "\u0120ONE": 16329, "\u0120teenage": 16330, "\u0120Physical": 16331, "\u0120Lang": 16332, "\u0120Celt": 16333, "\u0120sued": 16334, "ividually": 16335, "\u0120patience": 16336, "chair": 16337, "regular": 16338, "\u0120aug": 16339, "inv": 16340, "except": 16341, "\u0120Lil": 16342, "\u0120nest": 16343, "fd": 16344, "sum": 16345, "\u0120Chase": 16346, "Russia": 16347, "\u0120Jennifer": 16348, "\u0120offseason": 16349, "Overall": 16350, "Fore": 16351, "\u0120riot": 16352, "Aud": 16353, "former": 16354, "\u0120defenders": 16355, "\u0120CT": 16356, "iotic": 16357, "ribly": 16358, "\u0120automated": 16359, "\u0120penis": 16360, "\u0120insist": 16361, "\u0120diagram": 16362, "\u0120SQL": 16363, "\u0120Garc": 16364, "\u0120witch": 16365, "client": 16366, "ierra": 16367, "ambers": 16368, "\u0120recount": 16369, "far": 16370, "Very": 16371, "osterone": 16372, "\u0120appreciated": 16373, "\u0120Perfect": 16374, "Section": 16375, "\u0120doses": 16376, "ocaust": 16377, "\u0120costly": 16378, "\u0120grams": 16379, "\u0120Shi": 16380, "\u0120wrestling": 16381, "\u01201971": 16382, "\u0120trophy": 16383, "\u0120nerve": 16384, "\u0120Kaz": 16385, "\u0120Experience": 16386, "\u0120pledged": 16387, "\u0120playback": 16388, "\u0120creativity": 16389, "bye": 16390, "\u0120attackers": 16391, "\u0120holders": 16392, "\u0120Coach": 16393, "\u0120PhD": 16394, "\u0120transfers": 16395, "\u0120colored": 16396, "\u0120Hindu": 16397, "\u0120drown": 16398, "\u0120listened": 16399, "\u0120WA": 16400, "iasm": 16401, "PO": 16402, "\u0120appealing": 16403, "\u0120disclosed": 16404, "\u0120Chicken": 16405, "agging": 16406, "\u0120pleaded": 16407, "\u0120navigation": 16408, "\u0120Returns": 16409, "\u0120[[": 16410, "ROR": 16411, "EA": 16412, "\u0120photographer": 16413, "\u0120Rider": 16414, "ippers": 16415, "\u0120slice": 16416, "\u0120erect": 16417, "\u0120hed": 16418, "issance": 16419, "\u0120Vikings": 16420, "urious": 16421, "\u0120appet": 16422, "oubtedly": 16423, "Child": 16424, "\u0120authentic": 16425, "oos": 16426, "\u0120Making": 16427, "\u0120announcing": 16428, "\u0120bod": 16429, "\u0120meter": 16430, "\u0120Nine": 16431, "\u0120Rogue": 16432, "\u0120workforce": 16433, "\u0120renewed": 16434, "\u0120organisations": 16435, "acs": 16436, "PLE": 16437, "Short": 16438, "\u0120compounds": 16439, "\u0120Visit": 16440, "\u0120envelop": 16441, "earth": 16442, "\u0120supportive": 16443, "ggle": 16444, "\u0120Brussels": 16445, "\u0120Guild": 16446, "Create": 16447, "REL": 16448, "\u0120averaged": 16449, "\u01201969": 16450, "riages": 16451, "\u0120lengthy": 16452, "\u0120forgot": 16453, "Okay": 16454, "\u0120Erd": 16455, "\u0120dealer": 16456, "\u0120recession": 16457, "DD": 16458, "\u0120desperately": 16459, "\u0120hunger": 16460, "\u0120sticks": 16461, "\u0120mph": 16462, "\u0120Faith": 16463, "\u0120intentionally": 16464, "\u0120demol": 16465, "ueller": 16466, "\u0120Sale": 16467, "\u0120debris": 16468, "spring": 16469, "\u0120leap": 16470, ">>>>": 16471, "\u0120containers": 16472, "selling": 16473, "ranean": 16474, "attering": 16475, "\u0120commented": 16476, "\u0120CM": 16477, "onut": 16478, "\u0120woods": 16479, "especially": 16480, "\u0120organize": 16481, "ivic": 16482, "\u0120Woods": 16483, "anga": 16484, "squ": 16485, "\u0120maj": 16486, "amon": 16487, "\u0120axis": 16488, "\u01201974": 16489, "\u0120Denmark": 16490, "\u0120warrior": 16491, "\u0120Pand": 16492, "\u0120outlined": 16493, "\u0120BO": 16494, "insula": 16495, "zilla": 16496, "ebook": 16497, "\u0120dare": 16498, "\u0120searched": 16499, "\u0120navigate": 16500, "Sn": 16501, "writing": 16502, "\u0120united": 16503, "Japan": 16504, "\u0120Hebrew": 16505, "\u0120flame": 16506, "\u0120relies": 16507, "\u0120catching": 16508, "\u0120Sho": 16509, "\u0120imprisonment": 16510, "\u0120pockets": 16511, "\u0120closure": 16512, "\u0120Fam": 16513, "tim": 16514, "adequ": 16515, "Activity": 16516, "\u0120recruiting": 16517, "\u0120WATCH": 16518, "\u0120Argentina": 16519, "dest": 16520, "\u0120apologize": 16521, "oro": 16522, "\u0120lacks": 16523, "\u0120tuned": 16524, "\u0120Griffin": 16525, "\u0120infamous": 16526, "\u0120celebrity": 16527, "sson": 16528, "\u0120----------------------------------------------------------------": 16529, "\u0120Isis": 16530, "\u0120Display": 16531, "\u0120credibility": 16532, "\u0120economies": 16533, "\u0120headline": 16534, "\u0120Cowboys": 16535, "\u0120indef": 16536, "\u0120lately": 16537, "\u0120incentives": 16538, "button": 16539, "\u0120Mob": 16540, "Aut": 16541, "\u0120resigned": 16542, "\u0120Om": 16543, "camp": 16544, "\u0120profiles": 16545, "\u0120schemes": 16546, "olphins": 16547, "ayed": 16548, "Clinton": 16549, "enh": 16550, "\u0120Yahoo": 16551, "\u0120abst": 16552, "\u0120ank": 16553, "suits": 16554, "\u0120wished": 16555, "\u0120Marco": 16556, "udden": 16557, "\u0120sphere": 16558, "\u0120Bishop": 16559, "\u0120incorporated": 16560, "\u0120Plant": 16561, "114": 16562, "\u0120hated": 16563, "pic": 16564, "\u0120donate": 16565, "\u0120lined": 16566, "\u0120beans": 16567, "\u0120stealing": 16568, "\u0120costume": 16569, "\u0120sheriff": 16570, "\u0120forty": 16571, "\u0120intact": 16572, "\u0120adapted": 16573, "\u0120travelling": 16574, "bart": 16575, "\u0120nicely": 16576, "\u0120dried": 16577, "\u0120scal": 16578, "osity": 16579, "NOTE": 16580, "\u0120Bh": 16581, "\u0120Broncos": 16582, "\u0120Ign": 16583, "\u0120intimate": 16584, "\u0120chemistry": 16585, "\u0120optimal": 16586, "Deb": 16587, "\u0120Generation": 16588, "\u0120],": 16589, "ichi": 16590, "\u0120Wii": 16591, "\u0120YOUR": 16592, "ventions": 16593, "Write": 16594, "\u0120popul": 16595, "unning": 16596, "\u0120Wor": 16597, "Vol": 16598, "\u0120queen": 16599, "heads": 16600, "KK": 16601, "\u0120analyze": 16602, "opic": 16603, "earchers": 16604, "\u0120dot": 16605, "legraph": 16606, "astically": 16607, "\u0120upgrades": 16608, "\u0120cares": 16609, "\u0120extending": 16610, "\u0120freeze": 16611, "\u0120inability": 16612, "\u0120organs": 16613, "\u0120pretend": 16614, "\u0120outlet": 16615, "113": 16616, "olan": 16617, "\u0120Mall": 16618, "uling": 16619, "talk": 16620, "\u0120expressing": 16621, "\u0120Always": 16622, "\u0120Begin": 16623, "files": 16624, "\u0120licenses": 16625, "%%": 16626, "\u0120Mitt": 16627, "\u0120filters": 16628, "\u0120Milwaukee": 16629, "GN": 16630, "\u0120unfold": 16631, "Mo": 16632, "\u0120nutrition": 16633, "ppo": 16634, "Bo": 16635, "\u0120founding": 16636, "\u0120undermine": 16637, "\u0120easiest": 16638, "\u0120Czech": 16639, "\u0120Mack": 16640, "\u0120sexuality": 16641, "\u0120Nixon": 16642, "Win": 16643, "\u0120Arn": 16644, "\u0120Kin": 16645, "\u00e3\u0124\u00a3": 16646, "icer": 16647, "\u0120fortun": 16648, "\u0120surfaces": 16649, "aghd": 16650, "\u0120carriers": 16651, "\u0120PART": 16652, "\u0120Tib": 16653, "\u0120interval": 16654, "\u0120frustrating": 16655, "\u0120Ship": 16656, "\u0120Armed": 16657, "ffe": 16658, "\u0120boats": 16659, "\u0120Abraham": 16660, "inis": 16661, "\u0120suited": 16662, "thread": 16663, "iov": 16664, "abul": 16665, "\u0120Venezuela": 16666, "\u0120tom": 16667, "super": 16668, "\u0120castle": 16669, "although": 16670, "ioxide": 16671, "eches": 16672, "\u0120evolutionary": 16673, "\u0120negotiate": 16674, "\u0120confronted": 16675, "Remember": 16676, "\u0120170": 16677, "Such": 16678, "\u0120911": 16679, "mult": 16680, "\u0120Abyss": 16681, "urry": 16682, "kees": 16683, "spec": 16684, "\u0120Barbara": 16685, "\u0120belonging": 16686, "\u0120villain": 16687, "istani": 16688, "\u0120accountable": 16689, "\u0120portions": 16690, "\u0120Decl": 16691, "Ur": 16692, "\u0120Kate": 16693, "gre": 16694, "\u0120magazines": 16695, "UCK": 16696, "\u0120regulate": 16697, "omon": 16698, "\u0120Almost": 16699, "\u0120overview": 16700, "\u0120scram": 16701, "\u0120loot": 16702, "\u0120Fitz": 16703, "\u0120characteristic": 16704, "\u0120Snake": 16705, "say": 16706, "\u0120Rico": 16707, "\u0120trait": 16708, "\u0120Joined": 16709, "aucus": 16710, "\u0120adaptation": 16711, "\u0120Airlines": 16712, "\u0120archae": 16713, "\u0120Ide": 16714, "\u0120bikes": 16715, "\u0120literary": 16716, "\u0120influences": 16717, "\u0120Used": 16718, "Creat": 16719, "\u0120plea": 16720, "\u0120Defence": 16721, "\u0120Assass": 16722, "\u0120pond": 16723, "ULT": 16724, ")\"": 16725, "\u0120evaluated": 16726, "\u0120obtaining": 16727, "\u0120demographic": 16728, "\u0120vigil": 16729, "aley": 16730, "\u0120spouse": 16731, "\u0120Seahawks": 16732, "respons": 16733, "\u0120Belt": 16734, "umatic": 16735, "\u0120rises": 16736, "runner": 16737, "\u0120Michelle": 16738, "\u0120potent": 16739, "race": 16740, "\u0120PAC": 16741, "Find": 16742, "olesterol": 16743, "ISS": 16744, "\u0120Introduced": 16745, "resses": 16746, "ignment": 16747, "Os": 16748, "\u0120Tu": 16749, "\u0120Dex": 16750, "icides": 16751, "\u0120sparked": 16752, "\u0120Laura": 16753, "\u0120Bryant": 16754, "\u0120smiling": 16755, "\u0120Nexus": 16756, "\u0120defendants": 16757, "\u0120Catal": 16758, "\u0120dishes": 16759, "shaped": 16760, "\u0120prolong": 16761, "mt": 16762, "($": 16763, "\u00e3\u0122\u0124": 16764, "\u0120calculations": 16765, "\u0120Same": 16766, "\u0120piv": 16767, "HH": 16768, "\u0120cancelled": 16769, "\u0120grin": 16770, "\u0120territories": 16771, "istically": 16772, "Come": 16773, "\u0120Parent": 16774, "Project": 16775, "\u0120neglig": 16776, "\u0120Privacy": 16777, "\u0120ammo": 16778, "LECT": 16779, "olutely": 16780, "\u0120Epic": 16781, "\u0120misunder": 16782, "wal": 16783, "April": 16784, "mos": 16785, "pathy": 16786, "\u0120Carson": 16787, "\u0120albums": 16788, "\u0120Easy": 16789, "\u0120pistol": 16790, "<<": 16791, "\u0120\\(": 16792, "target": 16793, "help": 16794, "\u0120interpre": 16795, "conscious": 16796, "\u0120Housing": 16797, "\u0120Joint": 16798, "127": 16799, "\u0120beers": 16800, "science": 16801, "\u0120Firefox": 16802, "effective": 16803, "\u0120Cabin": 16804, "\u0120Okay": 16805, "\u0120Applic": 16806, "\u0120spacecraft": 16807, "\u0120SR": 16808, "vet": 16809, "\u0120Strange": 16810, "SB": 16811, "\u0120corps": 16812, "iberal": 16813, "efficient": 16814, "\u0120prevalence": 16815, "\u0120economists": 16816, "118": 16817, "Thread": 16818, "ordable": 16819, "ODE": 16820, "\u0120Cant": 16821, "=-=-": 16822, "ifiable": 16823, "\u0120Around": 16824, "\u0120pole": 16825, "\u0120willingness": 16826, "CLA": 16827, "\u0120Kid": 16828, "\u0120complement": 16829, "\u0120scattered": 16830, "\u0120inmates": 16831, "\u0120bleeding": 16832, "every": 16833, "\u0120queue": 16834, "\u0120Train": 16835, "\u0120hij": 16836, "\u0120melee": 16837, "pleted": 16838, "\u0120digit": 16839, "\u0120gem": 16840, "official": 16841, "\u0120lifting": 16842, "\u00d0\u00b5": 16843, "Requ": 16844, "itutes": 16845, "\u0120packaging": 16846, "\u0120Workers": 16847, "hran": 16848, "\u0120Lebanon": 16849, "olesc": 16850, "\u0120punished": 16851, "\u0120Juan": 16852, "\u0120jam": 16853, "\u0120Document": 16854, "\u0120mapping": 16855, "icates": 16856, "\u0120inevitably": 16857, "\u0120vanilla": 16858, "\u0120Ton": 16859, "\u0120watches": 16860, "\u0120leagues": 16861, "\u0120initiated": 16862, "degree": 16863, "portion": 16864, "\u0120recalls": 16865, "\u0120ruin": 16866, "\u0120melt": 16867, "IAN": 16868, "\u0120hem": 16869, "Exp": 16870, "\u0120baking": 16871, "\u0120Colomb": 16872, "atible": 16873, "\u0120radius": 16874, "plug": 16875, "\u0120IF": 16876, "etically": 16877, "\u0120fict": 16878, "HER": 16879, "\u0120Tap": 16880, "atinum": 16881, "\u0120ink": 16882, "\u0120coh": 16883, "\u0120Wizard": 16884, "both": 16885, "tex": 16886, "\u0120spends": 16887, "\u0120Currently": 16888, "\u0120Pit": 16889, "\u0120neurons": 16890, "ignt": 16891, "\u0120rall": 16892, "\u0120buses": 16893, "building": 16894, "\u0120adjustments": 16895, "\u0120cried": 16896, "iblical": 16897, "atted": 16898, "\u0120Zion": 16899, "\u0120Matter": 16900, "\u0120meditation": 16901, "\u0120Dennis": 16902, "\u0120ours": 16903, "\u0120Tab": 16904, "\u0120rankings": 16905, "ortal": 16906, "\u0120advers": 16907, "\u0120surrender": 16908, "\u0120Gob": 16909, "cium": 16910, "omas": 16911, "imeter": 16912, "\u0120multiplayer": 16913, "\u0120heroin": 16914, "\u0120optimistic": 16915, "\u0120indicator": 16916, "\u0120Brig": 16917, "\u0120grocery": 16918, "\u0120applicant": 16919, "\u0120Rocket": 16920, "vid": 16921, "Exception": 16922, "pent": 16923, "\u0120organizing": 16924, "\u0120encounters": 16925, "\u0120TOD": 16926, "\u0120jewel": 16927, "Save": 16928, "\u0120Christie": 16929, "\u0120heating": 16930, "\u0120lazy": 16931, "\u0120CP": 16932, "\u0120cousin": 16933, "Config": 16934, "\u0120regener": 16935, "\u0120nearest": 16936, "\u0120achieving": 16937, "ENS": 16938, "throw": 16939, "\u0120Richmond": 16940, "antle": 16941, "2002": 16942, "\u0120anten": 16943, "bird": 16944, "133": 16945, "\u0120narc": 16946, "raint": 16947, "unny": 16948, "\u0120Hispanic": 16949, "ournaments": 16950, "\u0120prophe": 16951, "\u0120Thailand": 16952, "\u0120Ti": 16953, "\u0120injection": 16954, "\u0120inherit": 16955, "ravis": 16956, "\u0120medi": 16957, "\u0120whoever": 16958, "\u0120DEBUG": 16959, "GP": 16960, "\u0120Hud": 16961, "Card": 16962, "prom": 16963, "\u0120por": 16964, "\u0120overhead": 16965, "Law": 16966, "\u0120violate": 16967, "\u0120heated": 16968, "\u0120descriptions": 16969, "\u0120achievements": 16970, "\u0120Beer": 16971, "\u0120Quant": 16972, "Was": 16973, "\u0120eighth": 16974, "\u0120Iv": 16975, "\u0120specialized": 16976, "UPDATE": 16977, "\u0120Delta": 16978, "Pop": 16979, "Jul": 16980, "\u0120Ask": 16981, "ophy": 16982, "\u0120newsletters": 16983, "\u0120Tool": 16984, "\u0120gard": 16985, "\u0120Confeder": 16986, "\u0120GMT": 16987, "\u0120Abbott": 16988, "\u0120immunity": 16989, "\u0120VM": 16990, "Islam": 16991, "\u0120implicit": 16992, "wd": 16993, "\u01201944": 16994, "ravity": 16995, "ometric": 16996, "\u0120surviving": 16997, "urai": 16998, "\u0120Prison": 16999, "\u0120rust": 17000, "\u0120Sketch": 17001, "\u0120bees": 17002, "\u0120Theory": 17003, "\u0120merit": 17004, "Tex": 17005, "chat": 17006, "\u0120mim": 17007, "\u0120paste": 17008, "\u0120Koch": 17009, "\u0120ignorance": 17010, "\u0120Shoot": 17011, "\u0120basement": 17012, "United": 17013, "\u0120Advis": 17014, "height": 17015, "\u0120foster": 17016, "\u0120detain": 17017, "information": 17018, "\u0120neural": 17019, "';": 17020, "\u0120proves": 17021, "allery": 17022, "\u0120invitation": 17023, "umbers": 17024, "\u0120cattle": 17025, "\u0120bicycle": 17026, "zi": 17027, "\u0120consultant": 17028, "\u0120apology": 17029, "\u0120Tiger": 17030, "\u0120123": 17031, "999": 17032, "\u0120individually": 17033, "rt": 17034, "igion": 17035, "\u0120Brazilian": 17036, "\u0120disturb": 17037, "\u0120entrepreneurs": 17038, "\u0120forests": 17039, "cerpt": 17040, "plates": 17041, "pher": 17042, "clipse": 17043, "\u0120twitter": 17044, "\u0120acids": 17045, "ographical": 17046, "hum": 17047, "\u0120Bald": 17048, "ifully": 17049, "\u0120compiler": 17050, "\u0120DA": 17051, "\u0120donor": 17052, "asi": 17053, "\u0120tribal": 17054, "lash": 17055, "\u0120Config": 17056, "\u0120applicants": 17057, "\u0120salaries": 17058, "135": 17059, "Putin": 17060, "\u0120Focus": 17061, "irs": 17062, "\u0120misconduct": 17063, "\u0120Haz": 17064, "\u0120eaten": 17065, "Mobile": 17066, "Muslim": 17067, "\u0120Marcus": 17068, "viol": 17069, "\u0120favorable": 17070, "\u0120stub": 17071, "adin": 17072, "\u0120Hob": 17073, "\u0120faithful": 17074, "\u0120electronics": 17075, "\u0120vacuum": 17076, "wait": 17077, "backed": 17078, "economic": 17079, "dist": 17080, "\u0120tenure": 17081, "\u0120sincere": 17082, "\u0120Together": 17083, "\u0120Wave": 17084, "\u0120progression": 17085, "\u0120denying": 17086, "\u0120distress": 17087, "braska": 17088, "third": 17089, "\u0120mixing": 17090, "\u0120colonial": 17091, "\u0120privately": 17092, "\u0120unrest": 17093, "aternity": 17094, "\u0120premises": 17095, "anti": 17096, "gregation": 17097, "\u0120licence": 17098, "\u0120Hind": 17099, "\u0120Samuel": 17100, "\u0120convincing": 17101, "\u0120Ace": 17102, "\u0120Rust": 17103, "\u0120Netanyahu": 17104, "\u0120handles": 17105, "\u0120Patch": 17106, "oriented": 17107, "aho": 17108, "\u0120Gonz": 17109, "\u0120hackers": 17110, "claimer": 17111, "\u0120customs": 17112, "\u0120Gran": 17113, "fighters": 17114, "\u0120luc": 17115, "\u0120manuscript": 17116, "arenthood": 17117, "\u0120devil": 17118, "\u0120warriors": 17119, "\u0120offenders": 17120, "William": 17121, "\u0120holidays": 17122, "\u0120nightmare": 17123, "\u0120lever": 17124, "ifferent": 17125, "Stat": 17126, "\u0120exhibition": 17127, "puted": 17128, "\u0120Pure": 17129, "\u0120alpha": 17130, "\u0120enthusiasm": 17131, "\u0120Representatives": 17132, "EAR": 17133, "\u0120Typ": 17134, "\u0120wheat": 17135, "\u0120Alf": 17136, "\u0120correction": 17137, "\u0120evangel": 17138, "ATT": 17139, "Miss": 17140, "\u0120soup": 17141, "\u0120implied": 17142, "param": 17143, "\u0120sexy": 17144, "\u0120Lux": 17145, "\u0120republic": 17146, "patch": 17147, "ablish": 17148, "\u0120icons": 17149, "\u0120fathers": 17150, "\u0120GET": 17151, "\u0120Carib": 17152, "\u0120regulated": 17153, "\u0120Cohen": 17154, "\u0120Bobby": 17155, "\u0120ner": 17156, "\u0120bent": 17157, "ventory": 17158, "\u0120Along": 17159, "\u0120EST": 17160, "\u0120Wallace": 17161, "\u0120murders": 17162, "rise": 17163, "kell": 17164, "\u0120Commonwealth": 17165, "\u0120nasty": 17166, "eta": 17167, "\u0120MIT": 17168, "\u0120administered": 17169, "\u0120genuinely": 17170, "Editor": 17171, "nick": 17172, "\u0120hydro": 17173, "********************************": 17174, "\u0120Ble": 17175, "\u0120fines": 17176, "\u0120gorge": 17177, "ausible": 17178, "rh": 17179, "\u0120apple": 17180, "mentioned": 17181, "\u0120rope": 17182, "otyp": 17183, "HR": 17184, "\u0120disappointing": 17185, "\u0120cage": 17186, "nik": 17187, "\u0120doubts": 17188, "\u0120FREE": 17189, "prints": 17190, "\u0120MUST": 17191, "\u0120vendors": 17192, "\u0120Inqu": 17193, "\u0120liberals": 17194, "\u0120contractor": 17195, "\u0120upside": 17196, "children": 17197, "\u0120tricky": 17198, "\u0120regulators": 17199, "charged": 17200, "liter": 17201, "\u0120***": 17202, "\u0120rebell": 17203, "lang": 17204, "\u0120locals": 17205, "\u0120physicians": 17206, "\u0120hey": 17207, "arse": 17208, "tm": 17209, "\u0120Lex": 17210, "\u0120behavioral": 17211, "successful": 17212, "FX": 17213, "\u0120brick": 17214, "ovic": 17215, "\u0120conform": 17216, "\u0120reviewing": 17217, "\u0120insights": 17218, "\u0120biology": 17219, "\u0120Remove": 17220, "\u0120Extra": 17221, "\u0120committing": 17222, "induced": 17223, "ignty": 17224, "igm": 17225, "\u0120atomic": 17226, "Common": 17227, "\u0120EM": 17228, "\u0120Pere": 17229, "\u0120Items": 17230, "eh": 17231, "\u0120preserved": 17232, "\u0120Hood": 17233, "\u0120prisoner": 17234, "\u0120bankruptcy": 17235, "\u0120gren": 17236, "ushes": 17237, "\u0120exploitation": 17238, "\u0120signatures": 17239, "\u0120finan": 17240, "],\"": 17241, "\u0120MR": 17242, "\u0120meg": 17243, "remlin": 17244, "\u0120musicians": 17245, "\u0120selecting": 17246, "\u0120examining": 17247, "INK": 17248, "lated": 17249, "Hi": 17250, "\u0120artic": 17251, "\u0120pets": 17252, "\u0120impair": 17253, "\u0120MAN": 17254, "\u0120tablets": 17255, "include": 17256, "Range": 17257, "\u0120caut": 17258, "\u0120logs": 17259, "\u0120mounting": 17260, "\u0120unaware": 17261, "\u0120dynamics": 17262, "\u0120Palestine": 17263, "\u0120Quarter": 17264, "\u0120Purple": 17265, "\u0120ma": 17266, "\u0120Import": 17267, "\u0120collections": 17268, "ciation": 17269, "\u0120successor": 17270, "\u0120clone": 17271, "\u0120aiming": 17272, "\u0120possessed": 17273, "\u0120sticking": 17274, "\u0120shaking": 17275, "\u0120locate": 17276, "\u0120Hockey": 17277, "Turn": 17278, "170": 17279, "\u0120fifteen": 17280, "\u0120Harrison": 17281, "\u0120continuously": 17282, "\u0120TC": 17283, "\u0120Valent": 17284, "\u0120Rescue": 17285, "\u0120bypass": 17286, "amount": 17287, "\u0120mast": 17288, "\u0120protects": 17289, "\u0120artistic": 17290, "\u0120sometime": 17291, "\u0120shoe": 17292, "\u0120shouted": 17293, "ificant": 17294, "etitive": 17295, "\u0120Register": 17296, "\u0120Jin": 17297, "\u0120concentrated": 17298, "lington": 17299, "onies": 17300, "\u0120generator": 17301, "yrim": 17302, "\u0120Armen": 17303, "\u0120clearing": 17304, "ido": 17305, "\u0120TW": 17306, "alph": 17307, "\u0120ladies": 17308, "Hard": 17309, "\u0120dialog": 17310, "\u0120inputs": 17311, "\u00e6\u013e": 17312, "\u0120poses": 17313, "\u0120slots": 17314, "\u0120Premium": 17315, "\u0120leaks": 17316, "\u0120bosses": 17317, "\u0120113": 17318, "course": 17319, "Acc": 17320, "\u0120Newton": 17321, "\u0120Austria": 17322, "\u0120Mage": 17323, "\u0120teaches": 17324, "abad": 17325, "\u0120wears": 17326, "\u0120cyl": 17327, "\u0120curse": 17328, "\u0120Sales": 17329, "\u0120Wings": 17330, "\u0120psy": 17331, "\u0120gaps": 17332, "\u0120Iceland": 17333, "\u0120Pinterest": 17334, "\u0120landlord": 17335, "\u0120definitions": 17336, "\u0120Ker": 17337, "\u0120sufficiently": 17338, "\u0120Pence": 17339, "\u0120Architect": 17340, "\u0120surpass": 17341, "\u0120114": 17342, "\u0120superhero": 17343, "\u0120Disease": 17344, "\u0120priests": 17345, "\u0120Culture": 17346, "\u0120definitive": 17347, "\u0120secretly": 17348, "\u0120Dance": 17349, "install": 17350, "chief": 17351, "\u0120Jessica": 17352, "Would": 17353, "Updated": 17354, "\u0120locker": 17355, "\u0120Kay": 17356, "\u0120memorial": 17357, "\u00e8\u00a6": 17358, "fat": 17359, "\u0120disgu": 17360, "\u0120flavors": 17361, "\u0120Baseball": 17362, "\u0120Resistance": 17363, "\u0120kicks": 17364, "\u0120env": 17365, "\u0120teenagers": 17366, "Dark": 17367, "\u0120CAR": 17368, "\u0120halt": 17369, "\u0120LG": 17370, "\u0120Gabriel": 17371, "\u0120fever": 17372, "\u0120satur": 17373, "\u0120mall": 17374, "\u0120affiliate": 17375, "\u0120Sleep": 17376, "\u0120Specific": 17377, "\u0120Vel": 17378, "\u0120jar": 17379, "\u0120Sacred": 17380, "\u0120Edwards": 17381, "\u0120ACL": 17382, "\u0120retained": 17383, "\u0120Giant": 17384, "\u0120limitation": 17385, "inces": 17386, "\u0120refusal": 17387, "\u0120Tale": 17388, "\u0120Butler": 17389, "\u0120accidents": 17390, "\u0120CSS": 17391, "\u0120imported": 17392, "\u0120Copy": 17393, "\u00ce\u00b1": 17394, "ERT": 17395, "zel": 17396, "\u0120divisions": 17397, "hots": 17398, "\u0120Alb": 17399, "\u0120DS": 17400, "Loader": 17401, "Washington": 17402, "atisf": 17403, "\u0120Creative": 17404, "\\.": 17405, "\u0120Autom": 17406, "redict": 17407, "\u0120receptor": 17408, "\u0120Carlos": 17409, "Method": 17410, "oka": 17411, "\u0120malicious": 17412, "\u0120stepping": 17413, ",[": 17414, "\u0120Dad": 17415, "\u0120attraction": 17416, "\u0120Effects": 17417, "\u0120Pirate": 17418, "\u0120Cer": 17419, "\u0120Industry": 17420, "\u0120Rud": 17421, "\u0120charter": 17422, "\u0120dining": 17423, "\u0120insists": 17424, "\u0120configure": 17425, "\u0120(#": 17426, "\u0120Simple": 17427, "\u0120Scroll": 17428, "UTC": 17429, "175": 17430, "\u0120Kon": 17431, "\u0120marketplace": 17432, "\u0120\u00e3\u0124": 17433, "\u0120refres": 17434, "\u0120gates": 17435, "erred": 17436, "\u0120Pod": 17437, "\u0120behave": 17438, "Frank": 17439, "node": 17440, "\u0120endorsed": 17441, "hett": 17442, "asive": 17443, "\u0120Homeland": 17444, "\u0120rides": 17445, "\u0120Leave": 17446, "erness": 17447, "\u0120flooding": 17448, "AFP": 17449, "\u0120risen": 17450, "\u0120continually": 17451, "\u0120unanim": 17452, "\u0120Contract": 17453, "\u0120Pas": 17454, "\u0120guided": 17455, "\u0120Chile": 17456, "bd": 17457, "\u0120succ": 17458, "ptic": 17459, "\u0120committees": 17460, "\u0120Luther": 17461, "\u0120Anyone": 17462, "\u0120sab": 17463, "124": 17464, "\u0120pixel": 17465, "\u0120Bak": 17466, "\u0120Tag": 17467, "\u0120Bennett": 17468, "Enter": 17469, "small": 17470, "\u0120Presidential": 17471, "\u0120pul": 17472, "\u0120contrace": 17473, "archive": 17474, "\u0120coastal": 17475, "\u0120Kids": 17476, "192": 17477, "\u00e2\u0122\u00b2": 17478, "icky": 17479, "INGTON": 17480, "\u0120wolf": 17481, "\u0120Stalin": 17482, "Tur": 17483, "idget": 17484, "amas": 17485, "\u0120Unless": 17486, "\u0120sponsor": 17487, "\u0120morph": 17488, "\u0120Choose": 17489, "\u0120runner": 17490, "\u0120unbel": 17491, "\u0120mud": 17492, "\u0120Mana": 17493, "\u0120dubbed": 17494, "\u0120godd": 17495, "urers": 17496, "window": 17497, "\u0120relied": 17498, "\u0120celebrating": 17499, "osc": 17500, "\u0120135": 17501, "\u0120lobbying": 17502, "\u0120incomplete": 17503, "\u0120restriction": 17504, "\u0120incap": 17505, "itus": 17506, "\u0120expectation": 17507, "\u0120Apollo": 17508, "\u0120intens": 17509, "\u0120sync": 17510, "GH": 17511, "\u0120manipulation": 17512, "BY": 17513, "\u0120spear": 17514, "\u0120breasts": 17515, "\u0120volcan": 17516, "ilia": 17517, "Material": 17518, "\u0120formats": 17519, "\u0120Bast": 17520, "\u0120parliamentary": 17521, "\u0120snake": 17522, "\u0120servants": 17523, "\u0120Trudeau": 17524, "\u0120Grim": 17525, "\u0120Arabic": 17526, "\u0120SCP": 17527, "\u0120Boys": 17528, "station": 17529, "\u0120prospective": 17530, "orde": 17531, "initialized": 17532, "\u0120bored": 17533, "ABLE": 17534, "\u0120accessed": 17535, "\u0120taxi": 17536, "\u0120Shell": 17537, "aiden": 17538, "ursed": 17539, "inates": 17540, "\u0120Insurance": 17541, "\u0120Pete": 17542, "September": 17543, "650": 17544, "\u0120adventures": 17545, "\u0120Cover": 17546, "\u0120tribute": 17547, "\u0120sketch": 17548, "\u0120empower": 17549, "\u0120\u00d8": 17550, "\u0120Glenn": 17551, "\u0120Daw": 17552, "=\\\"": 17553, "\u0120Politics": 17554, "\u0120guides": 17555, "\u0120dioxide": 17556, "\u0120Gore": 17557, "\u0120Bright": 17558, "\u0120Sierra": 17559, "\u0120valued": 17560, "cond": 17561, "\u0120pointer": 17562, "Select": 17563, "\u0120risky": 17564, "\u0120absorb": 17565, "images": 17566, "\u0120refuses": 17567, "\u0120bonuses": 17568, "___": 17569, "\u0120hilar": 17570, "\u0120Features": 17571, "220": 17572, "\u0120Collector": 17573, "Foot": 17574, "\u01201964": 17575, "culus": 17576, "\u0120dawn": 17577, "\u0120workout": 17578, "\u0120LO": 17579, "\u0120philosophical": 17580, "\u0120Sandy": 17581, "\u0120Youth": 17582, "\u0120liable": 17583, "Af": 17584, "blue": 17585, "\u0120overturn": 17586, "lessness": 17587, "\u0120Tribune": 17588, "\u0120Ing": 17589, "\u0120factories": 17590, "\u0120catches": 17591, "\u0120prone": 17592, "\u0120matrix": 17593, "\u0120login": 17594, "\u0120inacc": 17595, "\u0120exert": 17596, "sys": 17597, "\u0120needle": 17598, "\u0120Qur": 17599, "\u0120notified": 17600, "oulder": 17601, "tx": 17602, "\u0120reminds": 17603, "\u0120publishers": 17604, "\u0120nort": 17605, "\u0120git": 17606, "\u0120flies": 17607, "\u0120Emily": 17608, "\u0120flowing": 17609, "\u0120Alien": 17610, "\u0120Strateg": 17611, "\u0120hardest": 17612, "\u0120modification": 17613, "API": 17614, "\u0120MY": 17615, "\u0120crashes": 17616, "stairs": 17617, "number": 17618, "\u0120urging": 17619, "channel": 17620, "\u0120Falcon": 17621, "\u0120inhabitants": 17622, "\u0120terrifying": 17623, "\u0120utilize": 17624, "\u0120banner": 17625, "\u0120cigarettes": 17626, "\u0120senses": 17627, "\u0120Holmes": 17628, "\u0120practition": 17629, "\u0120Phillips": 17630, "otto": 17631, "\u0120compile": 17632, "Model": 17633, "\u0120Ko": 17634, "\u0120[]": 17635, "Americans": 17636, "\u0120Terms": 17637, "\u0120medications": 17638, "\u0120Ana": 17639, "\u0120fundamentally": 17640, "\u0120Notice": 17641, "\u0120weaker": 17642, "\u01200000": 17643, "\u0120garlic": 17644, "\u0120outbreak": 17645, "\u0120economist": 17646, "\u0120Birth": 17647, "\u0120obstacles": 17648, "arcer": 17649, "\u0120Orthodox": 17650, "\u0120placebo": 17651, "\u0120Crew": 17652, "aspberry": 17653, "\u0120Angels": 17654, "\u0120discharge": 17655, "\u0120destructive": 17656, "117": 17657, "\u0120Rising": 17658, "\u0120dairy": 17659, "late": 17660, "\u0120collision": 17661, "\u0120Tigers": 17662, "eanor": 17663, "ocumented": 17664, "\u0120Invalid": 17665, "\u0120dont": 17666, "\u0120Liter": 17667, "\u0120Va": 17668, "\u0120hydrogen": 17669, "\u0120variants": 17670, "\u0120Browns": 17671, "\u01201965": 17672, "\u0120indigenous": 17673, "\u0120trades": 17674, "\u0120remainder": 17675, "\u0120swept": 17676, "\u0120Impact": 17677, "\u0120redist": 17678, "\u0120unint": 17679, "graduate": 17680, "\u00e3\u0125\u0137": 17681, "\u0120WILL": 17682, "\u00e3\u0123\u00ae\u00e7": 17683, "\u0120Critical": 17684, "\u0120fisher": 17685, "\u0120vicious": 17686, "\u0120reversed": 17687, "Year": 17688, "\u0120Sox": 17689, "\u0120shootings": 17690, "\u0120filming": 17691, "\u0120touchdowns": 17692, "aires": 17693, "mel": 17694, "\u0120grandfather": 17695, "\u0120affection": 17696, "ingle": 17697, "\u0120overly": 17698, "Additional": 17699, "\u0120supreme": 17700, "\u0120Grad": 17701, "\u0120sporting": 17702, "\u0120mercy": 17703, "\u0120Brooks": 17704, "ounty": 17705, "\u0120performs": 17706, "\u0120tightly": 17707, "\u0120demons": 17708, "\u0120killings": 17709, "\u0120faction": 17710, "\u0120Nova": 17711, "auts": 17712, "\u0120undoubtedly": 17713, "arin": 17714, "\u0120underway": 17715, "rak": 17716, "\u0120liv": 17717, "\u0120Region": 17718, "\u0120briefing": 17719, "sers": 17720, "cloud": 17721, "\u0120Mik": 17722, "usp": 17723, "\u0120prediction": 17724, "azor": 17725, "\u0120portable": 17726, "\u0120Gand": 17727, "\u0120presenting": 17728, "\u01201080": 17729, "\u00c2\u00bb": 17730, "ushi": 17731, "\u0120Spark": 17732, "thereum": 17733, "\u0120justification": 17734, "\u0120Ny": 17735, "\u0120contractors": 17736, "mingham": 17737, "\u0120Style": 17738, "\u00e5\u0127": 17739, "\u0120Chronicles": 17740, "\u0120Picture": 17741, "\u0120proving": 17742, "\u0120wives": 17743, "sett": 17744, "\u0120molecules": 17745, "\u0120Fairy": 17746, "\u0120consisting": 17747, "\u0120pier": 17748, "alone": 17749, "inition": 17750, "\u0120nucle": 17751, "json": 17752, "\u0120gotta": 17753, "\u0120mobil": 17754, "\u0120verbal": 17755, "arium": 17756, "\u0120monument": 17757, "ucked": 17758, "\u0120256": 17759, "Tech": 17760, "minecraft": 17761, "\u0120Track": 17762, "\u0120tile": 17763, "\u0120compatibility": 17764, "asis": 17765, "\u0120sadd": 17766, "\u0120instructed": 17767, "\u0120Mueller": 17768, "\u0120lethal": 17769, "\u0120hormone": 17770, "\u0120orche": 17771, "else": 17772, "\u0120skelet": 17773, "\u0120entertaining": 17774, "\u0120minimize": 17775, "again": 17776, "\u0120undergo": 17777, "\u0120constraints": 17778, "\u0120cigarette": 17779, "\u0120Islamist": 17780, "\u0120travels": 17781, "\u0120Panthers": 17782, "lings": 17783, "Care": 17784, "\u0120lawsuits": 17785, "uras": 17786, "\u0120cryst": 17787, "\u0120lowered": 17788, "\u0120aerial": 17789, "\u0120combinations": 17790, "\u0120haun": 17791, "\u0120cha": 17792, "\u0120vine": 17793, "\u0120quantities": 17794, "\u0120linking": 17795, "bank": 17796, "\u0120soy": 17797, "Bill": 17798, "\u0120Angela": 17799, "\u0120recipient": 17800, "\u0120Protest": 17801, "\u0120socket": 17802, "\u0120solidarity": 17803, "\u0120\u00e2\u0128": 17804, "mill": 17805, "\u0120varies": 17806, "\u0120Pakistani": 17807, "Dragon": 17808, "\u0120une": 17809, "\u0120horizon": 17810, "\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142\u00c2\u0142": 17811, "\u0120provinces": 17812, "\u0120frankly": 17813, "\u0120enacted": 17814, "notes": 17815, "['": 17816, "\u0120192": 17817, "ocracy": 17818, "\u0120endorsement": 17819, "\u0120overtime": 17820, "True": 17821, "Lab": 17822, "licted": 17823, "\u0120DNC": 17824, "\u0120beats": 17825, "\u0120Jamie": 17826, "152": 17827, "\u0120INT": 17828, "Contact": 17829, "\u0120accounted": 17830, "hash": 17831, "\u0120Packers": 17832, "pires": 17833, "\u0120lesbian": 17834, "\u0120amendments": 17835, "\u0120hopeful": 17836, "\u0120Finland": 17837, "\u0120spotlight": 17838, "\u0120configured": 17839, "\u0120troubled": 17840, "\u0120gaze": 17841, "\u0120Calgary": 17842, "\u0120reliability": 17843, "\u0120insurg": 17844, "swer": 17845, "buy": 17846, "\u0120Skin": 17847, "\u0120pixels": 17848, "\u0120handgun": 17849, "\u0120paras": 17850, "\u0120categor": 17851, "\u0120EL": 17852, "\u0120Rex": 17853, "Indeed": 17854, "\u0120kinda": 17855, "\u0120conjunction": 17856, "\u0120Bryan": 17857, "\u0120Manufact": 17858, "yang": 17859, "Plus": 17860, "SQL": 17861, "ishment": 17862, "\u0120dominate": 17863, "\u0120nail": 17864, "\u0120oath": 17865, "\u0120erupt": 17866, "\u0120Fine": 17867, "itbart": 17868, "\u0120Chip": 17869, "\u0120Abd": 17870, "\u0120Nam": 17871, "\u0120buyer": 17872, "\u0120dissent": 17873, "Leaks": 17874, "Contin": 17875, "\u0120rider": 17876, "\u0120Someone": 17877, "\u0120illusion": 17878, "cin": 17879, "\u0120Boeing": 17880, "\u0120inadequ": 17881, "ovation": 17882, "iants": 17883, "\u0120rebuild": 17884, "450": 17885, "\u0120Destiny": 17886, "SW": 17887, "\u0120Till": 17888, "Hit": 17889, "iaz": 17890, "\u0120Bangl": 17891, "achers": 17892, "\u0120Reform": 17893, "\u0120segments": 17894, "\u0120systematic": 17895, "dc": 17896, "\u0120Conservatives": 17897, "\u0120portal": 17898, "hor": 17899, "\u0120Dragonbound": 17900, "\u0120dragged": 17901, "omo": 17902, "\u0120thee": 17903, "advert": 17904, "\u0120Reports": 17905, "\u0120Et": 17906, "\u0120barrels": 17907, "August": 17908, "\u0120comparisons": 17909, "\u0120hex": 17910, "\u0120anthrop": 17911, "\"[": 17912, "borough": 17913, "abi": 17914, "\u0120pictured": 17915, "playing": 17916, "\u0120Address": 17917, "\u0120Mirror": 17918, "Smith": 17919, "\u0120tires": 17920, "\u0120NPR": 17921, "AAAA": 17922, "\u0120classification": 17923, "\u0120Than": 17924, "\u0120Harm": 17925, "\u0120RA": 17926, "\u0120rejection": 17927, "mination": 17928, "\u0120ranged": 17929, "\u0120Falls": 17930, "DI": 17931, "Host": 17932, "\u00e3\u0124\u00b4": 17933, "\u0120Example": 17934, "listed": 17935, "thirds": 17936, "\u0120safegu": 17937, "brand": 17938, "\u0120probable": 17939, "Canada": 17940, "ITION": 17941, "\u0120Qaeda": 17942, "\u0120chick": 17943, "\u0120imports": 17944, "hit": 17945, "loc": 17946, "WW": 17947, "\u0120blew": 17948, "\u0120anytime": 17949, "\u0120wholes": 17950, "iked": 17951, "\u0120calculation": 17952, "create": 17953, "\u0120Ori": 17954, "\u0120upgraded": 17955, "\u0120appar": 17956, "utory": 17957, "\u0120Mol": 17958, "Brit": 17959, "\u0120Jong": 17960, "INAL": 17961, "\u0120Starting": 17962, "\u0120dice": 17963, "urtle": 17964, "\u0120relying": 17965, "closure": 17966, "\u0120profitable": 17967, "\u0120slaughter": 17968, "\u0120Manual": 17969, "caster": 17970, "\u0120\"$": 17971, "\u0120feather": 17972, "\u0120Simply": 17973, "ieves": 17974, "\u0120deterior": 17975, "\u0120PCI": 17976, "\u0120stamp": 17977, "\u0120flaws": 17978, "\u0120shade": 17979, "hammer": 17980, "\u0120passport": 17981, "\u0120conting": 17982, "amel": 17983, "\u0120observers": 17984, "\u0120neglect": 17985, "\u0120RB": 17986, "\u0120Brotherhood": 17987, "\u0120skeptical": 17988, "family": 17989, "usk": 17990, "\u0120emotionally": 17991, "\u00e2\u013b": 17992, "\u0120Beta": 17993, "asonable": 17994, "idity": 17995, "\u0120Mul": 17996, "\u0120kicking": 17997, "\u0120Carm": 17998, "ollah": 17999, "VERTIS": 18000, "\u0120Athen": 18001, "\u0120ladder": 18002, "\u0120Bullet": 18003, "\u00e5\u00a3": 18004, "0001": 18005, "\u0120Wildlife": 18006, "\u0120Mask": 18007, "\u0120Nan": 18008, "Rev": 18009, "\u0120unacceptable": 18010, "legal": 18011, "\u0120crowded": 18012, "agi": 18013, "\u0120Cox": 18014, "je": 18015, "\u0120morality": 18016, "\u0120fuels": 18017, "\u0120cables": 18018, "\u0120mankind": 18019, "\u0120Caribbean": 18020, "\u0120anchor": 18021, "\u0120byte": 18022, "\u0120Often": 18023, "\u0120Oz": 18024, "\u0120crafted": 18025, "\u0120historian": 18026, "\u0120Wu": 18027, "\u0120towers": 18028, "\u0120Citizens": 18029, "\u0120helm": 18030, "\u0120credentials": 18031, "\u0120singular": 18032, "\u0120Jesse": 18033, "\u0120tackles": 18034, "\u0120contempt": 18035, "\u0120afore": 18036, "\u0120Shadows": 18037, "\u0120nil": 18038, "\u0120urgent": 18039, "apple": 18040, "blood": 18041, "\u0120von": 18042, "\u0120offline": 18043, "\u0120breathe": 18044, "\u0120jumps": 18045, "\u0120irrelevant": 18046, "oxic": 18047, "omal": 18048, "important": 18049, "Jim": 18050, "\u0120gloves": 18051, "arming": 18052, "depth": 18053, "\u0120talents": 18054, "ookie": 18055, "\u0120SB": 18056, "\u0120palm": 18057, "uffs": 18058, "esta": 18059, "IGH": 18060, "\u0120canon": 18061, "\u0120Verizon": 18062, "\u0120Ple": 18063, "\u0120coupled": 18064, "velt": 18065, "\u0120fundraising": 18066, "\u0120Getting": 18067, "\u0120DLC": 18068, "\u0120mathematical": 18069, "\u0120HS": 18070, "\u0120Cardinals": 18071, "telling": 18072, "\u0120sponsors": 18073, "\u0120\u00cf": 18074, "\u0120Bulls": 18075, "option": 18076, "\u0120propose": 18077, "\u0120memorable": 18078, "\u0120embraced": 18079, "\u0120declining": 18080, "Health": 18081, "eda": 18082, "\u0120};": 18083, "\u0120spam": 18084, "mile": 18085, "\u0120pitcher": 18086, "\u0120Eight": 18087, "\u0120caring": 18088, "utic": 18089, "role": 18090, "\u0120airline": 18091, "ernandez": 18092, "\u0120Athlet": 18093, "\u0120certification": 18094, "uxe": 18095, "riger": 18096, "\u0120empir": 18097, "\u0120sensation": 18098, "\u0120dism": 18099, "\u0120bolt": 18100, "\u0120evolve": 18101, "House": 18102, "\u0120consultation": 18103, "\u0120Duty": 18104, "\u0120touches": 18105, "\u0120Nathan": 18106, "\u0120faint": 18107, "had": 18108, "\"(": 18109, "\u0120Consumer": 18110, "\u0120Extreme": 18111, "\u0120127": 18112, "\u0120Herm": 18113, "\u0120Sacrament": 18114, "izoph": 18115, "\u0120anxious": 18116, "ulously": 18117, "\u0120socially": 18118, "\u0120UTC": 18119, "\u0120solving": 18120, "\u0120Letter": 18121, "History": 18122, "educ": 18123, "Price": 18124, "));": 18125, "\u0120reload": 18126, "amic": 18127, "\u0120pork": 18128, "\u0120discourse": 18129, "\u0120tournaments": 18130, "airo": 18131, "\u0120Kur": 18132, "\u0120Costa": 18133, "\u0120violating": 18134, "\u0120interfere": 18135, "\u0120recreational": 18136, "uffle": 18137, "\u0120speeches": 18138, "\u0120needing": 18139, "\u0120remembers": 18140, "\u0120credited": 18141, "nia": 18142, "focused": 18143, "amera": 18144, "\u0120bru": 18145, "umbs": 18146, "\u0120Cuban": 18147, "\u0120preceding": 18148, "\u0120nonsense": 18149, "acial": 18150, "\u0120smartphones": 18151, "\u0120Stories": 18152, "Sports": 18153, "\u0120Emergency": 18154, "ouncing": 18155, "efined": 18156, "\u0120ber": 18157, "\u0120consulting": 18158, "\u0120masters": 18159, "heastern": 18160, ".\"[": 18161, "\u0120Running": 18162, "\u0120suscept": 18163, "\u0120Feng": 18164, "America": 18165, "prises": 18166, "stitial": 18167, "\u0120Weekly": 18168, "\u0120Greater": 18169, "modules": 18170, "ifter": 18171, "Graphics": 18172, "uler": 18173, "\u0120wholly": 18174, "\u0120suppress": 18175, "\u0120concealed": 18176, "\u0120happily": 18177, "\u0120accepts": 18178, "\u0120Enjoy": 18179, "\u0120rivers": 18180, "\u0120Except": 18181, "225": 18182, "\u0120NHS": 18183, "\u0120McConnell": 18184, "\u0120pussy": 18185, "ferred": 18186, "utable": 18187, "\u0120attain": 18188, "\u0120>=": 18189, "\u0120deposits": 18190, "rophic": 18191, "\u0120notorious": 18192, "\u0120Shaw": 18193, "ilitation": 18194, "\u0120epidemic": 18195, "allic": 18196, "\u0120smallest": 18197, "ovich": 18198, "\u0120accessories": 18199, "perties": 18200, "\u0120surplus": 18201, "\u0120Mech": 18202, "\u0120ambig": 18203, "\u0120Immigration": 18204, "\u0120chim": 18205, "eval": 18206, "\u0120practicing": 18207, "\u0120Mystery": 18208, "\u0120domains": 18209, "\u0120Silicon": 18210, "apps": 18211, "\u0120kilometers": 18212, "ea": 18213, "\u0120Smash": 18214, "\u0120warranty": 18215, "\u0120nost": 18216, "sil": 18217, "rev": 18218, "Jon": 18219, "\u0120Dublin": 18220, "\u0120tastes": 18221, "\u0120bout": 18222, "great": 18223, "error": 18224, "\u0120switches": 18225, "\u0120Bapt": 18226, "DO": 18227, "oki": 18228, "\u0120sourced": 18229, "produ": 18230, "\u0120attachment": 18231, "\u0120Issue": 18232, "\u0120Question": 18233, "Join": 18234, "\u0120fitted": 18235, "\u0120unlawful": 18236, "^^": 18237, "erek": 18238, "\u0120authentication": 18239, "\u0120stole": 18240, "\u0120accountability": 18241, "label": 18242, "Search": 18243, "\u0120albeit": 18244, "atican": 18245, "funded": 18246, "\u0120Adding": 18247, "\u0120IQ": 18248, "\u0120submar": 18249, "lit": 18250, "aque": 18251, "\u0120Learning": 18252, "\u0120integer": 18253, "Master": 18254, "\u0120Chrom": 18255, "\u0120premier": 18256, "Op": 18257, "\u0120Liu": 18258, "\u0120blessed": 18259, "\u0120Globe": 18260, "\u0120Response": 18261, "\u0120legitim": 18262, "\u0120Merkel": 18263, "\u0120disposal": 18264, "\u00c2\u00b4": 18265, "\u0120gauge": 18266, "peat": 18267, "\u0120induced": 18268, "\u0120questionable": 18269, "arthy": 18270, "\u0120Vit": 18271, "\u0120Feed": 18272, "Until": 18273, "Ut": 18274, "worthy": 18275, "RY": 18276, "\u0120Herald": 18277, "\u0120Hammer": 18278, "\u0120medal": 18279, "\u0120Rivers": 18280, "\u0120Hack": 18281, "\u0120clarify": 18282, "\u0120tracked": 18283, "\u0120autonomous": 18284, "\u0120tenant": 18285, "\u0120Qatar": 18286, "erie": 18287, "\u0120grim": 18288, "\u0120Monitor": 18289, "\u0120resistant": 18290, "\u0120Spec": 18291, "\u0120Wells": 18292, "NAS": 18293, "148": 18294, "\u0120miners": 18295, "iotics": 18296, "\u0120misses": 18297, "116": 18298, "gian": 18299, "git": 18300, "\u0120Eyes": 18301, "pres": 18302, "\u0120graduated": 18303, "\u0120angel": 18304, "\u0120synchron": 18305, "\u0120efficiently": 18306, "\u0120transmitted": 18307, "Harry": 18308, "\u0120globally": 18309, "ENCE": 18310, "\u0120Montana": 18311, "raged": 18312, "\u0120Prevention": 18313, "\u0120piss": 18314, "\u0120Ll": 18315, "\u0120shelf": 18316, "\u0120BJP": 18317, "\u0120Testament": 18318, "\u0120Late": 18319, "iker": 18320, "\u0120Happ": 18321, "\u0120Julian": 18322, "hall": 18323, "\u0120spont": 18324, "\u0120shutdown": 18325, "\u0120inconsistent": 18326, "\u0120subscribers": 18327, "\u0120skeleton": 18328, "\u0120Nebraska": 18329, "\u0120inspire": 18330, "\u0120Void": 18331, "Feed": 18332, "\u0120angles": 18333, "\u0120Springs": 18334, "\u0120benchmark": 18335, "\u0120vaccines": 18336, "izophren": 18337, "sexual": 18338, "uffed": 18339, "\u0120shine": 18340, "\u0120Kath": 18341, "\u0120gesture": 18342, "inea": 18343, "\u0120rip": 18344, "\u0120oppression": 18345, "\u0120conscience": 18346, "bt": 18347, "\u0120Lum": 18348, "\u0120incidence": 18349, "\u0120Fa": 18350, "wr": 18351, "\u0120mineral": 18352, "\u0120Spurs": 18353, "alky": 18354, "\u0120thunder": 18355, "\u0120opio": 18356, "Being": 18357, "\u0120Palm": 18358, "\u0120wasted": 18359, "\u0120lb": 18360, "iaries": 18361, "\u0120Initiative": 18362, "\u0120curric": 18363, "\u0120marker": 18364, "\u0120McL": 18365, "\u0120extensions": 18366, "\u0120Pv": 18367, "\u0120Arms": 18368, "\u0120offerings": 18369, "\u0120defenses": 18370, "\u0120vendor": 18371, "\u0120contradict": 18372, "\u0120Colin": 18373, "\u0120reddit": 18374, "\u0120peripher": 18375, "122": 18376, "\u0120sins": 18377, "Edit": 18378, "ICT": 18379, "Soft": 18380, "\u0120Shah": 18381, "\u0120administrator": 18382, "\u0120Trip": 18383, "\u0120pornography": 18384, "\u0120tuition": 18385, "inence": 18386, "\u0120Progress": 18387, "\u0120catalog": 18388, "\u0120suite": 18389, "\u0120hike": 18390, "\u0120reproductive": 18391, "engine": 18392, "\u0120drought": 18393, "\u0120Noah": 18394, "\u0120230": 18395, "\u0120dude": 18396, "\u0120relaxed": 18397, "\u0120partition": 18398, "\u0120participant": 18399, "\u0120telesc": 18400, "\u0120feas": 18401, "\u0120FF": 18402, "owner": 18403, "\u0120sweeping": 18404, "\u0120lenses": 18405, "\u0120matchup": 18406, "\u0120Repl": 18407, "ournals": 18408, "\u0120credible": 18409, "\u0120grandmother": 18410, "\u0120thermal": 18411, "\u0120subscribing": 18412, "\u0120identities": 18413, "colm": 18414, "UCT": 18415, "\u0120reluctant": 18416, "users": 18417, "\u0120Cort": 18418, "\u0120assisted": 18419, "OSS": 18420, "ATIONS": 18421, "ISH": 18422, "\u0120pharmaceutical": 18423, "icable": 18424, "adian": 18425, "\u0120Sonic": 18426, "\u0120Fury": 18427, "\u0120Mong": 18428, "AH": 18429, "\u0120Psychology": 18430, "\u0120phosph": 18431, "\u0120treats": 18432, "\u0143\u0136": 18433, "\u0120steadily": 18434, "\u0120Hello": 18435, "\u0120relates": 18436, "\u0120clue": 18437, "Expl": 18438, "auth": 18439, "\u0120revision": 18440, "\u0120eld": 18441, "osion": 18442, "\u0120bron": 18443, "144": 18444, "rikes": 18445, "\u0120mines": 18446, "\u0120blanket": 18447, "\u0120Fail": 18448, "eled": 18449, "\u0120Imagine": 18450, "\u0120Planned": 18451, "aic": 18452, "Request": 18453, "Mad": 18454, "\u0120Horse": 18455, "\u0120Eagle": 18456, "\u0120capac": 18457, "157": 18458, "\u0120ling": 18459, "\u0120Nice": 18460, "\u0120Parenthood": 18461, "minster": 18462, "ogs": 18463, "ensitive": 18464, "Nothing": 18465, "\u0120carn": 18466, "Fin": 18467, "\u0120PE": 18468, "\u0120rifles": 18469, "\u0120LP": 18470, "Sand": 18471, "\u0120guiActive": 18472, "\u0120tourist": 18473, "CNN": 18474, "\u0120unveiled": 18475, "\u0120predecessor": 18476, "}{": 18477, "uber": 18478, "\u0120offshore": 18479, "\u0120optical": 18480, "\u0120Rot": 18481, "\u0120Pearl": 18482, "eton": 18483, "\u0120stared": 18484, "\u0120farther": 18485, "atility": 18486, "contin": 18487, "\u0120Gy": 18488, "\u0120Foster": 18489, "\u0120Coc": 18490, "rients": 18491, "\u0120designing": 18492, "\u0120Economy": 18493, "ONG": 18494, "Women": 18495, "\u0120Nancy": 18496, "erver": 18497, "\u0120mascul": 18498, "\u0120casualties": 18499, "\u0120225": 18500, "\u0120Sullivan": 18501, "\u0120Choice": 18502, "\u0120aster": 18503, "ws": 18504, "\u0120hotels": 18505, "\u0120considerations": 18506, "\u0120couch": 18507, "\u0120Strip": 18508, "\u0120Gn": 18509, "\u0120manipulate": 18510, "lied": 18511, "\u0120synthetic": 18512, "\u0120assaulted": 18513, "\u0120offenses": 18514, "\u0120Drake": 18515, "\u0120impe": 18516, "October": 18517, "\u0120Heritage": 18518, "hl": 18519, "\u0120Blair": 18520, "Unlike": 18521, "\u0120grief": 18522, "\u0120450": 18523, "\u0120opted": 18524, "\u0120resignation": 18525, "ilo": 18526, "\u0120verse": 18527, "\u0120Tomb": 18528, "\u0120upt": 18529, "\u0120aired": 18530, "\u0120Hook": 18531, "\u0120MLB": 18532, "\u0120assumes": 18533, "outed": 18534, "\u0120Vers": 18535, "\u0120inferior": 18536, "\u0120bundle": 18537, "\u0120DNS": 18538, "ographer": 18539, "\u0120multip": 18540, "\u0120Souls": 18541, "\u0120illustrated": 18542, "\u0120tactic": 18543, "\u0120dressing": 18544, "\u0120duo": 18545, "Conf": 18546, "\u0120relent": 18547, "\u0120cant": 18548, "\u0120scarce": 18549, "\u0120candy": 18550, "\u0120CF": 18551, "\u0120affiliated": 18552, "\u0120sprint": 18553, "ylan": 18554, "\u0120Garcia": 18555, "\u0120junk": 18556, "Print": 18557, "exec": 18558, "Crit": 18559, "\u0120portrait": 18560, "iries": 18561, "\u0120OFF": 18562, "\u0120disputes": 18563, "WR": 18564, "Love": 18565, "\u00e3\u0123\u0126": 18566, "\u0120Reyn": 18567, "\u0120hipp": 18568, "opath": 18569, "\u0120floors": 18570, "\u0120Feel": 18571, "\u0120worries": 18572, "\u0120settlements": 18573, "\u0120Pos": 18574, "\u0120mosque": 18575, "\u0120finals": 18576, "\u0120crushed": 18577, "\u0120Probably": 18578, "\u0120Bot": 18579, "\u0120Mans": 18580, "\u0120Period": 18581, "\u0120sovereignty": 18582, "\u0120seller": 18583, "\u0120apost": 18584, "\u0120amateur": 18585, "\u0120dorm": 18586, "\u0120consuming": 18587, "\u0120armour": 18588, "\u0120Roose": 18589, "\u0120intensive": 18590, "\u0120eliminating": 18591, "\u0120Sunni": 18592, "\u0120Aleppo": 18593, "jin": 18594, "\u0120advise": 18595, "pal": 18596, "\u0120Halo": 18597, "\u0120descent": 18598, "\u0120simpler": 18599, "\u0120booth": 18600, "STR": 18601, "Later": 18602, "\u0120Cave": 18603, "===": 18604, "\u0120mol": 18605, "\u0120fist": 18606, "\u0120shotgun": 18607, "supp": 18608, "\u0120robbery": 18609, "Effect": 18610, "\u0120obscure": 18611, "\u0120Professional": 18612, "\u0120embassy": 18613, "\u0120militant": 18614, "\u0120incarcer": 18615, "\u0120generates": 18616, "\u0120launches": 18617, "\u0120administrators": 18618, "\u0120shaft": 18619, "\u0120circular": 18620, "\u0120freshman": 18621, "\u0120Wes": 18622, "\u0120Joel": 18623, "\u0120Drew": 18624, "\u0120Duncan": 18625, "\u0120Apparently": 18626, "sight": 18627, "\u0120Internal": 18628, "\u0120Individual": 18629, "\u0120FE": 18630, "\u0120bore": 18631, "\u0120Mt": 18632, "\u0120broadly": 18633, "\u0120Options": 18634, "ountain": 18635, "ipes": 18636, "\u0120Videos": 18637, "204": 18638, "\u0120hills": 18639, "\u0120simulation": 18640, "\u0120disappointment": 18641, "itan": 18642, "\u0120Laboratory": 18643, "\u0120upward": 18644, "\u0120boundary": 18645, "\u0120darker": 18646, "hart": 18647, "\u0120dominance": 18648, "Cong": 18649, "\u0120Oracle": 18650, "\u0120Lords": 18651, "\u0120scholarship": 18652, "\u0120Vincent": 18653, "ede": 18654, "\u0120Rah": 18655, "\u0120encourages": 18656, "rov": 18657, "\u0120quo": 18658, "\u0120premise": 18659, "\u0120Crisis": 18660, "\u0120Holocaust": 18661, "\u0120rhythm": 18662, "\u0120metric": 18663, "club": 18664, "\u0120transported": 18665, "\u0120nod": 18666, "\u0120Pist": 18667, "\u0120ancestors": 18668, "\u0120Freder": 18669, "thumbnails": 18670, "\u0120CE": 18671, "OND": 18672, "Phil": 18673, "venge": 18674, "\u0120Products": 18675, "castle": 18676, "\u0120qualifying": 18677, "\u0120Karen": 18678, "VERTISEMENT": 18679, "\u0120mighty": 18680, "\u0120explanations": 18681, "\u0120fixing": 18682, "Di": 18683, "\u0120declaring": 18684, "\u0120anonymity": 18685, "\u0120juven": 18686, "\u0120Nord": 18687, "\u0120Doom": 18688, "\u0120Actually": 18689, "Ok": 18690, "phis": 18691, "\u0120Desert": 18692, "\u0120116": 18693, "IK": 18694, "\u0120FM": 18695, "\u0120incomes": 18696, "VEL": 18697, "okers": 18698, "\u0120pecul": 18699, "\u0120lightweight": 18700, "gue": 18701, "\u0120accent": 18702, "\u0120increment": 18703, "\u0120Chan": 18704, "\u0120complaining": 18705, "\u0120Baghd": 18706, "\u0120midfielder": 18707, "\u0120overhaul": 18708, "Process": 18709, "\u0120Hollow": 18710, "\u0120Titans": 18711, "Small": 18712, "manuel": 18713, "\u0120Unity": 18714, "\u0120Events": 18715, "Sty": 18716, "\u0120disproportion": 18717, "nesty": 18718, "enes": 18719, "\u0120Cod": 18720, "\u0120demonstrations": 18721, "\u0120Crimson": 18722, "\u0120OH": 18723, "\u0120enrolled": 18724, "\u0120cel": 18725, "\u0120Brett": 18726, "\u0120aide": 18727, "\u0120heels": 18728, "\u0120broadband": 18729, "\u0120marking": 18730, "\u0120wizard": 18731, "\u0120NJ": 18732, "\u0120Chiefs": 18733, "\u0120ingredient": 18734, "\u0120dug": 18735, "\u0120Shut": 18736, "urchase": 18737, "endor": 18738, "\u0120farmer": 18739, "\u0120Goldman": 18740, "129": 18741, "155": 18742, "Order": 18743, "\u0120lion": 18744, "iably": 18745, "\u0120stain": 18746, "array": 18747, "ilitary": 18748, "\u0120FAQ": 18749, "\u0120exploded": 18750, "\u0120McCarthy": 18751, "\u0120Tweet": 18752, "\u0120Greens": 18753, "eking": 18754, "ln": 18755, "ensen": 18756, "\u0120motorcycle": 18757, "\u0120particle": 18758, "\u0120cholesterol": 18759, "Bron": 18760, "\u0120stair": 18761, "\u0120oxid": 18762, "\u0120desirable": 18763, "ibles": 18764, "\u0120theor": 18765, "forcing": 18766, "\u0120promotional": 18767, "ovo": 18768, "boot": 18769, "\u0120Bonus": 18770, "rawling": 18771, "\u0120shortage": 18772, "\u0120Psy": 18773, "\u0120recruited": 18774, "\u0120infants": 18775, "\u0120testosterone": 18776, "\u0120deduct": 18777, "\u0120distinctive": 18778, "\u0120firmware": 18779, "built": 18780, "145": 18781, "\u0120explored": 18782, "\u0120factions": 18783, "\u0120vide": 18784, "\u0120tattoo": 18785, "\u0120financially": 18786, "\u0120fatigue": 18787, "\u0120proceeding": 18788, "constitutional": 18789, "\u0120miser": 18790, "\u0120chairs": 18791, "gging": 18792, "ipple": 18793, "\u0120dent": 18794, "\u0120disreg": 18795, "\u00e7\u0136": 18796, "stant": 18797, "llo": 18798, "bps": 18799, "akening": 18800, "\u0120abnormal": 18801, "\u0120ERA": 18802, "\u00e5\u00a3\u00ab": 18803, "\u0120HBO": 18804, "\u0120MAR": 18805, "\u0120concess": 18806, "\u0120servant": 18807, "\u0120aspir": 18808, "lav": 18809, "\u0120Panel": 18810, "amo": 18811, "\u0120precip": 18812, "\u0120recordings": 18813, "\u0120proceeded": 18814, "\u0120colony": 18815, "\u0120Tang": 18816, "ablo": 18817, "\u0120stripped": 18818, "Left": 18819, "too": 18820, "\u0120potatoes": 18821, "\u0120finest": 18822, "%).": 18823, "\u0120crap": 18824, "\u0120Zach": 18825, "abases": 18826, "\u0120Goth": 18827, "\u0120billionaire": 18828, "wolf": 18829, "\u0120sanction": 18830, "SK": 18831, "\u0120logged": 18832, "Po": 18833, "eyed": 18834, "unal": 18835, "\u0120cricket": 18836, "\u0120armies": 18837, "\u0120uncovered": 18838, "Cloud": 18839, "\u00c3\u00b3n": 18840, "\u0120rebounds": 18841, "\u0120mes": 18842, "Oper": 18843, "Pac": 18844, "\u0120nationally": 18845, "\u0120inserted": 18846, "pict": 18847, "\u0120governance": 18848, "\u00d0\u00b8": 18849, "\u0120privileges": 18850, "GET": 18851, "\u0120favorites": 18852, "imity": 18853, "\u0120lover": 18854, "them": 18855, "empl": 18856, "\u0120gorgeous": 18857, "Ann": 18858, "\u0120slipped": 18859, "\u0120veto": 18860, "Bob": 18861, "\u0120slim": 18862, "ucc": 18863, "\u0120Fame": 18864, "uddenly": 18865, "\u0120denies": 18866, "\u0120Maur": 18867, "\u0120distances": 18868, "\u0120wanna": 18869, "tar": 18870, "\u0120SER": 18871, "\u0120\u00e2\u012a": 18872, "\u0120lemon": 18873, "athetic": 18874, "\u0120literal": 18875, "\u0120distinguished": 18876, "\u0120answering": 18877, "GI": 18878, "\u0120religions": 18879, "\u0120Philos": 18880, "\u0120Lay": 18881, "\u0120compos": 18882, "irements": 18883, "\u0120Kos": 18884, "inez": 18885, "rolling": 18886, "\u0120youngest": 18887, "andise": 18888, "\u0120Born": 18889, "\u0120altar": 18890, "amina": 18891, "\u0120Boot": 18892, "voc": 18893, "\u0120digging": 18894, "\u0120pressures": 18895, "\u0120len": 18896, "264": 18897, "\u0120assassination": 18898, "\u0120Birmingham": 18899, "\u0120Myth": 18900, "\u0120sovereign": 18901, "\u0120Artist": 18902, "\u0120Photograph": 18903, "\u0120depicted": 18904, "\u0120dispens": 18905, "orthy": 18906, "\u0120ambul": 18907, "integ": 18908, "\u0120Cele": 18909, "\u0120Tibet": 18910, "\u0120hierarchy": 18911, "\u0120cu": 18912, "\u0120preseason": 18913, "\u0120Peterson": 18914, "\u0120colours": 18915, "\u0120worrying": 18916, "\u0120backers": 18917, "\u0120Palmer": 18918, "\u0120\u00ce\u00bc": 18919, "\u0120contributor": 18920, "\u0120hearings": 18921, "\u0120urine": 18922, "\u0120\u00d9": 18923, "ourgeois": 18924, "Similar": 18925, "\u0120Zimmer": 18926, "something": 18927, "\u0120USC": 18928, "\u0120strengths": 18929, "\u0120FI": 18930, "\u0120logging": 18931, "Asked": 18932, "\u0120Thai": 18933, "inqu": 18934, "\u0120Walt": 18935, "\u0120crews": 18936, "itism": 18937, "301": 18938, "\u0120sharply": 18939, "umed": 18940, "\u0120redirect": 18941, "rators": 18942, "Inf": 18943, "\u0120Weapons": 18944, "\u0120teasp": 18945, "1999": 18946, "Live": 18947, "\u0120Especially": 18948, "\u0120Ster": 18949, "\u0120Veterans": 18950, "\u0120intro": 18951, "otherapy": 18952, "\u0120malware": 18953, "\u0120breeding": 18954, "\u0120molecular": 18955, "\u0120Route": 18956, "\u0120Comment": 18957, "ochem": 18958, "\u0120ain": 18959, "Season": 18960, "\u0120linebacker": 18961, "\u00c4\u00ab": 18962, "\u0120Economics": 18963, "esar": 18964, "\u0120Lives": 18965, "\u0120Emma": 18966, "\u0120kin": 18967, "\u0120Territ": 18968, "\u0120planted": 18969, "oton": 18970, "\u0120Butter": 18971, "\u0120Spons": 18972, "PER": 18973, "\u0120dungeon": 18974, "\u0120symbolic": 18975, "\u0120filmed": 18976, "\u0120diets": 18977, "\u0120concludes": 18978, "\u0120certainty": 18979, "\u0120Format": 18980, "\u0120strangers": 18981, "format": 18982, "\u0120Phase": 18983, "\u0120copied": 18984, "\u0120metres": 18985, "lda": 18986, "\u0120Users": 18987, "\u0120deliberate": 18988, "\u0120washed": 18989, "\u0120Lance": 18990, "imation": 18991, "\u0120improper": 18992, "\u0120Genesis": 18993, "ickr": 18994, "\u0120Kush": 18995, "\u0120realise": 18996, "\u0120embarrassing": 18997, "alking": 18998, "bucks": 18999, "\u0120verified": 19000, "\u0120outline": 19001, "years": 19002, "\u0120Income": 19003, "202": 19004, "\u0120zombies": 19005, "Final": 19006, "\u0120Millenn": 19007, "\u0120modifications": 19008, "\u0120Vision": 19009, "\u0120Moses": 19010, "verb": 19011, "iterranean": 19012, "\u0120Jet": 19013, "\u0120naval": 19014, "\u0120Agg": 19015, "\u0120url": 19016, "\u0120victories": 19017, "\u0120nonetheless": 19018, "\u0120injust": 19019, "\u0120Fact": 19020, "\u00e7\u013c": 19021, "\u0120insufficient": 19022, "review": 19023, "facebook": 19024, "\u0120negotiating": 19025, "\u0120guarantees": 19026, "imen": 19027, "utenberg": 19028, "\u0120gambling": 19029, "\u0120congr": 19030, "Loading": 19031, "\u0120nevertheless": 19032, "\u0120presidents": 19033, "\u0120Industrial": 19034, "\u0120118": 19035, "\u0120poured": 19036, "\u0120Tory": 19037, "\u0120175": 19038, "\u0120:=": 19039, "Scott": 19040, "angered": 19041, "Tok": 19042, "\u0120organizers": 19043, "Mat": 19044, "\u0120Growth": 19045, "\u0120adul": 19046, "\u0120ensures": 19047, "\u0120117": 19048, "\u00e9\u00be\u012f\u00e5": 19049, "\u0120massacre": 19050, "\u0120grades": 19051, "before": 19052, "ADVERTISEMENT": 19053, "\u0120Slow": 19054, "\u0120MMA": 19055, "\u00e2\u0122\u0136\"": 19056, "\u0120Vatican": 19057, "Qaeda": 19058, "\u0120owe": 19059, "6666": 19060, "\u0120Sorry": 19061, "\u0120Grass": 19062, "\u0120backgrounds": 19063, "\u0120exhausted": 19064, "\u0120clan": 19065, "\u0120compromised": 19066, "\u0120Elf": 19067, "\u0120Isaac": 19068, "enson": 19069, "Invest": 19070, "IFA": 19071, "\u0120interrupted": 19072, "\u00e3\u0125\u012b\u00e3\u0125\u00a9": 19073, "\u0120twisted": 19074, "\u0120Dragons": 19075, "Mode": 19076, "\u0120Kremlin": 19077, "\u0120fertil": 19078, "heres": 19079, "phan": 19080, "\u0120Node": 19081, "fed": 19082, "\u0120Orc": 19083, "\u0120unwilling": 19084, "Cent": 19085, "\u0120priorit": 19086, "\u0120graduates": 19087, "\u0120subjective": 19088, "\u0120issuing": 19089, "\u0120Lt": 19090, "\u0120viewer": 19091, "\u0120woke": 19092, "Thus": 19093, "brook": 19094, "\u0120depressed": 19095, "\u0120bracket": 19096, "\u0120Gor": 19097, "\u0120Fighting": 19098, "\u0120striker": 19099, "Report": 19100, "\u0120Portugal": 19101, "\u0120neo": 19102, "wed": 19103, "199": 19104, "\u0120fleeing": 19105, "shadow": 19106, "identified": 19107, "USE": 19108, "Steam": 19109, "\u0120stretched": 19110, "\u0120revelations": 19111, "arted": 19112, "\u0120Dw": 19113, "\u0120alignment": 19114, "eston": 19115, "\u0120Jared": 19116, "Sep": 19117, "\u0120blogs": 19118, "update": 19119, "gom": 19120, "risk": 19121, "\u0120clash": 19122, "\u0120Hour": 19123, "\u0120runtime": 19124, "\u0120unwanted": 19125, "\u0120scam": 19126, "\u0120rack": 19127, "\u0120enlight": 19128, "onest": 19129, "\u0120Ferr": 19130, "\u0120convictions": 19131, "\u0120piano": 19132, "\u0120circulation": 19133, "\u0120Welcome": 19134, "\u0120backlash": 19135, "\u0120Wade": 19136, "\u0120receivers": 19137, "otive": 19138, "Jeff": 19139, "\u0120networking": 19140, "\u0120Prep": 19141, "\u0120Explorer": 19142, "\u0120lecture": 19143, "\u0120uploaded": 19144, "\u0120Meat": 19145, "BLE": 19146, "\u0120Nazis": 19147, "\u0120Synd": 19148, "stud": 19149, "roots": 19150, "rians": 19151, "\u0120portrayed": 19152, "\u0120??": 19153, "\u0120Buddha": 19154, "sun": 19155, "Robert": 19156, "\u0120Complex": 19157, "\u0120oversee": 19158, "\u0120stealth": 19159, "Title": 19160, "\u0120Jobs": 19161, "\u0120Kum": 19162, "\u0120appreciation": 19163, "\u0120MOD": 19164, "\u0120basics": 19165, "\u0120clips": 19166, "\u0120nursing": 19167, "\u0120proposition": 19168, "\u0120realised": 19169, "\u0120NYC": 19170, "\u0120allocated": 19171, "rium": 19172, "aran": 19173, "\u0120Production": 19174, "\u0120Vote": 19175, "\u0120smugg": 19176, "\u0120hunter": 19177, "azer": 19178, "\u0120Changes": 19179, "\u0120fluct": 19180, "yon": 19181, "Array": 19182, "\u0120kits": 19183, "Water": 19184, "\u0120uncommon": 19185, "\u0120resting": 19186, "ells": 19187, "would": 19188, "\u0120pursued": 19189, "\u0120assertion": 19190, "ometown": 19191, "\u0120Mosul": 19192, "\u0120Platform": 19193, "iolet": 19194, "\u0120shareholders": 19195, "\u0120trails": 19196, "Pay": 19197, "\u0120Enforcement": 19198, "types": 19199, "\u0120Anonymous": 19200, "\u0120satisfying": 19201, "ilogy": 19202, "\u0120('": 19203, "wave": 19204, "city": 19205, "Steve": 19206, "\u0120confrontation": 19207, "\u0120Eld": 19208, "Capt": 19209, "ahan": 19210, "htm": 19211, "\u0120Ctrl": 19212, "ONS": 19213, "230": 19214, "ifa": 19215, "holding": 19216, "\u0120delicate": 19217, "\u0120jaw": 19218, "\u0120Going": 19219, "orum": 19220, "Sal": 19221, "\u0120dull": 19222, "\u0120Beth": 19223, "\u0120prisons": 19224, "\u0120ego": 19225, "\u0120Elsa": 19226, "avorite": 19227, "\u0120Gang": 19228, "\u0120Nuclear": 19229, "\u0120spider": 19230, "atsu": 19231, "\u0120sampling": 19232, "\u0120absorbed": 19233, "\u0120Pharm": 19234, "ieth": 19235, "\u0120bucket": 19236, "\u0120Recomm": 19237, "OF": 19238, "\u0120Factory": 19239, "ANCE": 19240, "\u0120bacter": 19241, "Has": 19242, "\u0120Observ": 19243, "121": 19244, "\u0120premiere": 19245, "Develop": 19246, "\u0120currencies": 19247, "Cast": 19248, "\u0120accompanying": 19249, "\u0120Nashville": 19250, "\u0120fatty": 19251, "\u0120Brend": 19252, "\u0120locks": 19253, "\u0120centered": 19254, "\u0120UT": 19255, "aughs": 19256, "orie": 19257, "\u0120Affordable": 19258, "vance": 19259, "DL": 19260, "emet": 19261, "\u0120throne": 19262, "\u0120Bluetooth": 19263, "\u0120naming": 19264, "ifts": 19265, "ADE": 19266, "\u0120corrected": 19267, "\u0120promptly": 19268, "\u0120STR": 19269, "\u0120genome": 19270, "\u0120cope": 19271, "\u0120valley": 19272, "\u0120rounded": 19273, "\u0120Kend": 19274, "alion": 19275, "pers": 19276, "\u0120tourism": 19277, "\u0120stark": 19278, "vl": 19279, "\u0120blowing": 19280, "\u0120Schedule": 19281, "std": 19282, "\u0120unhappy": 19283, "\u0120litigation": 19284, "cedes": 19285, "\u0120android": 19286, "\u0120integral": 19287, "erers": 19288, "uded": 19289, "tax": 19290, "\u0120reiter": 19291, "\u0120Motors": 19292, "ociated": 19293, "\u0120wonders": 19294, "\u0120Apost": 19295, "ucking": 19296, "\u0120Roosevelt": 19297, "fram": 19298, "\u0120yields": 19299, "\u0120constitutes": 19300, "awk": 19301, "Interest": 19302, "\u0120interim": 19303, "\u0120breakthrough": 19304, "\u0120Cher": 19305, "\u0120prosec": 19306, "\u0120Dj": 19307, "\u0120MT": 19308, "Resp": 19309, "\u0120PT": 19310, "\u0120sperm": 19311, "edit": 19312, "BT": 19313, "Linux": 19314, "country": 19315, "league": 19316, "\u0120dick": 19317, "\u0120oct": 19318, "\u0120inserting": 19319, "\u0120scra": 19320, "\u0120Brewing": 19321, "\u01201966": 19322, "\u0120runners": 19323, "\u0120plun": 19324, "idy": 19325, "\u0120Dian": 19326, "\u0120dysfunction": 19327, "\u0120exclusion": 19328, "\u0120disgr": 19329, "\u0120incorporate": 19330, "\u0120reconc": 19331, "\u0120nominated": 19332, "\u0120Archer": 19333, "draw": 19334, "achelor": 19335, "\u0120writings": 19336, "\u0120shallow": 19337, "\u0120hast": 19338, "\u0120BMW": 19339, "\u0120RS": 19340, "\u0120thigh": 19341, "\u01201963": 19342, "\u0120lamb": 19343, "\u0120favored": 19344, "agle": 19345, "\u0120cooler": 19346, "\u0120Hours": 19347, "\u0120GU": 19348, "\u0120Origin": 19349, "\u0120glimpse": 19350, "--------------------": 19351, "Lim": 19352, "\u0120cheek": 19353, "\u0120jealous": 19354, "-'": 19355, "\u0120harness": 19356, "\u0120Poison": 19357, "\u0120disabilities": 19358, "neapolis": 19359, "\u0120outlook": 19360, "\u0120notify": 19361, "\u0120Indianapolis": 19362, "\u0120abrupt": 19363, "nsic": 19364, "\u0120encrypted": 19365, "\u0120forfe": 19366, "reath": 19367, "\u0120rabb": 19368, "\u0120foundations": 19369, "\u0120compliment": 19370, "\u0120Interview": 19371, "\u0120Swe": 19372, "\u0120adolesc": 19373, "\u0120monitors": 19374, "\u0120Sacramento": 19375, "\u0120timely": 19376, "\u0120contempl": 19377, "\u0120positioned": 19378, "\u0120posters": 19379, "phies": 19380, "iovascular": 19381, "void": 19382, "\u0120Fifth": 19383, "\u0120investigative": 19384, "OUN": 19385, "\u0120integrate": 19386, "\u0120INC": 19387, "isha": 19388, "iblings": 19389, "\u0120Request": 19390, "\u0120Rodriguez": 19391, "\u0120slides": 19392, "\u0120DX": 19393, "\u0120feminism": 19394, "\u0120datas": 19395, "\u0120bend": 19396, "irus": 19397, "\u0120Nigeria": 19398, "Fox": 19399, "Change": 19400, "\u0120airplane": 19401, "\u0120Laden": 19402, "\u0120publicity": 19403, "ixty": 19404, "\u0120commitments": 19405, "\u0120aggregate": 19406, "\u0120displaying": 19407, "\u0120Arrow": 19408, "\u0120122": 19409, "\u0120respects": 19410, "android": 19411, "six": 19412, "\u0120Sha": 19413, "\u0120restoration": 19414, ")\\": 19415, "WS": 19416, "oys": 19417, "\u0120illustrate": 19418, "without": 19419, "126": 19420, "\u0120\u00e2\u0136\u0124": 19421, "\u0120pickup": 19422, "nels": 19423, "\u0120....": 19424, "food": 19425, "\u0120Fen": 19426, ")?": 19427, "\u0120phenomena": 19428, "\u0120companions": 19429, "\u0120Write": 19430, "\u0120spill": 19431, "\u0120bridges": 19432, "\u0120Updated": 19433, "\u0120Fo": 19434, "\u0120insects": 19435, "ASHINGTON": 19436, "\u0120scare": 19437, "iltr": 19438, "\u0120Zhang": 19439, "\u0120severity": 19440, "\u0120indul": 19441, "149": 19442, "\u0120Coffee": 19443, "\u0120norms": 19444, "\u0120pulse": 19445, "\u0120FT": 19446, "\u0120horrific": 19447, "\u0120Destroy": 19448, "\u0120JSON": 19449, "\u0120olive": 19450, "\u0120discusses": 19451, "Rest": 19452, "Elect": 19453, "\u0120Winn": 19454, "\u0120Surviv": 19455, "\u0120Hait": 19456, "Sure": 19457, "oped": 19458, "\u0120rooted": 19459, "\u0120Ske": 19460, "\u0120Bronze": 19461, "\u0120lol": 19462, "Default": 19463, "\u0120commodity": 19464, "redited": 19465, "\u0120libertarian": 19466, "\u0120forbidden": 19467, "\u0120gran": 19468, "\u00e0\u00a8": 19469, "\u0120lag": 19470, "enz": 19471, "drive": 19472, "\u0120mathematics": 19473, "\u0120wires": 19474, "\u0120critically": 19475, "\u0120carbohyd": 19476, "\u0120Chancellor": 19477, "\u0120Eddie": 19478, "\u0120banning": 19479, "\u0120Fri": 19480, "\u0120complications": 19481, "etric": 19482, "\u0120Bangladesh": 19483, "\u0120bandwidth": 19484, "Stop": 19485, "\u0120Originally": 19486, "\u0120halfway": 19487, "ynasty": 19488, "shine": 19489, "\u0120tales": 19490, "rities": 19491, "avier": 19492, "\u0120spinning": 19493, "\u0120WHO": 19494, "\u0120neighbourhood": 19495, "bach": 19496, "\u0120commerce": 19497, "\u0120Sle": 19498, "BU": 19499, "\u0120entrepreneur": 19500, "\u0120peculiar": 19501, "\u0120Comments": 19502, "fre": 19503, "320": 19504, "ICS": 19505, "\u0120imagery": 19506, "\u0120Canon": 19507, "\u0120Electronic": 19508, "short": 19509, "((": 19510, "Dig": 19511, "\u0120commem": 19512, "uced": 19513, "\u0120inclined": 19514, "\u0120Summon": 19515, "\u0120cliff": 19516, "\u0120Mediterranean": 19517, "\u0120poetry": 19518, "\u0120prosperity": 19519, "\u0120Rece": 19520, "\u0120pills": 19521, "member": 19522, "\u0120finale": 19523, "unc": 19524, "\u0120Gig": 19525, "\u00e4\u00bd": 19526, "\u0120lod": 19527, "\u0120backward": 19528, "-+": 19529, "\u0120Forward": 19530, "\u0120thri": 19531, "sure": 19532, "\u0120soap": 19533, "\u0120FX": 19534, "RES": 19535, "\u0120Sexual": 19536, "oulos": 19537, "\u0120foolish": 19538, "\u0120righteous": 19539, "\u0120coff": 19540, "terrorism": 19541, "ustain": 19542, "oter": 19543, "\u0120abuses": 19544, "next": 19545, "\u0120abusive": 19546, "\u0120thereafter": 19547, "\u0120prohibition": 19548, "\u0120SUP": 19549, "\u0120dip": 19550, "\u0120ripped": 19551, "\u0120inherited": 19552, "\u0120bats": 19553, "stru": 19554, "GT": 19555, "\u0120flawed": 19556, "phabet": 19557, "\u0120fog": 19558, "doors": 19559, "\u0120imaging": 19560, "\u0120digits": 19561, "\u0120Hungary": 19562, "\u0120arrog": 19563, "\u0120teachings": 19564, "\u0120protocols": 19565, "\u0120Banks": 19566, "\u00e0\u00b8": 19567, "pound": 19568, "\u0120Curt": 19569, ".\")": 19570, "./": 19571, "\u0120exemption": 19572, "endix": 19573, "\u0120Mull": 19574, "\u0120improves": 19575, "\u0120Gamer": 19576, "dimensional": 19577, "Icon": 19578, "\u0120Margaret": 19579, "Status": 19580, "dates": 19581, "\u0120intends": 19582, "\u0120depict": 19583, "\u0120parked": 19584, "Joe": 19585, "\u0120Marines": 19586, "chnology": 19587, "!).": 19588, "\u0120judged": 19589, "\u0120weights": 19590, "Ray": 19591, "\u0120apartments": 19592, "hester": 19593, "\u0120reinforce": 19594, "\u0120offender": 19595, "occup": 19596, "\u0120sore": 19597, "ept": 19598, "\u0120PHP": 19599, "\u0120Brow": 19600, "\u0120authorization": 19601, "\u0120Risk": 19602, "\u0120Delaware": 19603, "\u0120QU": 19604, "\u0120notifications": 19605, "\u0120sunlight": 19606, "\u0120exclude": 19607, "dat": 19608, "\u0120mesh": 19609, "\u0120Sudan": 19610, "\u0120belonged": 19611, "\u0120subway": 19612, "\u0120noon": 19613, "\u0120Interior": 19614, "olics": 19615, "\u0120Lakers": 19616, "\u0120coding": 19617, "Disclaimer": 19618, "Calif": 19619, "Old": 19620, "\u0120disl": 19621, "?????": 19622, "\u0120confirms": 19623, "\u0120recruitment": 19624, "\u0120homicide": 19625, "Consider": 19626, "\u0120Jeffrey": 19627, "fty": 19628, "};": 19629, "\u0120objection": 19630, "doing": 19631, "\u0120Leo": 19632, "Want": 19633, "\u0120glow": 19634, "\u0120Clarke": 19635, "\u0120Norman": 19636, "\u0120verification": 19637, "\u0120packet": 19638, "\u0120Formula": 19639, "\u0120plag": 19640, "esville": 19641, "\u0120shouting": 19642, "\u0120ov": 19643, "\u0120REC": 19644, "\u0120Bub": 19645, "\u0120ninth": 19646, "\u0120energ": 19647, "\u0120validity": 19648, "\u0120ups": 19649, "jack": 19650, "\u0120neighboring": 19651, "\u0120Nec": 19652, "eworks": 19653, "\u0120Hab": 19654, "arez": 19655, "\u0120spine": 19656, "\u0120eventual": 19657, "\u0120Leaders": 19658, "\u0120Carn": 19659, "\u0120probation": 19660, "\u0120romance": 19661, "msg": 19662, "\u0120Mechanical": 19663, "ERY": 19664, "Rock": 19665, "\u0120partisan": 19666, "Node": 19667, "assets": 19668, "minent": 19669, "\u0120foreigners": 19670, "\u0120testify": 19671, "\u0120Usually": 19672, "lords": 19673, "\u0120Gren": 19674, "\u0120Powell": 19675, "BIL": 19676, "\u0120sr": 19677, "\u0120addict": 19678, "\u0120shells": 19679, "\u0120sigh": 19680, "\u0120Yale": 19681, "ternity": 19682, "\u0120750": 19683, "EU": 19684, "\u0120Rifle": 19685, "\u0120patron": 19686, "ema": 19687, "\u0120Bannon": 19688, "anity": 19689, "\u0120tropical": 19690, "\u0120VII": 19691, "cross": 19692, "Everything": 19693, "\u0120ISO": 19694, "\u0120humble": 19695, "assing": 19696, "\u0120FIG": 19697, "\u0120updating": 19698, "yson": 19699, "\u0120calcium": 19700, "\u0120competent": 19701, "\u0120steering": 19702, "Prot": 19703, "\u0120SY": 19704, "\u0120Finals": 19705, "\u0120Rug": 19706, "159": 19707, "137": 19708, "\u0120Golf": 19709, "\u0120126": 19710, "\u0120accommodation": 19711, "\u0120Hughes": 19712, "\u0120aesthetic": 19713, "artisan": 19714, "\u0120Twilight": 19715, "\u0120prince": 19716, "\u0120Agriculture": 19717, "\u0120Disco": 19718, "\u0120precedent": 19719, "\u0120typing": 19720, "authorized": 19721, "Option": 19722, "\u0120Aub": 19723, "lishes": 19724, "acht": 19725, "mag": 19726, "Peter": 19727, "\u0120UFO": 19728, "monton": 19729, "\u0120Lith": 19730, "\u0120arom": 19731, "\u0120securing": 19732, "\u0120confined": 19733, "private": 19734, "\u0120swords": 19735, "\u0120markers": 19736, "\u0120metabolic": 19737, "select": 19738, "\u0120Curse": 19739, "\u0120Ot": 19740, "gressive": 19741, "\u0120incumb": 19742, "\u0120Saga": 19743, "\u0120priced": 19744, "\u0120clearance": 19745, "Content": 19746, "\u0120drilling": 19747, "\u0120notices": 19748, "\u0120bourgeois": 19749, "\u0120vest": 19750, "\u0120cookie": 19751, "\u0120Guardians": 19752, "rys": 19753, "inyl": 19754, "\u0120124": 19755, "\u0120plausible": 19756, "ongh": 19757, "\u0120Odin": 19758, "\u0120conception": 19759, "\u0120Yuk": 19760, "\u0120Baghdad": 19761, "\u0120Flag": 19762, "Austral": 19763, "\u0120IBM": 19764, "\u0120internationally": 19765, "\u0120WikiLeaks": 19766, "IED": 19767, "\u0120cyn": 19768, "\u0120chooses": 19769, "\u0120Pill": 19770, "\u0120combining": 19771, "\u0120radi": 19772, "\u0120Mohammed": 19773, "defense": 19774, "atching": 19775, "Subject": 19776, "iciency": 19777, "Frame": 19778, "\u0120{\"": 19779, "\u0120chess": 19780, "\u0120timer": 19781, "190": 19782, "\u0120tin": 19783, "\u0120ordinance": 19784, "emetery": 19785, "\u0120accusing": 19786, "\u0120noticeable": 19787, "\u0120centres": 19788, "\u0120lid": 19789, "\u0120Mills": 19790, "imgur": 19791, "\u0120zoom": 19792, "ergic": 19793, "\u0120compression": 19794, "prim": 19795, "find": 19796, "\u0120surg": 19797, "\u0120pand": 19798, "\u0120Kee": 19799, "\u0120Chad": 19800, "cellence": 19801, "oyle": 19802, "\u0120socialism": 19803, "\u0120Travis": 19804, "\u0120MHz": 19805, "\u0120guild": 19806, "ALLY": 19807, "\u0120Subscribe": 19808, "\u0120Related": 19809, "\u0120occurrence": 19810, "itching": 19811, "\u0120fictional": 19812, "\u0120crush": 19813, "\u0120EA": 19814, "cod": 19815, "mix": 19816, "\u0120Triple": 19817, "\u0120retrieve": 19818, "\u0120stimulus": 19819, "\u0120psychiat": 19820, "\u0120Door": 19821, "\u0120homosexuality": 19822, "\u0120elementary": 19823, "\u0120cellular": 19824, "idian": 19825, "\u0120Laun": 19826, "\u0120intriguing": 19827, "\u0120foam": 19828, "\u0120Bass": 19829, "idi": 19830, "itsu": 19831, "\u0120assure": 19832, "\u0120congrat": 19833, "\u0120businessman": 19834, "\u0120Boost": 19835, "close": 19836, "\u0120lied": 19837, "\u0120sciences": 19838, "\u0120Omega": 19839, "\u0120Graphics": 19840, "\u0120<=": 19841, "spoken": 19842, "\u0120connectivity": 19843, "Saturday": 19844, "\u0120Avengers": 19845, "\u0120toggle": 19846, "\u0120ankle": 19847, "\u0120nationalist": 19848, "model": 19849, "\u0120Pool": 19850, "ophobia": 19851, "Var": 19852, "\u0120Mons": 19853, "atories": 19854, "\u0120aggressively": 19855, "Clear": 19856, "Forge": 19857, "acters": 19858, "\u0120hedge": 19859, "\u0120pipes": 19860, "\u0120blunt": 19861, "\u0120sq": 19862, "\u0120remotely": 19863, "Wed": 19864, "asers": 19865, "\u0120refriger": 19866, "\u0120tiles": 19867, "\u0120rescued": 19868, "\u0120comprised": 19869, "insky": 19870, "\u0120manif": 19871, "avanaugh": 19872, "\u0120prolifer": 19873, "\u0120aligned": 19874, "xml": 19875, "\u0120triv": 19876, "\u0120coordination": 19877, "\u0120PER": 19878, "\u0120Quote": 19879, "134": 19880, "bf": 19881, "\u0120Saw": 19882, "\u0120termination": 19883, "\u0120190": 19884, "\u0120additions": 19885, "\u0120trio": 19886, "\u0120projections": 19887, "\u0120positively": 19888, "\u0120inclusive": 19889, "\u0120membr": 19890, "1990": 19891, "older": 19892, "\u0120practiced": 19893, "inkle": 19894, "Arch": 19895, "\u0120starters": 19896, "arius": 19897, "\u0120intermediate": 19898, "\u0120Benef": 19899, "\u0120Killer": 19900, "\u0120interventions": 19901, "\u0120Kil": 19902, "\u0120Flying": 19903, "Inv": 19904, "\u0120premature": 19905, "\u0120psychiatric": 19906, "\u0120indie": 19907, "\u0120collar": 19908, "\u0120Rainbow": 19909, "afi": 19910, "\u0120disruption": 19911, "\u0120FOX": 19912, "casting": 19913, "\u0120misdem": 19914, "cro": 19915, "\u0120wipe": 19916, "ardon": 19917, "\u0120bast": 19918, "\u0120Tommy": 19919, "\u0120Representative": 19920, "\u0120belly": 19921, "\u0120PO": 19922, "\u0120Breitbart": 19923, "132": 19924, "\u0120messaging": 19925, "Should": 19926, "References": 19927, "\u0120GRE": 19928, "istical": 19929, "LP": 19930, "\u0120Cav": 19931, "\u0120Crazy": 19932, "\u0120intuitive": 19933, "keeping": 19934, "\u0120Moss": 19935, "\u0120discontin": 19936, "\u0120Module": 19937, "\u0120unrelated": 19938, "\u0120Practice": 19939, "\u0120Transport": 19940, "\u0120statistically": 19941, "orns": 19942, "\u0120sized": 19943, "pu": 19944, "\u0120caf": 19945, "\u0120Worlds": 19946, "\u0120Rodgers": 19947, "\u0120Lun": 19948, "\u0120Comic": 19949, "living": 19950, "\u0120cared": 19951, "\u0120climbed": 19952, "){": 19953, "\u0120consisted": 19954, "\u0120medieval": 19955, "folk": 19956, "\u0120hacked": 19957, "\u0120dire": 19958, "\u0120Hermione": 19959, "\u0120tended": 19960, "ceans": 19961, "Daniel": 19962, "went": 19963, "\u0120legislators": 19964, "\u0120redes": 19965, "games": 19966, "\u0120gn": 19967, "amiliar": 19968, "\u0120++": 19969, "ggy": 19970, "threat": 19971, "\u0120magnet": 19972, "\u0120perceive": 19973, "\u0120zip": 19974, "\u0120indictment": 19975, "\u0120critique": 19976, "gard": 19977, "\u0120Safe": 19978, "\u0120Cream": 19979, "\u0120advent": 19980, "oba": 19981, "\u0120vowed": 19982, "ousands": 19983, "\u0120ski": 19984, "\u0120abortions": 19985, "uart": 19986, "\u0120stunned": 19987, "\u0120advancing": 19988, "\u0120lacked": 19989, "\u0120\\\"": 19990, "\u0120schizophren": 19991, "\u0120elegant": 19992, "\u0120conferences": 19993, "\u0120canceled": 19994, "\u0120Hudson": 19995, "\u0120Hopefully": 19996, "\u0120trump": 19997, "\u0120frequencies": 19998, "\u0120meteor": 19999, "\u0120Junior": 20000, "\u0120Fleet": 20001, "\u0120Malcolm": 20002, "\u0120Tools": 20003, "\u0120........": 20004, "\u0120hobby": 20005, "\u0120Europeans": 20006, "\u01201500": 20007, "\u0120Into": 20008, "\u0120sway": 20009, "\u0120Appro": 20010, "\u0120Compl": 20011, "Community": 20012, "\u0120tide": 20013, "\u0120Summit": 20014, "\u00e4\u00bb": 20015, "\u0120intervals": 20016, "\u0120Ether": 20017, "\u0120habitat": 20018, "\u0120Stevens": 20019, "lishing": 20020, "\u0120Domain": 20021, "\u0120triggers": 20022, "\u0120chasing": 20023, "\u0120charm": 20024, "\u0120Flower": 20025, "itored": 20026, "\u0120blessing": 20027, "\u0120textures": 20028, "Five": 20029, "\u0120liquor": 20030, "RP": 20031, "FIN": 20032, "\u01201962": 20033, "CAR": 20034, "Unknown": 20035, "\u0120resil": 20036, "\u0120Lily": 20037, "\u0120abundance": 20038, "\u0120predictable": 20039, "rar": 20040, "\u0120bullshit": 20041, "leen": 20042, "chet": 20043, "Mor": 20044, "Much": 20045, "\u00e4\u00b9": 20046, "\u0120emphasized": 20047, "\u0120crust": 20048, "\u0120primitive": 20049, "\u0120enjoyable": 20050, "\u0120Pictures": 20051, "\u0120teammate": 20052, "pler": 20053, "\u0120Tol": 20054, "\u0120Kane": 20055, "\u0120summoned": 20056, "thy": 20057, "rama": 20058, "\u0120Honda": 20059, "\u0120realizing": 20060, "\u0120quicker": 20061, "\u0120concentrate": 20062, "clear": 20063, "\u0120210": 20064, "\u0120Erdogan": 20065, "aris": 20066, "\u0120responds": 20067, "\u0120BI": 20068, "\u0120eligibility": 20069, "\u0120pushes": 20070, "\u0120Idaho": 20071, "\u0120aggrav": 20072, "\u0120ruins": 20073, "urations": 20074, "\u0120bans": 20075, "\u0120anat": 20076, "share": 20077, "\u0120grind": 20078, "hin": 20079, "umen": 20080, "\u0120utilities": 20081, "\u0120Yankees": 20082, "\u0120databases": 20083, "\u0120DD": 20084, "\u0120displaced": 20085, "\u0120dependencies": 20086, "\u0120stimulation": 20087, "hun": 20088, "houses": 20089, "\u0120Pretty": 20090, "\u0120Ravens": 20091, "\u0120TODAY": 20092, "\u0120associates": 20093, "\u0120therape": 20094, "cled": 20095, "\u0120deer": 20096, "\u0120repairs": 20097, "rentice": 20098, "\u0120receptors": 20099, "\u0120remed": 20100, "\u0120Ce": 20101, "\u0120marriages": 20102, "\u0120ballots": 20103, "\u0120Soldier": 20104, "\u0120hilarious": 20105, "opl": 20106, "138": 20107, "\u0120inherently": 20108, "\u0120ignorant": 20109, "\u0120bounce": 20110, "\u0120Easter": 20111, "RELATED": 20112, "\u0120Currency": 20113, "EV": 20114, "\u00e3\u0125\u0140": 20115, "\u0120Lead": 20116, "\u0120deceased": 20117, "Brien": 20118, "\u0120Musk": 20119, "JS": 20120, "\u0120merge": 20121, "hearted": 20122, "creat": 20123, "mitt": 20124, "mund": 20125, "\u0120\u00e2\u0122\u012d": 20126, "\u0120Bag": 20127, "\u0120projection": 20128, "\u0120java": 20129, "\u0120Standards": 20130, "\u0120Leonard": 20131, "\u0120coconut": 20132, "\u0120Population": 20133, "\u0120traject": 20134, "\u0120imply": 20135, "\u0120curiosity": 20136, "\u0120DB": 20137, "\u0120Fresh": 20138, "\u0120Por": 20139, "\u0120heavier": 20140, "neys": 20141, "gomery": 20142, "\u0120deserved": 20143, "\u0120phrases": 20144, "\u0120GC": 20145, "\u0120yeast": 20146, "desc": 20147, "Death": 20148, "\u0120reboot": 20149, "\u0120metadata": 20150, "ICAL": 20151, "\u0120repay": 20152, "\u0120Independence": 20153, "\u0120suburban": 20154, "icals": 20155, "\u0120atop": 20156, "\u0120allocation": 20157, "generation": 20158, "\u0120Gram": 20159, "\u0120moisture": 20160, "\u0120pine": 20161, "\u0120Liberals": 20162, "\u0120aides": 20163, "\u0120underest": 20164, "\u0120Berry": 20165, "\u0120ceremon": 20166, "370": 20167, "astrous": 20168, "\u0120Pirates": 20169, "\u0120tense": 20170, "\u0120Industries": 20171, "\u0120Appeals": 20172, "\u0120Near": 20173, "\u0120\u00e8\u00a3\u0131\u00e7": 20174, "\u0120lovers": 20175, "\u0120CAP": 20176, "\u0120Craw": 20177, "\u0120giants": 20178, "\u0120efficacy": 20179, "Element": 20180, "\u0120Behavior": 20181, "\u0120Toyota": 20182, "\u0120intest": 20183, "Priv": 20184, "AI": 20185, "\u0120maneuver": 20186, "\u0120perfection": 20187, "\u0120bang": 20188, "paper": 20189, "rill": 20190, "George": 20191, "border": 20192, "inters": 20193, "\u0120Seth": 20194, "\u0120clues": 20195, "\u0120Levi": 20196, "\u0120Revenue": 20197, "147": 20198, "\u0120vapor": 20199, "\u0120fortunate": 20200, "\u0120threatens": 20201, "\u0120vet": 20202, "\u0120dependency": 20203, "ersed": 20204, "article": 20205, "\u0120Blizzard": 20206, "\u0120chlor": 20207, "\u0120minus": 20208, "\u0120Bills": 20209, "\u0120cryptocurrency": 20210, "\u0120metabolism": 20211, "tering": 20212, "\u0120pestic": 20213, "steps": 20214, "\u0120Treasure": 20215, "racted": 20216, "\u0120Constant": 20217, "\u0120temp": 20218, "139": 20219, "\u0120Detective": 20220, "urally": 20221, "\u0120recovering": 20222, "\u0120cortex": 20223, "\u0120144": 20224, "closed": 20225, "\u0120prejudice": 20226, "aunted": 20227, "\u0120storms": 20228, "\u0120NOW": 20229, "\u0120machinery": 20230, "Address": 20231, "\u0120compelled": 20232, "270": 20233, "\u0120despair": 20234, "bane": 20235, "\u0120vegetable": 20236, "\u0120beds": 20237, "Learn": 20238, "\u0120colorful": 20239, "\u0120spike": 20240, "\u0120margins": 20241, "\u0120sympathy": 20242, "\u0120workshop": 20243, "\u0120CBC": 20244, "Sat": 20245, "\u0120burns": 20246, "\u0120Gender": 20247, "\u0120129": 20248, "\u0120Cable": 20249, "\u0120debts": 20250, "\u0120Theresa": 20251, "\u0120reflecting": 20252, "\u0120airst": 20253, "\u0120rim": 20254, "ramid": 20255, "\u0120weaknesses": 20256, "Writ": 20257, "oggle": 20258, "ti": 20259, "\u0120Charge": 20260, "\u0120weighed": 20261, "\u0120(.": 20262, "\u0120laughter": 20263, "\u0120router": 20264, "\u0120Democracy": 20265, "Dear": 20266, "\u0120hasht": 20267, "\u0120dy": 20268, "\u0120hints": 20269, "running": 20270, "\u0120finishes": 20271, "arus": 20272, "Mass": 20273, "result": 20274, "ascus": 20275, "\u0120vintage": 20276, "\u0120conqu": 20277, "\u0120wildly": 20278, "acist": 20279, "\u0120lingu": 20280, "\u0120protagonist": 20281, "strom": 20282, "teenth": 20283, "\u0120Solo": 20284, "mac": 20285, "filled": 20286, "\u0120renown": 20287, "itives": 20288, "\u0120motive": 20289, "\u0120Antar": 20290, "\u0120Mann": 20291, "\u0120Adjust": 20292, "\u0120rockets": 20293, "\u0120troubling": 20294, "ei": 20295, "\u0120organisms": 20296, "assis": 20297, "Christian": 20298, "\u0120145": 20299, "\u0120Hass": 20300, "\u0120swall": 20301, "\u0120wax": 20302, "\u0120Survival": 20303, "VS": 20304, "\u0120Murd": 20305, "vd": 20306, "standard": 20307, "\u0120dragons": 20308, "\u0120acceleration": 20309, "rational": 20310, "final": 20311, "\u0120paired": 20312, "\u0120Ethereum": 20313, "\u0120interfaces": 20314, "\u0120resent": 20315, "\u0120artifacts": 20316, "\u00c5\u00ab": 20317, "arel": 20318, "\u0120competitor": 20319, "\u0120Nicholas": 20320, "\u0120Surface": 20321, "cpp": 20322, "\u0120Tot": 20323, "\u0120economically": 20324, "\u0120organised": 20325, "\u0120enforced": 20326, "inho": 20327, "\u0120varieties": 20328, "\u0120abdom": 20329, "\u0120Bailey": 20330, "idav": 20331, "\u0120Salv": 20332, "paid": 20333, "\u0120altitude": 20334, "essert": 20335, "\u0120Gutenberg": 20336, "area": 20337, "opoulos": 20338, "\u0120professors": 20339, "iggs": 20340, "\u0120Fate": 20341, "hey": 20342, "\u01203000": 20343, "Dist": 20344, "\u0120twins": 20345, "cill": 20346, "\u0120Maps": 20347, "\u0120traps": 20348, "\u0120weed": 20349, "\u0120Kiss": 20350, "\u0120yoga": 20351, "\u0120recipients": 20352, "\u0120Westminster": 20353, "\u0120pools": 20354, "\u0120Walmart": 20355, "188": 20356, "\u0120Schools": 20357, "attack": 20358, "\u0120ARM": 20359, "paragraph": 20360, "Warning": 20361, "jl": 20362, "\u0120selfish": 20363, "anchez": 20364, "\u0120Heights": 20365, "Fre": 20366, "\u0120Soph": 20367, "\u0120--------------------------------": 20368, "tml": 20369, "333": 20370, "\u0120raids": 20371, "\u0120satellites": 20372, "KEY": 20373, "\u0120lasts": 20374, "\u00d1\u0124": 20375, "Ins": 20376, "\u0120Dame": 20377, "\u0120unpredict": 20378, "///": 20379, "ghai": 20380, "\u0120artillery": 20381, "\u0120cruise": 20382, "\u0120gel": 20383, "\u0120Cabinet": 20384, "\u0120blows": 20385, "\u0120Esp": 20386, "\u0120proximity": 20387, "othe": 20388, "\u0120Skills": 20389, "\u0120Upper": 20390, "obo": 20391, "\u0120NDP": 20392, "\u0120enjoys": 20393, "\u0120repeating": 20394, "\u0120Construction": 20395, "\u0120Questions": 20396, "Hillary": 20397, "\u0120uint": 20398, "\u0120processors": 20399, "\u0120Gibson": 20400, "\u0120Multiple": 20401, "qa": 20402, "\u0120Bom": 20403, "\u0120Miles": 20404, "ventional": 20405, "\u0120hurts": 20406, "skin": 20407, "\u0120AIDS": 20408, "\u0120advisers": 20409, "\u0120Root": 20410, "\u0120methodology": 20411, "\u0120Dale": 20412, "\u0120deton": 20413, "\u0120Knowledge": 20414, "sequently": 20415, "\u0120121": 20416, "\u0120connects": 20417, "Cy": 20418, "\u0120Danger": 20419, "\u0120contributors": 20420, "\u0120Bent": 20421, "\u0120brass": 20422, "\u0120Guns": 20423, "into": 20424, "\u0120Fortune": 20425, "\u0120broker": 20426, "balance": 20427, "\u0120lengths": 20428, "\u0120vic": 20429, "\u0120averaging": 20430, "\u0120appropriately": 20431, "\u0120Camera": 20432, "\u0120sandwich": 20433, "\u0120CDC": 20434, "\u0120coordinate": 20435, "\u0120navig": 20436, "\u0120goodness": 20437, "laim": 20438, "\u0120brake": 20439, "\u0120extremist": 20440, "\u0120Wake": 20441, "\u0120Mend": 20442, "\u0120Tiny": 20443, "\u0120COL": 20444, "\u0120RF": 20445, "\u0120Dual": 20446, "\u0120Wine": 20447, "Case": 20448, "\u0120refined": 20449, "\u0120lamp": 20450, "Lead": 20451, "\u0120bapt": 20452, "\u0120Carb": 20453, "\u0120Sadd": 20454, "\u0120Minneapolis": 20455, "PDF": 20456, "Early": 20457, "\u0120Hidden": 20458, "Its": 20459, "\u0120TIME": 20460, "\u0120pap": 20461, "\u0120commissioned": 20462, "\u0120Few": 20463, "\u0120Colts": 20464, "\u0120Bren": 20465, "\u0120bothered": 20466, "\u0120likewise": 20467, "Exper": 20468, "\u0120Schw": 20469, "cry": 20470, "nn": 20471, "\u0120Mitch": 20472, "imon": 20473, "MG": 20474, "bm": 20475, "UMP": 20476, "rays": 20477, "\u0120registry": 20478, "\u0120270": 20479, "achine": 20480, "rella": 20481, "anting": 20482, "00000": 20483, "\u0120ruined": 20484, "spot": 20485, "\u0120ta": 20486, "\u0120maximize": 20487, "\u0120inconven": 20488, "Dead": 20489, "Human": 20490, "Enabled": 20491, "\u0120Marie": 20492, "\u0120chill": 20493, "\u0120Paradise": 20494, "\u0120starring": 20495, "\u0120Latino": 20496, "\u0120Protocol": 20497, "\u0120EVER": 20498, "\u0120suppliers": 20499, "message": 20500, "\u0120Brock": 20501, "\u0120serum": 20502, "\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a": 20503, "\u0120encomp": 20504, "\u0120ambition": 20505, "uese": 20506, "\u0120arrows": 20507, "Andrew": 20508, "\u0120antenna": 20509, "\u01201961": 20510, "\u0120Bark": 20511, "\u0120bool": 20512, "\u00e3\u0124\u00aa": 20513, "\u0120Storage": 20514, "\u0120railway": 20515, "\u0120tougher": 20516, "\u0120Cad": 20517, "\u0120washing": 20518, "Py": 20519, "']": 20520, "embed": 20521, "\u0120Memphis": 20522, "ackle": 20523, "\u0120famously": 20524, "\u0120Fortunately": 20525, "ovies": 20526, "\u0120mindset": 20527, "\u0120sneak": 20528, "\u0120Dh": 20529, "RAW": 20530, "\u0120Simpson": 20531, "\u0120livest": 20532, "\u0120landmark": 20533, "\u0120cement": 20534, "Low": 20535, "\u0120thrilled": 20536, "\u0120Course": 20537, "inel": 20538, "\u0120chuck": 20539, "idate": 20540, "global": 20541, "\u0120whit": 20542, "\u0120\u00ef\u00bf\u00bd": 20543, "adays": 20544, "ski": 20545, "\u0120SV": 20546, "\u0120viruses": 20547, "306": 20548, "\u0120Respons": 20549, "\u0120theaters": 20550, "\u0120Branch": 20551, "\u0120Geneva": 20552, "\u0120MK": 20553, "\u0120unbeliev": 20554, "\u0120communist": 20555, "Original": 20556, "\u0120Received": 20557, "\u0120Transfer": 20558, "\u0120Arg": 20559, "Input": 20560, "\u0120Strategy": 20561, "\u0120palace": 20562, "thening": 20563, "Dri": 20564, "\u0120sentencing": 20565, "umbnail": 20566, "\u0120pins": 20567, "recy": 20568, "\u0120siblings": 20569, "Getting": 20570, "\u0120BU": 20571, "\u0120Northwest": 20572, "\u0120prolonged": 20573, "\u0120Sakura": 20574, "Comb": 20575, "\u0120Bour": 20576, "\u0120inadequate": 20577, "\u0120Kash": 20578, "\u0120username": 20579, "\u0120Improve": 20580, "\u0120battling": 20581, "\u0120MAC": 20582, "\u0120curriculum": 20583, "\u0120soda": 20584, "\u0120Cannon": 20585, "\u0120sensible": 20586, "spons": 20587, "December": 20588, "\u0120wicked": 20589, "\u0120Pengu": 20590, "\u0120dictators": 20591, "\u0120Hearts": 20592, "ogyn": 20593, "\u0120similarities": 20594, "\u0120Stats": 20595, "\u0120hollow": 20596, "itations": 20597, "\":[": 20598, "\u0120hover": 20599, "\u0120Listen": 20600, "sch": 20601, "Sund": 20602, "\u0120cad": 20603, "\u0120Parks": 20604, "\u0120lur": 20605, "\u0120hype": 20606, "\u0120Lem": 20607, "NAME": 20608, "isure": 20609, "Friday": 20610, "\u0120shoots": 20611, "\u0120closes": 20612, "\u0120db": 20613, "\u0120Ridge": 20614, "\u0120Different": 20615, "\u0120replies": 20616, "\u0120Broadway": 20617, "opers": 20618, "\u0120intoler": 20619, "\u0120Zeus": 20620, "akespe": 20621, "\u0120proprietary": 20622, "\u0120requesting": 20623, "\u0120controllers": 20624, "\u0120MIN": 20625, "imedia": 20626, "becca": 20627, "\u0120expans": 20628, "\u0120oils": 20629, "Bot": 20630, "\u0120Chand": 20631, "\u0120printer": 20632, "\u0120topped": 20633, "\u0120POL": 20634, "\u0120Earlier": 20635, "Social": 20636, "avin": 20637, "\u0120decreases": 20638, "\u0120Seb": 20639, "\u0120specifications": 20640, "\u0120Blast": 20641, "\u0120Kurt": 20642, "\u0120freel": 20643, "Brown": 20644, "\u0120dilig": 20645, "roe": 20646, "\u0120Problem": 20647, "\u0120Quad": 20648, "\u0120decentral": 20649, "\u0120Vector": 20650, "anut": 20651, "\u0120plugins": 20652, "\u0120Gregory": 20653, "\u0120fucked": 20654, "elines": 20655, "\u0120Ambassador": 20656, "take": 20657, "\u0120cleans": 20658, "ongyang": 20659, "Anonymous": 20660, "stro": 20661, "\"}": 20662, "aline": 20663, "\u0120Odd": 20664, "\u0120Eug": 20665, "216": 20666, "\u0120boil": 20667, "\u0120Powers": 20668, "\u0120nurses": 20669, "Obviously": 20670, "\u0120Technical": 20671, "\u0120exceeded": 20672, "ORS": 20673, "\u0120extremists": 20674, "\u0120traces": 20675, "expl": 20676, "\u0120comr": 20677, "\u0120Sach": 20678, ")/": 20679, "\u0120masks": 20680, "\u0120sci": 20681, "Bon": 20682, "\u0120regression": 20683, "wegian": 20684, "\u0120advisor": 20685, "itures": 20686, "\u0120Vo": 20687, "example": 20688, "\u0120Instruct": 20689, "\u0120siege": 20690, "\u0120reductions": 20691, "ptr": 20692, "\u0120statutory": 20693, "\u0120removes": 20694, "\u0120puck": 20695, "redits": 20696, "\u0120bee": 20697, "\u0120salad": 20698, "\u0120promotions": 20699, "\u0120Joshua": 20700, "withstanding": 20701, "ETH": 20702, "\u0120Cha": 20703, "imus": 20704, "\u0120expenditure": 20705, "aunting": 20706, "\u0120delighted": 20707, "\u0120155": 20708, "beh": 20709, "\u0120carpet": 20710, "\u0120Spart": 20711, "\u0120jungle": 20712, "lists": 20713, "\u0120bullying": 20714, "\u0120Nobel": 20715, "\u0120Glen": 20716, "\u0120referenced": 20717, "\u0120introduces": 20718, "sein": 20719, "\u0120chopped": 20720, "glass": 20721, "\u0120Wrest": 20722, "\u0120neutrality": 20723, "\u0120\u00e2\u013b": 20724, "\u0120investigator": 20725, "\u0120shelves": 20726, "\u0120unconstitutional": 20727, "\u0120reproduction": 20728, "\u0120merchant": 20729, "mia": 20730, "\u0120metrics": 20731, "\u0120explosives": 20732, "\u0120Sonia": 20733, "\u0120bodily": 20734, "\u0120thickness": 20735, "\u0120predominantly": 20736, "\u0120Ability": 20737, "\u0120monitored": 20738, "ICH": 20739, "\u0120].": 20740, "\u0120Martinez": 20741, "\u0120visibility": 20742, "\u0120queries": 20743, "\u0120genocide": 20744, "\u0120Warfare": 20745, "Query": 20746, "\u0120studios": 20747, "\u0120embry": 20748, "\u0120corridor": 20749, "\u0120cleaned": 20750, "complete": 20751, "\u0120MH": 20752, "\u0120enrollment": 20753, "INGS": 20754, "\u0120impacted": 20755, "\u0120disastrous": 20756, "\u0120Yun": 20757, "\u0120Claire": 20758, "\u0120Basically": 20759, "yt": 20760, "usterity": 20761, "\u0120indirectly": 20762, "wik": 20763, "\u0120dod": 20764, "\u0120Carr": 20765, "\u0120amp": 20766, "\u0120prohibit": 20767, "\u0120Initial": 20768, "\u0120Rd": 20769, "iji": 20770, "\u0120educate": 20771, "corn": 20772, "iott": 20773, "\u0120Beauty": 20774, "\u0120detective": 20775, "\u0120Conn": 20776, "since": 20777, "\u0120stagger": 20778, "\u0120obese": 20779, "\u0120bree": 20780, "ologic": 20781, "isse": 20782, "walker": 20783, "\u0120blades": 20784, "\u0120lawful": 20785, "func": 20786, "\u0120Behind": 20787, "\u0120appetite": 20788, "\u0120(*": 20789, "\u0120tennis": 20790, "\u0120offspring": 20791, "\u0120jets": 20792, "\u0120structured": 20793, "\u0120aforementioned": 20794, "Nov": 20795, "\u0120scaling": 20796, "fill": 20797, "\u0120stew": 20798, "\u0120curb": 20799, "\u0120Stephan": 20800, "edIn": 20801, "SF": 20802, "obic": 20803, "\u00e9\u0143\u0136": 20804, "oug": 20805, "\u0120MM": 20806, "\u0120genetically": 20807, "opez": 20808, "136": 20809, "\u0120umb": 20810, "ancers": 20811, "\u0120cohort": 20812, "\u0120merchandise": 20813, "\u0120imposing": 20814, "\u0120Legislature": 20815, "\u0120Archive": 20816, "ivia": 20817, "\u0120Naval": 20818, "\u0120offences": 20819, "\u0120miracle": 20820, "\u0120snapped": 20821, "\u0120foes": 20822, "\u0120extensively": 20823, "\u0120Raf": 20824, "\u0120cater": 20825, "edience": 20826, "Kit": 20827, "\u0120Bin": 20828, "\u0120recommends": 20829, "\u0120Cities": 20830, "\u0120rigid": 20831, "\u0120READ": 20832, "\u0120Noble": 20833, "\u0120Tian": 20834, "\u0120certificates": 20835, "antis": 20836, "oiler": 20837, "\u0120Buddhist": 20838, "did": 20839, "\u0120surveyed": 20840, "\u0120downward": 20841, "\u0120prints": 20842, "\u0120Motion": 20843, "ronics": 20844, "\u0120Sans": 20845, "ossibly": 20846, "uctions": 20847, "\u0120colonies": 20848, "\u0120Danish": 20849, "unit": 20850, "\u0120spoil": 20851, "\u0120advisory": 20852, "berries": 20853, "Plan": 20854, "\u0120specification": 20855, "ophers": 20856, "\u0120Resource": 20857, "\u0120shirts": 20858, "prisingly": 20859, "communications": 20860, "\u0120trivial": 20861, "\u0120mentioning": 20862, "isexual": 20863, "\u0120supplements": 20864, "\u0120supervision": 20865, "BP": 20866, "vor": 20867, "\u0120wit": 20868, "\u0120cooldown": 20869, "\u0120plaintiff": 20870, "\u0120Reviews": 20871, "\u0120Sri": 20872, "\u0120Mint": 20873, "\u0120Sugar": 20874, "\u0120afterward": 20875, "\u0120Priest": 20876, "\u0120Investment": 20877, "ogene": 20878, "\u0120Taking": 20879, "\u0120stretching": 20880, "\u0120inflammation": 20881, "\u0120Tehran": 20882, "\u0120lining": 20883, "\u0120freezing": 20884, "\u0120Entity": 20885, "\u0120inspiring": 20886, "special": 20887, "price": 20888, "\u0120sue": 20889, "\u0120Porter": 20890, "ounge": 20891, "ETA": 20892, "\u0120Derek": 20893, "\u0120Luis": 20894, "uo": 20895, "ymph": 20896, "\u0120exterior": 20897, "ihil": 20898, "\u0120Ashley": 20899, "inator": 20900, "\u0120nutrients": 20901, "\u0120Thrones": 20902, "\u0120finances": 20903, "\u0120Inspect": 20904, "\u0120specially": 20905, "\u0120Required": 20906, "\u0120PTS": 20907, "\u0120Violence": 20908, "ointed": 20909, "shots": 20910, "\u0120excerpt": 20911, "coon": 20912, "INS": 20913, "\u0120Gri": 20914, "\u0120recognised": 20915, "Week": 20916, "Young": 20917, "\u0120vom": 20918, "isle": 20919, "\u0120Curry": 20920, "\u0120Buddh": 20921, "\u0120notebook": 20922, "\u0120durable": 20923, "/?": 20924, "\u0120Gad": 20925, "\u0120Pupp": 20926, "\u0120forgive": 20927, "park": 20928, "\u0120personalities": 20929, "analysis": 20930, "clamation": 20931, "\u0120elevator": 20932, "\u0120warehouse": 20933, "\u0120Role": 20934, "unn": 20935, "\u0120illustration": 20936, "\u0120Scan": 20937, "\u0120atmospheric": 20938, "Import": 20939, "ANC": 20940, "ricted": 20941, "fu": 20942, "010": 20943, "\u0120arche": 20944, "\u0120rewarded": 20945, "akespeare": 20946, "\u0120internally": 20947, "\u0120RBI": 20948, "alker": 20949, "\u0120elephant": 20950, "owitz": 20951, "\u0120Pizza": 20952, "\u0120bipartisan": 20953, "\u00c3\u00a9s": 20954, "\u0120slowed": 20955, "\u0120Stark": 20956, "\u0120override": 20957, "OUS": 20958, "\u0120320": 20959, "undreds": 20960, "\u0120Deck": 20961, "\u0120Census": 20962, "bee": 20963, "146": 20964, "otor": 20965, "\u0120ip": 20966, "\u0120ub": 20967, "ocations": 20968, "\u0120Button": 20969, "rice": 20970, "\u0120cripp": 20971, "fff": 20972, "\u0120originated": 20973, "\u0120overwhelmed": 20974, "appa": 20975, "\u0120foremost": 20976, "\u00e2\u0122\u0133": 20977, "\u0120LEG": 20978, "release": 20979, "eatured": 20980, "atches": 20981, "\u0120reps": 20982, "\u0120lending": 20983, "\u0120Reference": 20984, "\u0120Client": 20985, "165": 20986, "venth": 20987, "Complete": 20988, "\u0120Patrol": 20989, "\u0120sworn": 20990, "cam": 20991, "\u0120shuttle": 20992, "\u0120Ralph": 20993, "\u0120hometown": 20994, "-,": 20995, "onal": 20996, "\u0120BP": 20997, "\u00e5\u0131": 20998, "\u0120persuade": 20999, "\u0120Alexand": 21000, "\u0120combines": 21001, "\u0120vivid": 21002, "\u0120Lag": 21003, "\u0120encoding": 21004, "\u0120salvation": 21005, "wen": 21006, "\u0120Recovery": 21007, "iya": 21008, "University": 21009, "\u0120Biden": 21010, "\u0120budgets": 21011, "\u0120Texans": 21012, "fits": 21013, "\u0120honored": 21014, "\u0120python": 21015, "TD": 21016, "###": 21017, "clone": 21018, "\u0120blink": 21019, "\u0120Liquid": 21020, "\u0120unemployed": 21021, "\u0120clashes": 21022, "\u0120Counsel": 21023, "\u0120directing": 21024, "\u0120punct": 21025, "\u0120Falcons": 21026, "\u0120shark": 21027, "\u0120Damascus": 21028, "\u0120jeans": 21029, "\u0120embark": 21030, "\u0120seize": 21031, "\u0120upwards": 21032, "280": 21033, "\u0120Ez": 21034, "\u0120Anything": 21035, "\u0120exotic": 21036, "lower": 21037, "\u0120Creator": 21038, "\u0120Um": 21039, "\u0120suburbs": 21040, "berger": 21041, "\u0120Wend": 21042, "\u0120mint": 21043, "\u0120XX": 21044, "\u0120Dro": 21045, "\u0120suffers": 21046, "\u0120herb": 21047, "tree": 21048, "\u0120fragile": 21049, "\u0120flooded": 21050, "\u0120Alcohol": 21051, "olean": 21052, "nyder": 21053, "\u0120KO": 21054, "Fram": 21055, "\u0120136": 21056, "\u0120owed": 21057, "\u0120Melee": 21058, "\u0120Hash": 21059, "\u0120whisk": 21060, "\u0120sudo": 21061, "rr": 21062, "Quick": 21063, "appro": 21064, "\u0120ii": 21065, "\u0120Examples": 21066, "hee": 21067, "\u0120promotes": 21068, "perature": 21069, "kar": 21070, "\u0120Honor": 21071, "\u0120sodium": 21072, "\u0120Lif": 21073, "rosso": 21074, "intendent": 21075, "\u0120correspondent": 21076, "Found": 21077, "secret": 21078, "\u0120identifies": 21079, "agne": 21080, "\u0120lou": 21081, "\u0120PP": 21082, "\u0120coincidence": 21083, "move": 21084, "\u0120militia": 21085, "\u0120infiltr": 21086, "\u0120Primary": 21087, "\u0120pitching": 21088, "\u0120Ib": 21089, "\u0120GOOD": 21090, "\u00e3\u0124\u00b8": 21091, "\u0120Wizards": 21092, "iral": 21093, "\u0120Venus": 21094, "RR": 21095, "\u0120\u00e2\u0122\u0137": 21096, "\u0120Casey": 21097, "\u0120sadly": 21098, "\u0120admire": 21099, "\u0120embarrassed": 21100, "cb": 21101, "Mel": 21102, "\u0120tubes": 21103, "\u0120beautifully": 21104, "\u0120Queensland": 21105, "Below": 21106, "rez": 21107, "quet": 21108, "pleasant": 21109, "\u0120\u00c2\u00ab": 21110, "Camp": 21111, "\u0120decisive": 21112, "1998": 21113, "\u0120Lamb": 21114, "utton": 21115, "hn": 21116, "\u0120Jagu": 21117, "aunder": 21118, "\u0120Cord": 21119, "\u0120clerk": 21120, "\u0120caffe": 21121, "\u0120wiped": 21122, "\u0120reim": 21123, "\u0120Mountains": 21124, "\u0120imprisoned": 21125, "\u0120develops": 21126, "\u0120Pra": 21127, "\u0120modeling": 21128, "Anyone": 21129, "ancel": 21130, "\u0120Sit": 21131, "\u0120shields": 21132, "\u0120lawn": 21133, "\u0120cardiovascular": 21134, "\u0120demonstrating": 21135, "\u0120parse": 21136, "\u0120Israelis": 21137, "\u0120euros": 21138, "143": 21139, "\u0120glorious": 21140, "inski": 21141, "ecd": 21142, "\u0120conditioning": 21143, "\u0120helpless": 21144, "\u0120microsc": 21145, "\u0120Harbor": 21146, "\u0120stakes": 21147, "\u0120260": 21148, "\u0120unequ": 21149, "\u0120Floyd": 21150, "\u0120damp": 21151, "\u0120apparatus": 21152, "\u0120Laws": 21153, "\u0120counters": 21154, "\u0120induce": 21155, "atable": 21156, "\u0120Ahmed": 21157, "\u0120slam": 21158, "November": 21159, "\u0120persist": 21160, "\u0120imminent": 21161, "\u00c3\u00a1n": 21162, "\u0120shred": 21163, "\u0120phases": 21164, "\u0120Edmonton": 21165, "\u0120Armstrong": 21166, "\u0120Meet": 21167, "\u0120Kitty": 21168, "\u00d1\u0122": 21169, "circ": 21170, "\u0120Adult": 21171, "\u0120arose": 21172, "\u0120Xen": 21173, "Dan": 21174, "gow": 21175, "\u0120superf": 21176, "\u0120Admir": 21177, "\u0120endure": 21178, "\u0120keyword": 21179, "yrus": 21180, "\u0120yarn": 21181, "\u0120pathway": 21182, "\u0120Hopkins": 21183, "midt": 21184, "\u0120censorship": 21185, "dependent": 21186, "\u0120instructor": 21187, "Sources": 21188, "\u0120toe": 21189, "\u0120balloon": 21190, "Nob": 21191, "\u0120swear": 21192, "\u0120Castro": 21193, "\u0120gloss": 21194, "\u0120Kavanaugh": 21195, "\u0120remarkably": 21196, "Photos": 21197, "\u0120Nom": 21198, "\u0120Southeast": 21199, "yers": 21200, "\u0120validation": 21201, "\u0120cannon": 21202, "\u0120Victory": 21203, "\u0120Pierre": 21204, "\u0120cautious": 21205, "Audio": 21206, "\u0120fetch": 21207, "\u0120Gift": 21208, "\u0120Hyp": 21209, "\u0120remedy": 21210, "ZE": 21211, "\u0120scent": 21212, "\u0120beard": 21213, "\u0120Rut": 21214, "-\"": 21215, "\u0120patents": 21216, "Hy": 21217, "\u0120unjust": 21218, "\u0120potato": 21219, "\u0120forthcoming": 21220, "\u0120chef": 21221, "\u0120Rift": 21222, "affe": 21223, "\u0120ROM": 21224, "\u0120Launch": 21225, "\u0120pads": 21226, "\u0120Neo": 21227, "\u0120onset": 21228, "\u0120squeeze": 21229, "safe": 21230, "\u0120prefix": 21231, "\u0120TM": 21232, "\u0120Nearly": 21233, "\u0120Clinical": 21234, "\u0120Mental": 21235, "otiation": 21236, "\u0120Unic": 21237, "antry": 21238, "\u0120Cir": 21239, "\u0120epit": 21240, "\u00c3\u00a6": 21241, "\u0120extracted": 21242, "versely": 21243, "riad": 21244, "\u0120strains": 21245, "\u0120tops": 21246, "\u0120poem": 21247, "\u0120Randy": 21248, "\u0120Maple": 21249, "THER": 21250, "upiter": 21251, "\u0120SSD": 21252, "\u013c\u00e9": 21253, "\u0120uncon": 21254, "pering": 21255, "\u0120slept": 21256, "iners": 21257, "\u0120underwater": 21258, "\u0120Evidence": 21259, "gone": 21260, "205": 21261, "\u0120historians": 21262, "\u0120synthesis": 21263, "\u0120frog": 21264, "basketball": 21265, "\u0120vibrant": 21266, "\u0120subord": 21267, "\u0120365": 21268, "\u0120Dial": 21269, "\u0120cooperate": 21270, "HAHA": 21271, "\u0120greeted": 21272, "158": 21273, "\u0120jazz": 21274, "\u0120intox": 21275, "\u0120Walking": 21276, "\u0120supervisor": 21277, "\u0120Fusion": 21278, "\u0120Mercedes": 21279, "send": 21280, "Ham": 21281, "sd": 21282, "nl": 21283, "\u0120tours": 21284, "\u0120FIFA": 21285, "\u0120culp": 21286, "gd": 21287, "304": 21288, "\u0120pleas": 21289, "\u0120illustrates": 21290, "\u0120Colombia": 21291, "\u0120highlighting": 21292, "\u0120Summary": 21293, "\u0120exposing": 21294, "\u0120Dru": 21295, "\u0120irony": 21296, "ritional": 21297, "\u0120Carroll": 21298, "\u0120Ellis": 21299, "Pict": 21300, "\u0120Rapt": 21301, "\u0120adapter": 21302, "\u0120unm": 21303, "\u0120corpse": 21304, "\u0120celebrities": 21305, "Den": 21306, "atum": 21307, "\u0120Apocalypse": 21308, "\u0120Wag": 21309, "lining": 21310, "\u0120hormones": 21311, "Rub": 21312, "\u0120Xi": 21313, "\u0120Vaults": 21314, "208": 21315, "alkyrie": 21316, "inosaur": 21317, "\u0120feeds": 21318, "vity": 21319, "\u0120defeating": 21320, "Wait": 21321, "\u0120emphasize": 21322, "\u0120Steelers": 21323, "yrinth": 21324, "leys": 21325, "\u0120Whenever": 21326, "Currently": 21327, "\u0120Clock": 21328, "\u0120collectively": 21329, "anyon": 21330, "\u0120JP": 21331, "\u0120mentality": 21332, "\u0120downloads": 21333, "\u0120surroundings": 21334, "\u0120Barnes": 21335, "\u0120flagship": 21336, "\u0120indicators": 21337, "\u0120grapp": 21338, "January": 21339, "\u0120Elemental": 21340, "\u0120Athena": 21341, "ibal": 21342, "\u0120sights": 21343, "\u0120capita": 21344, "\u0120Treaty": 21345, "\u0120voiced": 21346, "\u0120Gaz": 21347, "lette": 21348, "\u0120ya": 21349, "\u0120expired": 21350, "Legend": 21351, "Hot": 21352, "nature": 21353, "\u0120unstable": 21354, "\u0120280": 21355, "\u00c3\u00ba": 21356, "Comment": 21357, "ALE": 21358, "\u0120quests": 21359, "\u0120handler": 21360, "nis": 21361, "\u0120versatile": 21362, "\u0120conceal": 21363, "engeance": 21364, "\u0120Interactive": 21365, "\u0120obsessed": 21366, "\u0120Dogs": 21367, "\u0120cracked": 21368, "Sound": 21369, "sv": 21370, "\u0120Dylan": 21371, "roads": 21372, "fx": 21373, "\u0120Catholics": 21374, "\u0120Hag": 21375, "\u0120slammed": 21376, "\u0120glowing": 21377, "sale": 21378, "\u0120tissues": 21379, "\u0120Chi": 21380, "nee": 21381, "\u0120cher": 21382, "sic": 21383, "urrection": 21384, "\u0120bacon": 21385, "ulatory": 21386, ").\"": 21387, "\u0120irregular": 21388, "FORM": 21389, "assed": 21390, "\u0120intentional": 21391, "\u0120compensate": 21392, "\u0120Speaking": 21393, "\u0120Sets": 21394, "153": 21395, "\u0120conventions": 21396, "bands": 21397, "emade": 21398, "\u0120ecc": 21399, "\u0120Winston": 21400, "\u0120Assassin": 21401, "\u0120Belgian": 21402, "\u0120dependence": 21403, "\u0120niche": 21404, "\u0120bark": 21405, "\u0120Jazz": 21406, "\u0120disadvantage": 21407, "\u0120gasoline": 21408, "\u0120165": 21409, "\u00e7\u013c\u0126": 21410, "essa": 21411, "module": 21412, "angular": 21413, "OY": 21414, "\u0120Treatment": 21415, "itas": 21416, "olation": 21417, "\u0120Arnold": 21418, "\u0120feud": 21419, "\u0120Nest": 21420, "\u0120theatre": 21421, "ewater": 21422, "\u0120minors": 21423, "olicy": 21424, "\u0120Haven": 21425, "division": 21426, "\u0120trunk": 21427, "Far": 21428, "\u0120Pull": 21429, "\u0120capturing": 21430, "\u01201800": 21431, "\u0120Teen": 21432, "\u0120exempl": 21433, "\u0120clinics": 21434, "\u0120Burg": 21435, "\u0120substit": 21436, "\u0120payload": 21437, "\u0120Lav": 21438, "\u0120Troy": 21439, "\u0120Witness": 21440, "\u0120fragments": 21441, "\u0120passwords": 21442, "\u0120gospel": 21443, "\u0120Gin": 21444, "\u0120tenants": 21445, "olith": 21446, "Six": 21447, "Previous": 21448, "\u0120Ages": 21449, "\u0120Darwin": 21450, "\u0120blat": 21451, "\u0120empathy": 21452, "smith": 21453, "bag": 21454, "\u0120Echo": 21455, "\u0120Camb": 21456, "\u0120Madd": 21457, "\u0120Boo": 21458, "\u0120rede": 21459, "\u0120Burning": 21460, "\u0120smoothly": 21461, "\u0120Adrian": 21462, "\u0120Vampire": 21463, "\u0120Monsters": 21464, "steam": 21465, "Style": 21466, "Ma": 21467, "rea": 21468, "\u0120Dwar": 21469, "alyst": 21470, "ursor": 21471, "\u0120elimination": 21472, "\u0120crypto": 21473, "cht": 21474, "\u0120Eternal": 21475, "\u00e2\u0122\u00a6]": 21476, "\u0120Sorce": 21477, "Ill": 21478, "NER": 21479, "\u0120uh": 21480, "Conclusion": 21481, "wage": 21482, "\u0120respir": 21483, "\u0120reminis": 21484, "hetical": 21485, "\u0120gy": 21486, "\u0120utilized": 21487, "icidal": 21488, "\u01201900": 21489, "\u0120hunters": 21490, "\u0120Swan": 21491, "\u0120React": 21492, "\u0120visitor": 21493, "\u0120Thanksgiving": 21494, "308": 21495, "Posts": 21496, "\u0120hips": 21497, "1997": 21498, "omers": 21499, "\u0120knocking": 21500, "\u0120Vehicle": 21501, "\u0120til": 21502, "\u0120138": 21503, "\u0120mi": 21504, "\u0120Investigation": 21505, "\u0120Kenya": 21506, "\u0120casino": 21507, "\u0120motives": 21508, "\u0120regain": 21509, "rex": 21510, "\u0120weekends": 21511, "\u0120stabbed": 21512, "boro": 21513, "\u0120exploited": 21514, "\u0120HAVE": 21515, "\u0120Television": 21516, "cock": 21517, "\u0120preparations": 21518, "\u0120endeav": 21519, "\u0120Remote": 21520, "\u0120Maker": 21521, "\u0120Produ": 21522, "\u0120Evan": 21523, "\u0120informational": 21524, "\u0120Louisville": 21525, "154": 21526, "\u0120Dreams": 21527, "\u0120plots": 21528, "\u0120Runner": 21529, "\u0120hurting": 21530, "\u0120academy": 21531, "\u0120Montgomery": 21532, "nm": 21533, "\u0120Lanc": 21534, "\u0120Alz": 21535, "210": 21536, "elong": 21537, "\u0120retailer": 21538, "\u0120arising": 21539, "\u0120rebellion": 21540, "\u0120blonde": 21541, "played": 21542, "\u0120instrumental": 21543, "Cross": 21544, "\u0120retention": 21545, "\u0120therapeutic": 21546, "\u0120seas": 21547, "\u0120infantry": 21548, "\u0120Clint": 21549, "\u0120prompting": 21550, "\u0120bitch": 21551, "\u0120stems": 21552, "\u0120Kra": 21553, "\u0120thesis": 21554, "\u0120Bog": 21555, "rued": 21556, "\u0120kings": 21557, "\u0120clay": 21558, "ificent": 21559, "\u0120YES": 21560, "\u0120Thing": 21561, "\u0120Cubs": 21562, "veyard": 21563, "elsh": 21564, "inarily": 21565, "\u0120Ey": 21566, "\u0120Rolling": 21567, "\u0120evolving": 21568, "India": 21569, "\u0120recognizes": 21570, "\u0120graduation": 21571, "isers": 21572, "\u0120fertility": 21573, "\u0120Milan": 21574, "Command": 21575, "\u0120boxing": 21576, "\u01201943": 21577, "\u0120gluten": 21578, "\u0120Emir": 21579, "\u0120idol": 21580, "\u0120conceived": 21581, "\u0120Creation": 21582, "Merit": 21583, "uddy": 21584, "ussions": 21585, "\u0120Lieutenant": 21586, "ietal": 21587, "\u0120unchanged": 21588, "\u0120Scale": 21589, "\u0120Crimea": 21590, "balls": 21591, "atorial": 21592, "\u0120depths": 21593, "\u0120empirical": 21594, "\u0120transm": 21595, "\u0120unsafe": 21596, "missible": 21597, "comfort": 21598, "156": 21599, "\u0120mechanic": 21600, "002": 21601, "lins": 21602, "\u0120smoked": 21603, "Pos": 21604, "\u0120slowing": 21605, "\u0120lav": 21606, "Texas": 21607, "\u0120cheating": 21608, "\u0120Metropolitan": 21609, "ethyl": 21610, "\u0120discovering": 21611, "asse": 21612, "\u0120pencil": 21613, "\u0120Pyongyang": 21614, "\u0120closet": 21615, "\u0120Sheet": 21616, "\u0120Entry": 21617, "oustic": 21618, "\u0120myst": 21619, "erate": 21620, "ariat": 21621, "\u0120minerals": 21622, "\u0120musician": 21623, "\u0120Pul": 21624, "\u0120Maz": 21625, "249": 21626, "\u0120permissions": 21627, "\u0120iv": 21628, "enary": 21629, "ickers": 21630, "\u0120Bing": 21631, "hea": 21632, "enable": 21633, "\u0120griev": 21634, "\u0120asserted": 21635, "\u0120Colonel": 21636, "\u0120affidav": 21637, "wo": 21638, "\u0120seated": 21639, "\u0120Ride": 21640, "\u0120paintings": 21641, "\u0120Pix": 21642, "\u0120137": 21643, "ishi": 21644, "umbai": 21645, "gotten": 21646, "\u0120Earl": 21647, "\u0120inning": 21648, "\u0120census": 21649, "\u0120travelled": 21650, "\u0120Consult": 21651, "185": 21652, "bind": 21653, "\u0120simplicity": 21654, "\u0120overlooked": 21655, "\u0120Helpful": 21656, "\u0120monkey": 21657, "\u0120overwhelmingly": 21658, "Blood": 21659, "\u0120Flint": 21660, "\u0120Jama": 21661, "\u0120Present": 21662, "\u0120Rage": 21663, "\u0120TA": 21664, "ptive": 21665, "\u0120turnout": 21666, "wald": 21667, "\u0120Dolphins": 21668, "\u0120VPN": 21669, "\u0120onion": 21670, "\u0120crafting": 21671, "mma": 21672, "\u0120Mercury": 21673, "\u0120arrange": 21674, "\u0120alerts": 21675, "\u0120OT": 21676, "zbollah": 21677, "\u0120gases": 21678, "\u0120Richardson": 21679, "sal": 21680, "lar": 21681, "\u0120frost": 21682, "\u0120lowering": 21683, "\u0120acclaim": 21684, "\u0120startups": 21685, "\u0120Gain": 21686, "essment": 21687, "\u0120guardian": 21688, "\u00e4\u00ba\u00ba": 21689, "\u0120Pie": 21690, "\u0120Links": 21691, "\u0120merits": 21692, "\u0120awake": 21693, "\u0120parental": 21694, "\u0120exceeds": 21695, "\u0120idle": 21696, "\u0120Pilot": 21697, "\u0120eBay": 21698, "\u0120Accept": 21699, "ipeg": 21700, "Cam": 21701, "\u0120Kot": 21702, "\u0120traders": 21703, "olitics": 21704, "unker": 21705, "\u0120Pale": 21706, "osi": 21707, "anmar": 21708, "\u01201947": 21709, "\u0120Fell": 21710, "estial": 21711, "itating": 21712, "GF": 21713, "\u0120Sr": 21714, "ifted": 21715, "\u0120connector": 21716, "\u0120Bone": 21717, "illes": 21718, "260": 21719, "hma": 21720, "\u0120overlap": 21721, "\u0120GitHub": 21722, "\u0120cleaner": 21723, "\u0120Baptist": 21724, "\u0120WAS": 21725, "\u0120lungs": 21726, "\u00d1\u0123": 21727, "\u0120BUT": 21728, "\u0120cite": 21729, "\u0120pitched": 21730, "reatment": 21731, "\u0120trophies": 21732, "\u0120Nu": 21733, "386": 21734, "\u0120Pride": 21735, "\u0120attendees": 21736, "[]": 21737, "179": 21738, "\u0120spatial": 21739, "\u0120prizes": 21740, "\u0120Religion": 21741, "\u0120showcase": 21742, "\u0120Category": 21743, "vidia": 21744, "Target": 21745, "Property": 21746, "?,": 21747, "\u0120fusion": 21748, "pie": 21749, "\u0120UCLA": 21750, "\u0120soundtrack": 21751, "\u0120princess": 21752, "\u0120Caval": 21753, "should": 21754, "\u0120limbs": 21755, "Background": 21756, "\u0120lonely": 21757, "\u0120cores": 21758, "\u0120Tail": 21759, "sheet": 21760, "\u0120132": 21761, "Ra": 21762, "\u00e3\u0124\u00ab": 21763, "\u0120Bolt": 21764, "\u0120booked": 21765, "\u0120administer": 21766, "\u0120equals": 21767, "wy": 21768, "\u0120observing": 21769, "\u0120Baron": 21770, "\u0120Adobe": 21771, "\u0120virgin": 21772, "\u0120Socialist": 21773, "Move": 21774, "ghazi": 21775, "\u0120Linda": 21776, "212": 21777, "\u0120brewing": 21778, "\u0120merchants": 21779, "burse": 21780, "\u0120divor": 21781, "\u0120metals": 21782, "\u0120Ner": 21783, "\u0120sums": 21784, "\u0120Enemy": 21785, "\u0120envision": 21786, "\u0120granting": 21787, "\u0120Honey": 21788, "\u0120Skyrim": 21789, "\u0120socio": 21790, "graded": 21791, "\u0120selective": 21792, "WASHINGTON": 21793, "\u01201948": 21794, "\u0120Sirius": 21795, "\u0120Gross": 21796, "activity": 21797, "\u0120Ivan": 21798, "\u0120furious": 21799, "BSD": 21800, "\u0120Previous": 21801, "\u0120responsive": 21802, "\u0120charitable": 21803, "\u0120leaning": 21804, "\u0120Pew": 21805, "\u0120violates": 21806, "\\\\\\\\\\\\\\\\": 21807, "\u0120Coming": 21808, "wire": 21809, "\u0120poet": 21810, "\u0120resolutions": 21811, "command": 21812, "\u0120Portuguese": 21813, "\u0120nickname": 21814, "\u0120deaf": 21815, "February": 21816, "\u0120recognise": 21817, "\u0120entirety": 21818, "\u0120seasonal": 21819, "placed": 21820, "\u0120Telegraph": 21821, "\u0120microphone": 21822, "ouring": 21823, "\u0120grains": 21824, "\u0120governed": 21825, "\u0120postp": 21826, "\u0120Waters": 21827, "inement": 21828, "\u0120undocumented": 21829, "\u0120Comcast": 21830, "\u0120fox": 21831, "\u0120assaults": 21832, "reon": 21833, "many": 21834, "\u0120Jenkins": 21835, "\u0120Anyway": 21836, "\u0120assessments": 21837, "\u0120downs": 21838, "\u0120Mouse": 21839, "\u0120superb": 21840, "kt": 21841, "\u0120Dow": 21842, "\u0120taxation": 21843, "401": 21844, "\u0120smiles": 21845, "\u0120undertaken": 21846, "\u0120exh": 21847, "\u0120enthusiastic": 21848, "\u0120twent": 21849, "\u0120governmental": 21850, "\u0120autonomy": 21851, "\u0120Technologies": 21852, "\u0120Chain": 21853, "\u0120prevalent": 21854, "fb": 21855, "\u0120nicotine": 21856, "ogram": 21857, "job": 21858, "\u0120awaiting": 21859, "\u0120Menu": 21860, "\u0120deputies": 21861, "kov": 21862, "ishops": 21863, "Button": 21864, "\u0120Shanghai": 21865, "\u0120diesel": 21866, "\u0120Duck": 21867, "Ryan": 21868, "\u0120PCs": 21869, "NF": 21870, "jury": 21871, "ente": 21872, "\u0120inaccurate": 21873, "eddy": 21874, "Whatever": 21875, "\u0120showc": 21876, "\u0120Nad": 21877, "odus": 21878, "etr": 21879, "\u0120plaintiffs": 21880, "\u0120WOR": 21881, "\u0120Assange": 21882, "\u0120privat": 21883, "\u0120premiums": 21884, "\u0120tam": 21885, "URL": 21886, "\u0120elites": 21887, "\u0120Ranger": 21888, "ottenham": 21889, "\u0120Hoff": 21890, "\u0120Athens": 21891, "\u0120definite": 21892, "\u0120sighed": 21893, "\u0120evenly": 21894, "211": 21895, "\u0120Amber": 21896, "akia": 21897, "\u0120mailing": 21898, "\u0120crashing": 21899, "\u0120Confederate": 21900, "rugged": 21901, "Wal": 21902, "\u0120Depths": 21903, "\u0120juvenile": 21904, "\u0120reactor": 21905, "Introduction": 21906, "\u0120Deluxe": 21907, "1995": 21908, "\u0120Sanchez": 21909, "\u0120Mead": 21910, "ivable": 21911, ":-": 21912, "\u0120Planning": 21913, "\u0120Trap": 21914, "quin": 21915, "\u0120Protect": 21916, "vered": 21917, "Information": 21918, "\u0120kidney": 21919, "innamon": 21920, "las": 21921, "\u0120policing": 21922, "\u0120tolerate": 21923, "\u0120Qi": 21924, "\u0120biased": 21925, "Fort": 21926, "\u0120Ki": 21927, "save": 21928, "\u0120privileged": 21929, "\u0120beasts": 21930, "\u0120Glas": 21931, "\u0120Cinem": 21932, "\u0120comeback": 21933, "Sunday": 21934, "\u0120extinction": 21935, "hops": 21936, "\u0120transmit": 21937, "\u0120doubles": 21938, "\u0120Flat": 21939, "167": 21940, "\u0120disputed": 21941, "\u0120injustice": 21942, "foo": 21943, "Vict": 21944, "roleum": 21945, "\u0120Julie": 21946, "Context": 21947, "\u0120Rarity": 21948, "issue": 21949, "Component": 21950, "\u0120counseling": 21951, "anne": 21952, "dark": 21953, "\u0120objections": 21954, "uilt": 21955, "\u0120gast": 21956, "\u0120plac": 21957, "\u0120unused": 21958, "\u00e3\u0125\u0129": 21959, "\u0120Trial": 21960, "\u0120Jas": 21961, "hedral": 21962, "obb": 21963, "\u0120temporal": 21964, "\u0120PRO": 21965, "\u0120NW": 21966, "\u0120Anniversary": 21967, "Large": 21968, "\u0120therm": 21969, "\u0120david": 21970, "\u0120systemic": 21971, "\u0120Shir": 21972, "mut": 21973, "\u0120Nept": 21974, "address": 21975, "\u0120scanning": 21976, "\u0120understandable": 21977, "\u0120canvas": 21978, "Cat": 21979, "\u0120Zoo": 21980, "\u0120angels": 21981, "LO": 21982, "\u0120Statement": 21983, "\u0120Sig": 21984, "ovable": 21985, "\u0120Away": 21986, "sharing": 21987, "ocrats": 21988, "stated": 21989, "\u0120weighing": 21990, "Nor": 21991, "wild": 21992, "Bey": 21993, "\u0120astonishing": 21994, "\u0120Reynolds": 21995, "\u0120opener": 21996, "\u0120trainer": 21997, "\u0120surgical": 21998, "pn": 21999, "\u0120adjusting": 22000, "wheel": 22001, "\u0120frown": 22002, "ervative": 22003, "\u0120suspend": 22004, "Within": 22005, "tein": 22006, "\u0120obstacle": 22007, "\u0120liberties": 22008, "ymes": 22009, "\u0120uranium": 22010, "ansom": 22011, "anol": 22012, "uba": 22013, "\u0120Loss": 22014, "\u0120arous": 22015, "\u0120Henderson": 22016, "Wow": 22017, "spl": 22018, "cur": 22019, "\u0120\u00c2\u0143": 22020, "\u0120theirs": 22021, "Damage": 22022, "\u0120downloading": 22023, "\u0120discern": 22024, "\u0120Sto": 22025, "\u0120Fla": 22026, "\u0120hath": 22027, "\u0120Aj": 22028, "\u0120unpleasant": 22029, "European": 22030, "expensive": 22031, "\u0120screenshot": 22032, "\u0120UV": 22033, "\u0120allied": 22034, "\u0120Persian": 22035, "\u0120monopoly": 22036, "\u0120atom": 22037, "\u0120Redskins": 22038, "\"><": 22039, "\u0120cancell": 22040, "\u0120cinema": 22041, "131": 22042, "fair": 22043, "\u0120Alfred": 22044, "\u0120duck": 22045, "args": 22046, "223": 22047, "\u0120ISI": 22048, "\u0120signaling": 22049, "inar": 22050, "\u0120laughs": 22051, "\u0120forwards": 22052, "\u0120reckless": 22053, "\u0120listeners": 22054, "ativity": 22055, "\u0120vastly": 22056, "nant": 22057, "Less": 22058, "\u0120Hunting": 22059, "\u0120Scientific": 22060, "ITED": 22061, "\u0120knight": 22062, "\u0120HTC": 22063, "usa": 22064, "tmp": 22065, "\u0120rude": 22066, "\u0120Legendary": 22067, "\u0120arises": 22068, "Bad": 22069, "\u0120Claim": 22070, "peg": 22071, "\u0120realities": 22072, "Think": 22073, "\u0120\u00c2\u00b0": 22074, "\u0120rode": 22075, "\u0120strive": 22076, "\u0120anecd": 22077, "\u0120shorts": 22078, "\u0120hypothes": 22079, "\u0120coordinated": 22080, "\u0120Gandhi": 22081, "\u0120FPS": 22082, "RED": 22083, "\u0120susceptible": 22084, "\u0120shrink": 22085, "\u0120Chart": 22086, "Help": 22087, "\u0120ion": 22088, "deep": 22089, "ribes": 22090, "\u0120Kai": 22091, "\u0120Customer": 22092, "Summary": 22093, "\u0120cough": 22094, "wife": 22095, "\u0120lend": 22096, "\u0120positioning": 22097, "\u0120lottery": 22098, "\u0120Canyon": 22099, "\u0120fade": 22100, "\u0120bronze": 22101, "\u0120Kenny": 22102, "\u0120boasts": 22103, "\u0120Enhanced": 22104, "record": 22105, "\u0120emergence": 22106, "\u0120akin": 22107, "\u0120Bert": 22108, "itous": 22109, "\u00e2\u0138\u0133": 22110, "\u0120stip": 22111, "\u0120exchanged": 22112, "omore": 22113, "alsh": 22114, "\u0120reservoir": 22115, "\u0120standpoint": 22116, "WM": 22117, "\u0120initiate": 22118, "\u0120decay": 22119, "\u0120brewery": 22120, "\u0120terribly": 22121, "\u0120mortal": 22122, "levard": 22123, "\u0120revis": 22124, "NI": 22125, "elo": 22126, "\u0120confess": 22127, "\u0120MSNBC": 22128, "\u0120submissions": 22129, "Controller": 22130, "\u0120202": 22131, "\u0120Ruth": 22132, "});": 22133, "\u0120Azure": 22134, "\u0120.\"": 22135, "206": 22136, "\u0120Marketing": 22137, "\u0120laund": 22138, "iencies": 22139, "\u0120renowned": 22140, "\u0120Trou": 22141, "\u0120NGO": 22142, "blems": 22143, "\u0120terrified": 22144, "\u0120warns": 22145, "\u0120pert": 22146, "\u0120unsure": 22147, "480": 22148, "alez": 22149, "ultz": 22150, "\u0120Outside": 22151, "\u0120styl": 22152, "\u0120Underground": 22153, "\u0120panc": 22154, "\u0120dictionary": 22155, "\u0120foe": 22156, "riminal": 22157, "\u0120Norwegian": 22158, "\u0120jailed": 22159, "\u0120maternal": 22160, "\u00c3\u00a9e": 22161, "\u0120Lucy": 22162, "cop": 22163, "Cho": 22164, "\u0120unsigned": 22165, "\u0120Zelda": 22166, "\u0120Insider": 22167, "\u0120Continued": 22168, "\u0120133": 22169, "\u0120Naruto": 22170, "\u0120Majority": 22171, "169": 22172, "\u0120Wo": 22173, "\u00e3\u0124\u0135": 22174, "\u0120pastor": 22175, "\u0120informal": 22176, "\u00d0\u00bd": 22177, "anthrop": 22178, "join": 22179, "\u00e3\u0123\u0139": 22180, "itational": 22181, "NP": 22182, "\u0120Writing": 22183, "fn": 22184, "\u0120Bever": 22185, "195": 22186, "\u0120yelling": 22187, "\u0120drastically": 22188, "\u0120eject": 22189, "\u0120neut": 22190, "\u0120thrive": 22191, "\u0120Frequ": 22192, "oux": 22193, "\u0120possesses": 22194, "\u0120Senators": 22195, "\u0120DES": 22196, "\u0120Shakespeare": 22197, "\u0120Franco": 22198, "\u0120LB": 22199, "uchi": 22200, "\u0120incarn": 22201, "\u0120founders": 22202, "Function": 22203, "\u0120brightness": 22204, "\u0120BT": 22205, "\u0120whale": 22206, "\u0120Theater": 22207, "mass": 22208, "\u0120Doll": 22209, "Something": 22210, "\u0120echoed": 22211, "\u0120Hex": 22212, "crit": 22213, "afia": 22214, "\u0120goddess": 22215, "\u0120eleven": 22216, "\u0120Preview": 22217, "\u0120Aurora": 22218, "\u0120401": 22219, "ulsive": 22220, "\u0120Logan": 22221, "inburgh": 22222, "\u0120Centers": 22223, "\u0120ONLY": 22224, "\u0120Aid": 22225, "\u0120paradox": 22226, "\u0120hurd": 22227, "\u0120LC": 22228, "Due": 22229, "court": 22230, "\u0120offended": 22231, "\u0120evaluating": 22232, "\u0120Matthews": 22233, "\u0120tomb": 22234, "\u0120payroll": 22235, "\u0120extraction": 22236, "\u0120Hands": 22237, "ifi": 22238, "\u0120supernatural": 22239, "\u0120COMM": 22240, "]=": 22241, "dogs": 22242, "\u0120512": 22243, "\u0120Meeting": 22244, "Richard": 22245, "\u0120Maximum": 22246, "\u0120ideals": 22247, "Things": 22248, "mand": 22249, "\u0120Regardless": 22250, "\u0120humili": 22251, "buffer": 22252, "Little": 22253, "\u0120Dani": 22254, "\u0120Nak": 22255, "\u0120liberation": 22256, "\u0120Abe": 22257, "\u0120OL": 22258, "\u0120stuffed": 22259, "aca": 22260, "inda": 22261, "raphic": 22262, "\u0120mosqu": 22263, "\u0120campaigning": 22264, "\u0120occupy": 22265, "Squ": 22266, "rina": 22267, "\u0120Wel": 22268, "\u0120VS": 22269, "\u0120physic": 22270, "\u0120puls": 22271, "rint": 22272, "oaded": 22273, "ETF": 22274, "\u0120Archives": 22275, "\u0120venues": 22276, "hner": 22277, "\u0120Turbo": 22278, "\u0120lust": 22279, "\u0120appealed": 22280, "quez": 22281, "ilib": 22282, "\u0120Timothy": 22283, "\u0120omn": 22284, "dro": 22285, "\u0120obsession": 22286, "\u0120Savage": 22287, "1996": 22288, "Global": 22289, "Jes": 22290, "214": 22291, "\u0120sliding": 22292, "\u0120disappro": 22293, "\u0120Magical": 22294, "\u0120voluntarily": 22295, "gb": 22296, "aney": 22297, "\u0120prophet": 22298, "\u0120Rein": 22299, "\u0120Julia": 22300, "\u0120Worth": 22301, "aurus": 22302, "\u0120bounds": 22303, "ieu": 22304, ")))": 22305, "\u0120crore": 22306, "\u0120Citizen": 22307, "Sky": 22308, "\u0120columnist": 22309, "\u0120seekers": 22310, "ondo": 22311, "ISA": 22312, "\u0120Length": 22313, "\u0120nostalg": 22314, "\u0120newcom": 22315, "\u0120detrim": 22316, "entric": 22317, "375": 22318, "\u0120GE": 22319, "\u0120autop": 22320, "\u0120academics": 22321, "AppData": 22322, "\u0120Shen": 22323, "\u0120idiot": 22324, "\u0120Transit": 22325, "\u0120teaspoon": 22326, "Wil": 22327, "KO": 22328, "\u0120Comedy": 22329, ">,": 22330, "\u0120populated": 22331, "WD": 22332, "\u0120pigs": 22333, "\u0120Oculus": 22334, "\u0120sympathetic": 22335, "\u0120marathon": 22336, "198": 22337, "\u0120seizure": 22338, "sided": 22339, "\u0120dop": 22340, "irtual": 22341, "Land": 22342, "\u0120Floor": 22343, "osaurs": 22344, "...]": 22345, "\u0120los": 22346, "\u0120subsidiary": 22347, "EY": 22348, "\u0120Parts": 22349, "\u0120Stef": 22350, "\u0120Judiciary": 22351, "\u0120134": 22352, "\u0120mirrors": 22353, "\u0120ket": 22354, "times": 22355, "\u0120neurolog": 22356, "\u0120cav": 22357, "\u0120Guest": 22358, "\u0120tumor": 22359, "scill": 22360, "\u0120Lloyd": 22361, "Est": 22362, "\u0120clearer": 22363, "\u0120stereotypes": 22364, "\u0120dur": 22365, "nothing": 22366, "Reddit": 22367, "\u0120negotiated": 22368, "------------------------": 22369, "235": 22370, "\u0120flown": 22371, "\u0120Seoul": 22372, "\u0120Resident": 22373, "\u0120SCH": 22374, "\u0120disappearance": 22375, "\u0120Vince": 22376, "grown": 22377, "\u0120grabs": 22378, "ril": 22379, "\u0120Infinite": 22380, "\u0120Twenty": 22381, "\u0120pedestrian": 22382, "\u0120jersey": 22383, "\u0120Fur": 22384, "\u0120Infinity": 22385, "\u0120Elliott": 22386, "\u0120mentor": 22387, "\u0120morally": 22388, "\u0120obey": 22389, "secure": 22390, "iffe": 22391, "\u0120antibiotics": 22392, "angled": 22393, "\u0120Freeman": 22394, "\u0120Introduction": 22395, "Jun": 22396, "\u0120marsh": 22397, "icans": 22398, "\u0120EVENTS": 22399, "ochond": 22400, "Wall": 22401, "iculty": 22402, "\u0120misdemeanor": 22403, "\u0120ly": 22404, "Thomas": 22405, "\u0120Resolution": 22406, "\u0120animations": 22407, "\u0120Dry": 22408, "\u0120intercourse": 22409, "\u0120Newcastle": 22410, "\u0120Hog": 22411, "\u0120Equipment": 22412, "177": 22413, "\u0120territorial": 22414, "\u0120archives": 22415, "203": 22416, "Filter": 22417, "\u0120Munich": 22418, "\u0120commanded": 22419, "\u0120Wand": 22420, "\u0120pitches": 22421, "\u0120Croat": 22422, "\u0120ratios": 22423, "\u0120Mits": 22424, "\u0120accumulated": 22425, "\u0120Specifically": 22426, "\u0120gentleman": 22427, "acerb": 22428, "\u0120penn": 22429, "\u0120aka": 22430, "\u0120Fuk": 22431, "\u0120intervene": 22432, "\u0120Refuge": 22433, "\u0120Alzheimer": 22434, "\u0120succession": 22435, "ohan": 22436, "does": 22437, "Lord": 22438, "\u0120separat": 22439, "\u0120correspondence": 22440, "\u0120shiny": 22441, "Prior": 22442, "\u0120sulf": 22443, "\u0120miserable": 22444, "\u0120dedication": 22445, "().": 22446, "\u0120specialists": 22447, "\u0120defects": 22448, "\u0120Cult": 22449, "\u0120Xia": 22450, "\u0120jeopard": 22451, "\u0120Ore": 22452, "Ability": 22453, "\u0120lear": 22454, "\u0120ambitions": 22455, "\u0120BMI": 22456, "\u0120Arabs": 22457, "\u01201942": 22458, "\u0120preservation": 22459, "ificate": 22460, "\u0120ashamed": 22461, "loss": 22462, "\u0120Restaur": 22463, "\u0120resemble": 22464, "\u0120enrich": 22465, "\u0120KN": 22466, "\u0120Clan": 22467, "float": 22468, "\u0120playable": 22469, "ITT": 22470, "\u0120harmony": 22471, "arrison": 22472, "\u0120Weinstein": 22473, "were": 22474, "\u0120poisoning": 22475, "\u0120Comput": 22476, "\u0120WordPress": 22477, "major": 22478, "\u0120Valve": 22479, "Fan": 22480, "\u0120Throw": 22481, "\u0120Romans": 22482, "\u0120Depression": 22483, "ados": 22484, "\u0120tortured": 22485, "\u0120balancing": 22486, "bottom": 22487, "\u0120acquiring": 22488, "\u0120Monte": 22489, "ardi": 22490, "\u0120aura": 22491, "\u0120##": 22492, "\u0120Standing": 22493, "\u0120Atlas": 22494, "CF": 22495, "\u0120intrins": 22496, "\u0120Benghazi": 22497, "\u0120camping": 22498, "\u0120tapped": 22499, "blade": 22500, "strous": 22501, "\u0120Rabb": 22502, "\u0120Written": 22503, "tip": 22504, "\u0120Neigh": 22505, "sterdam": 22506, "\u0120Allow": 22507, "\u0120Healing": 22508, "\u0120Rhod": 22509, "num": 22510, "\u0120caffeine": 22511, "\u0120Percent": 22512, "\u0120boo": 22513, "\u0120apples": 22514, "305": 22515, "\u0120welcoming": 22516, "\u0120applaud": 22517, "\u0120austerity": 22518, "\u00c2\u00b1": 22519, "\u0120Reality": 22520, "efe": 22521, "\u00e5\u00ae": 22522, "\u0120sucks": 22523, "\u0120tabs": 22524, "\u0120PayPal": 22525, "\u0120backpack": 22526, "\u0120gifted": 22527, "abulary": 22528, "\u0120Scout": 22529, "irteen": 22530, "\u0120chin": 22531, "\u0120omitted": 22532, "\u0120negatively": 22533, "\u0120accessing": 22534, "\u0120Earn": 22535, "\u0120ambulance": 22536, "\u0120headphones": 22537, "\u0120205": 22538, "\u0120Refresh": 22539, "president": 22540, "\u0120Kitchen": 22541, "\u0120Entered": 22542, "\u0120Snyder": 22543, "005": 22544, "omical": 22545, "\u0120borrowed": 22546, "\u0120Nem": 22547, "\u0120aviation": 22548, "\u0120stall": 22549, "rimination": 22550, "\u0120uniforms": 22551, "itime": 22552, "\u0120Simmons": 22553, "energy": 22554, "ablished": 22555, "yy": 22556, "qualified": 22557, "\u0120rallies": 22558, "\u0120Stuart": 22559, "flight": 22560, "\u0120gangs": 22561, "rag": 22562, "\u0120vault": 22563, "lux": 22564, "\u0120Compar": 22565, "\u0120designation": 22566, "209": 22567, "\u0120Jos": 22568, "dollar": 22569, "zero": 22570, "\u0120wells": 22571, "303": 22572, "\u0120constituents": 22573, "\u0120heck": 22574, "\u0120cows": 22575, "\u0120commanders": 22576, "\u0120differential": 22577, "\u0120Catherine": 22578, "299": 22579, "\u0120valve": 22580, "\u0120brace": 22581, "\u0120perspectives": 22582, "cert": 22583, "fact": 22584, "icularly": 22585, "\u0120McN": 22586, "planes": 22587, "\u0120intric": 22588, "\u0120peas": 22589, "ovan": 22590, "\u0120tossed": 22591, "retch": 22592, "\u0120Lopez": 22593, "\u0120unfamiliar": 22594, "death": 22595, "\u0120Apart": 22596, "\u0120Chang": 22597, "\u0120relieved": 22598, "rophe": 22599, "\u0120airports": 22600, "\u0120freak": 22601, "util": 22602, "Mill": 22603, "\u0120Chin": 22604, "\u0120Owen": 22605, "male": 22606, "\u0120Broken": 22607, "\u0120Winds": 22608, "rob": 22609, "rising": 22610, "\u0120firefighters": 22611, "\u0120authoritarian": 22612, "\u0120148": 22613, "Bitcoin": 22614, "external": 22615, "\u0120browsers": 22616, "ichever": 22617, "orian": 22618, "\u0120unb": 22619, "\u0120poke": 22620, "\u0120Zot": 22621, "Mid": 22622, "\u0120Popular": 22623, "\u0120covert": 22624, "\u0120contributes": 22625, "\u0120650": 22626, "\u0120contention": 22627, "Gate": 22628, "\u0120consoles": 22629, "\u0120chromos": 22630, "\u0120IX": 22631, "\u0120visually": 22632, "\u0120Eisen": 22633, "\u0120jewelry": 22634, "\u0120delegation": 22635, "\u0120accelerate": 22636, "\u0120Riley": 22637, "\u0120slope": 22638, "\u0120indoor": 22639, "itially": 22640, "\u0120hugely": 22641, "\u0120tunnels": 22642, "\u0120fined": 22643, "\u0120directive": 22644, "\u0120forehead": 22645, "ustomed": 22646, "\u0120skate": 22647, "Music": 22648, "gas": 22649, "\u0120recognizing": 22650, "ambo": 22651, "\u0120overweight": 22652, "\u0120Grade": 22653, "\u00d9\u012c": 22654, "\u0120sounding": 22655, "\u0120locking": 22656, "\u0120REM": 22657, "Store": 22658, "\u0120excav": 22659, "\u0120Likewise": 22660, "\u0120Lights": 22661, "\u0120elbow": 22662, "\u0120Supply": 22663, "wic": 22664, "\u0120handsome": 22665, "1994": 22666, "Coll": 22667, "\u0120adequately": 22668, "\u0120Associate": 22669, "\u0120strips": 22670, "\u0120crackdown": 22671, "\u0120marvel": 22672, "\u0120Kun": 22673, "\u0120passages": 22674, "@@@@": 22675, "\u0120Tall": 22676, "\u0120thoughtful": 22677, "namese": 22678, "\u0120prostitution": 22679, "business": 22680, "\u0120ballistic": 22681, "personal": 22682, "cig": 22683, "izational": 22684, "Round": 22685, "\u0120\u00c2\u0142\u0120\u00c2\u0142\u0120\u00c2\u0142\u0120\u00c2\u0142": 22686, "\u0120Coleman": 22687, "\u0120admitting": 22688, "\u0120Plug": 22689, "\u0120bitcoins": 22690, "\u0120Suz": 22691, "\u0120fairness": 22692, "\u0120supplier": 22693, "\u0120catastrophic": 22694, "\u0120Helen": 22695, "oqu": 22696, "Marc": 22697, "\u0120Articles": 22698, "gie": 22699, "\u0120endangered": 22700, "\u0120destiny": 22701, "\u0120Volt": 22702, "olia": 22703, "axis": 22704, "\u0120cheat": 22705, "\u0120unified": 22706, "ICO": 22707, "quote": 22708, "302": 22709, "\u0120Sed": 22710, "\u0120suppression": 22711, "\u0120analyzing": 22712, "\u0120squat": 22713, "\u0120figuring": 22714, "\u0120coordinates": 22715, "\u0120chunks": 22716, "\u01201946": 22717, "\u0120subp": 22718, "\u0120wiki": 22719, "\u0120Forbes": 22720, "\u0120Jupiter": 22721, "\u0120Erik": 22722, "imer": 22723, "\u0120Commercial": 22724, "\\)": 22725, "\u0120legitimacy": 22726, "\u0120dental": 22727, "\u0120Mean": 22728, "\u0120deficits": 22729, "550": 22730, "Originally": 22731, "\u0120Horror": 22732, "\u0120contamination": 22733, "llah": 22734, "\u0120confisc": 22735, "\u0120Clare": 22736, "TB": 22737, "\u0120Failed": 22738, "aned": 22739, "\u0120ruler": 22740, "\u0120Controller": 22741, "\u0120feminists": 22742, "Fix": 22743, "gay": 22744, "207": 22745, "\u0120rabbit": 22746, "Third": 22747, "owntown": 22748, "\u0120glue": 22749, "\u0120volatile": 22750, "\u0120shining": 22751, "\u0120foll": 22752, "\u0120impaired": 22753, "\u0120supers": 22754, "\u00e6\u012a": 22755, "\u0120clutch": 22756, "\u013c\u00e9\u0128\u0134": 22757, "\u0120prolet": 22758, "\u0120(!": 22759, "\u0120yelled": 22760, "\u0120Kiev": 22761, "\u0120Ern": 22762, "\u0120Shock": 22763, "KB": 22764, "\u0120situated": 22765, "query": 22766, "\u0120Nas": 22767, "\u0120annex": 22768, "character": 22769, "\u0120Holiday": 22770, "\u0120automation": 22771, "\u0120Jill": 22772, "\u0120Remastered": 22773, "\u0120linem": 22774, "\u0120wilderness": 22775, "\u0120Horizon": 22776, "\u0120Guinea": 22777, "AZ": 22778, "\u0120mainland": 22779, "\u0120secrecy": 22780, "LEASE": 22781, "\u0120punk": 22782, "\u0120Province": 22783, "(),": 22784, "Speed": 22785, "\u0120handing": 22786, "\u0120Sebast": 22787, "Sir": 22788, "rase": 22789, "\u0120journals": 22790, "\u0120congest": 22791, "\u0120Tut": 22792, "irrel": 22793, "\u0120schizophrenia": 22794, "\u0120misogyn": 22795, "healthy": 22796, "Iron": 22797, "\u0120reacted": 22798, "-$": 22799, "252": 22800, "\u0120plural": 22801, "\u0120plum": 22802, "\u0120bargain": 22803, "\u0120grounded": 22804, "finder": 22805, "\u0120disse": 22806, "\u0120Laz": 22807, "OOD": 22808, "\u0120atroc": 22809, "Factory": 22810, "\u0120minions": 22811, "\u0120ori": 22812, "\u0120Brave": 22813, "\u0120PRE": 22814, "\u0120Myanmar": 22815, "\u0120Hod": 22816, "\u0120expedition": 22817, "\u0120explode": 22818, "\u0120Coord": 22819, "\u0120extr": 22820, "\u0120Brief": 22821, "\u0120ADHD": 22822, "\u0120hardcore": 22823, "feeding": 22824, "\u0120dile": 22825, "\u0120Fruit": 22826, "\u0120vaccination": 22827, "\u0120Mao": 22828, "osphere": 22829, "\u0120contests": 22830, "-|": 22831, "\u0120fren": 22832, "isphere": 22833, "Rom": 22834, "\u0120Sharp": 22835, "\u0120Trend": 22836, "\u0120disconnect": 22837, "\u00e2\u0122\u00a2\u00e2\u0122\u00a2": 22838, "\u0120persecution": 22839, "Earth": 22840, "\u0120healthier": 22841, "384": 22842, "\u0120cob": 22843, "\u0120Trinity": 22844, "OWS": 22845, "ANN": 22846, "\u0120specialty": 22847, "\u0120gru": 22848, "\u0120cooperative": 22849, "why": 22850, "Starting": 22851, "\u0120Issues": 22852, "stre": 22853, "ensor": 22854, "\u0120185": 22855, "Adv": 22856, "!?": 22857, "\u0120Revel": 22858, "emia": 22859, "\u0120Hulk": 22860, "\u0120celebrations": 22861, "\u0120Sou": 22862, "raud": 22863, "\u0120Klein": 22864, "\u0120unreal": 22865, "context": 22866, "\u0120partnerships": 22867, "\u0120adopting": 22868, "tical": 22869, "\u0120splash": 22870, "\u0120Hezbollah": 22871, "category": 22872, "cyclop": 22873, "xton": 22874, "\u0120Dot": 22875, "urdy": 22876, "tz": 22877, "\u0120envelope": 22878, "\u0120NL": 22879, "\u00e2\u0137": 22880, "\u0120wherein": 22881, "Spec": 22882, "184": 22883, "\u0120telev": 22884, "aliation": 22885, "\u0120myths": 22886, "\u00e5\u00b0": 22887, "\u0120rigorous": 22888, "\u0120communicating": 22889, "\u0120observer": 22890, "\u0120rehe": 22891, "\u0120Wash": 22892, "\u0120apologized": 22893, "\u0120Tin": 22894, "\u0120expenditures": 22895, "workers": 22896, "document": 22897, "\u0120hesitate": 22898, "\u0120Lenin": 22899, "\u0120unpredictable": 22900, "\u0120renewal": 22901, "cler": 22902, "okia": 22903, "\u0120CONT": 22904, "\u0120postseason": 22905, "Tokens": 22906, "\u0120exacerb": 22907, "\u0120betting": 22908, "\u0120147": 22909, "\u0120elevation": 22910, "Wood": 22911, "\u0120Solomon": 22912, "194": 22913, "004": 22914, "output": 22915, "\u0120redund": 22916, "\u0120Mumbai": 22917, "\u0120pH": 22918, "\u0120reproduce": 22919, "\u0120Duration": 22920, "MAX": 22921, "\u0120bog": 22922, "CBS": 22923, "\u0120Balance": 22924, "\u0120Sgt": 22925, "\u0120Recent": 22926, "\u0120cd": 22927, "\u0120popped": 22928, "\u0120incompet": 22929, "prop": 22930, "ayan": 22931, "guy": 22932, "Pacific": 22933, "\u0120tyr": 22934, "\u0120{{": 22935, "\u0120Mystic": 22936, "\u0120Dana": 22937, "\u0120masturb": 22938, "\u0120geometry": 22939, "\u00c3\u00a2": 22940, "\u0120Correct": 22941, "\u0120trajectory": 22942, "\u0120distracted": 22943, "\u0120foo": 22944, "\u0120Welsh": 22945, "Luc": 22946, "mith": 22947, "\u0120rugby": 22948, "\u0120respiratory": 22949, "\u0120triangle": 22950, "\u0120215": 22951, "\u0120undergraduate": 22952, "\u0120Superior": 22953, "changing": 22954, "_-": 22955, "\u0120rightly": 22956, "\u0120referee": 22957, "\u0120lucrative": 22958, "\u0120unauthorized": 22959, "\u0120resembles": 22960, "\u0120GNU": 22961, "\u0120Derby": 22962, "\u0120pathways": 22963, "\u0120Led": 22964, "\u0120endurance": 22965, "\u0120stint": 22966, "\u0120collector": 22967, "Fast": 22968, "\u0120dots": 22969, "\u0120nationals": 22970, "\u0120Securities": 22971, "\u0120whip": 22972, "Param": 22973, "\u0120learns": 22974, "Magic": 22975, "\u0120detailing": 22976, "moon": 22977, "\u0120broadcasting": 22978, "\u0120baked": 22979, "265": 22980, "holm": 22981, "\u0120Sah": 22982, "\u0120Hussein": 22983, "\u0120Courtesy": 22984, "174": 22985, "\u0120146": 22986, "\u0120geographic": 22987, "peace": 22988, "\u0120judging": 22989, "\u0120Stern": 22990, "Bur": 22991, "\u0120storyline": 22992, "Gun": 22993, "\u0120Stick": 22994, "245": 22995, "307": 22996, "\u00e3\u0124\u00b4\u00e3\u0125\u00b3": 22997, "\u0120Administrator": 22998, "\u0120burnt": 22999, "\u0120pave": 23000, "choes": 23001, "Exec": 23002, "\u0120campuses": 23003, "Result": 23004, "\u0120mutations": 23005, "\u0120Charter": 23006, "\u0120captures": 23007, "\u0120compares": 23008, "\u0120badge": 23009, "Scient": 23010, "\u0120erad": 23011, "iery": 23012, "oi": 23013, "ettes": 23014, "\u0120Estate": 23015, "\u0120strap": 23016, "\u0120proudly": 23017, "\u0120fried": 23018, "\u0120withdrawn": 23019, "\u0120Voy": 23020, "phony": 23021, "Items": 23022, "\u0120Pierce": 23023, "bard": 23024, "\u0120annotation": 23025, "anton": 23026, "illon": 23027, "Impro": 23028, "...)": 23029, "\u0120happier": 23030, "------": 23031, "adjust": 23032, "\u0120staffers": 23033, "\u0120activism": 23034, "\u0120perf": 23035, "\u0120alright": 23036, "Need": 23037, "\u0120commence": 23038, "\u0120opioid": 23039, "\u0120Amanda": 23040, "Es": 23041, "\u0120Pars": 23042, "\u0120Kaw": 23043, "Works": 23044, "248": 23045, "\u0120indo": 23046, "tc": 23047, "endant": 23048, "\u0120Moto": 23049, "\u0120legalization": 23050, "OTE": 23051, "\u0120tasked": 23052, "\u0120tsp": 23053, "\u0120ACTIONS": 23054, "166": 23055, "\u0120refreshing": 23056, "\u0120NR": 23057, "\u0120Perez": 23058, "\u0120infringement": 23059, "SY": 23060, "Listen": 23061, "inning": 23062, "ku": 23063, "\u0120rotate": 23064, "program": 23065, "arah": 23066, "Design": 23067, "\u0120(\u00c2\u00a3": 23068, "\u0120storing": 23069, "\u0120warrants": 23070, "\u0120judgement": 23071, "\u0120Brist": 23072, "usually": 23073, "photo": 23074, "\u0120Ran": 23075, "\u0120Pine": 23076, "\u0120outrageous": 23077, "\u0120Valentine": 23078, "luence": 23079, "\u0120Everybody": 23080, "Altern": 23081, "\u0120relevance": 23082, "\u0120terminated": 23083, "\u0120dessert": 23084, "\u0120fulfilled": 23085, "\u0120prosecuted": 23086, "\u0120Words": 23087, "\u0120migrant": 23088, "\u0120cultivation": 23089, "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 23090, "idelity": 23091, "\u0120Vern": 23092, "\u0120Login": 23093, "\u0120metaphor": 23094, "\u0120Tip": 23095, "\u0120recruits": 23096, "\u0120Pig": 23097, "ribing": 23098, "\u0120enthusiasts": 23099, "exper": 23100, "\u0120frightening": 23101, "\u0120Hair": 23102, "anson": 23103, "strate": 23104, "\u0120hi": 23105, "Height": 23106, "\u0120owning": 23107, "none": 23108, "\u0120dislike": 23109, "\u0120knives": 23110, "pherd": 23111, "\u0120loudly": 23112, "\u0120APIs": 23113, "Display": 23114, "\u0120Lac": 23115, "\u0120USS": 23116, "abl": 23117, "verages": 23118, "Jew": 23119, "\u0120172": 23120, "\u0120Historical": 23121, "atoon": 23122, "\u0120Physics": 23123, "intern": 23124, "\u0120warmth": 23125, "\u0120topp": 23126, "DM": 23127, "\u0120gunman": 23128, "\u0120emperor": 23129, "odi": 23130, "\u00e3\u0125\u00a3": 23131, "inatory": 23132, "\u0120Rib": 23133, "\u0120131": 23134, "\u0120Saturn": 23135, "\u0120Shining": 23136, "\u0120waking": 23137, "Quotes": 23138, "\u0120comedian": 23139, "enberg": 23140, "\u00c2\u00bd": 23141, "\u0120believers": 23142, "\u0120paperwork": 23143, "custom": 23144, "\u0120lev": 23145, "\u0120lament": 23146, "\u0120pouring": 23147, "222": 23148, "political": 23149, "\u0120Supplement": 23150, "maid": 23151, "\u0120cruelty": 23152, "\u0120tread": 23153, "ysics": 23154, "Aw": 23155, "rites": 23156, "\u0120modifier": 23157, "\u0120Position": 23158, "Adam": 23159, "lb": 23160, "ubs": 23161, "\u0120imperfect": 23162, "\u0120clusters": 23163, "\u0120Engineer": 23164, "\u0120Cherry": 23165, "\u0120inauguration": 23166, "\u0120Sau": 23167, "\u0120embodiment": 23168, "\u0120Uncle": 23169, "\u0120overr": 23170, "\u0120explosions": 23171, "cule": 23172, "\u0120Princeton": 23173, "\u0120Andrea": 23174, "\u0120incorrectly": 23175, "\u0120earnest": 23176, "\u0120pilgr": 23177, "\u0120Sprint": 23178, "\u0120sleeve": 23179, "\u0120hears": 23180, "\u0120Amazing": 23181, "\u0120browsing": 23182, "agin": 23183, "\u0120homeland": 23184, "\u0120haw": 23185, "\u0120diving": 23186, "istered": 23187, "178": 23188, "\u0120bargaining": 23189, "\u0120Arcade": 23190, "\u0120delegate": 23191, "terson": 23192, "................................................................": 23193, "\u0120Jacksonville": 23194, "275": 23195, "\u0120stagn": 23196, "\u0120adam": 23197, "\u0120Sherman": 23198, "CB": 23199, "\u0120suburb": 23200, "\u0120Foods": 23201, "\u0120converting": 23202, "\u0120Arist": 23203, "\u0120chambers": 23204, "love": 23205, "\u0120amino": 23206, "\u0120Gan": 23207, "\u0120madness": 23208, "mc": 23209, "\u0120USE": 23210, "defined": 23211, "\u0120ultr": 23212, "indust": 23213, "\u0120wolves": 23214, "lance": 23215, "Additionally": 23216, "\u0120cracks": 23217, "asia": 23218, "\u0120Reason": 23219, "\u0120Pump": 23220, "\u0120accidental": 23221, "\u0120Laser": 23222, "\u0120Rid": 23223, "\u0120initialized": 23224, "elli": 23225, "\u0120unnamed": 23226, "\u0120noun": 23227, "\u0120Passed": 23228, "\u0120hostage": 23229, "\u0120Ethiop": 23230, "shirts": 23231, "\u0120unrel": 23232, "\u0120Embassy": 23233, "\u01201941": 23234, "\u0120atoms": 23235, "\u0120purported": 23236, "164": 23237, "\u0120Fi": 23238, "\u0120gallons": 23239, "\u0120Monica": 23240, "\u0120pg": 23241, "enment": 23242, "\u0120sorted": 23243, "\u0120Gospel": 23244, "\u0120heights": 23245, "\u0120traced": 23246, "\u0120undergoing": 23247, "Shell": 23248, "\u0120sacks": 23249, "\u0120proportions": 23250, "\u0120halluc": 23251, "Font": 23252, "acet": 23253, "\u0120warmer": 23254, "\u0120INTER": 23255, "\u0120grabbing": 23256, "Plug": 23257, "\u0120realization": 23258, "\u0120Burke": 23259, "\u0120enchant": 23260, "ATER": 23261, "\u0120Seed": 23262, "\u0120abundant": 23263, "FM": 23264, "\u0120civic": 23265, "Vs": 23266, "isi": 23267, "\u0120vow": 23268, "\u0120reper": 23269, "\u0120Partnership": 23270, "\u0120penetration": 23271, "\u0120axe": 23272, "\u0120shattered": 23273, "\u0120Zombies": 23274, "\u0120vinyl": 23275, "\u0120Alert": 23276, "eon": 23277, "\u0120obliged": 23278, "\u0120Illust": 23279, "\u0120Plaza": 23280, "\u0120Frontier": 23281, "\u0120davidjl": 23282, "\u0120Serial": 23283, "\u0120Hav": 23284, "\u0120Nutrition": 23285, "Bi": 23286, "\u0120\u00e2\u0138\u012a": 23287, "\u0120Jays": 23288, "linux": 23289, "\u0120hurry": 23290, "\u0120voy": 23291, "\u0120hopeless": 23292, "\u0120Stealth": 23293, "\u0120\u00e3\u0123": 23294, "essors": 23295, "ttle": 23296, "borg": 23297, "\u0120Safari": 23298, "fell": 23299, "\u0120wary": 23300, "due": 23301, "\u0120Above": 23302, "Ha": 23303, "ELL": 23304, "\u0120notor": 23305, "\u0120Won": 23306, "Too": 23307, "\u0120occupations": 23308, "\u0120possessions": 23309, "\u0120inviting": 23310, "\u0120predators": 23311, "\u0120accelerated": 23312, "\u0120157": 23313, "uterte": 23314, "\u0120Cube": 23315, "east": 23316, "account": 23317, "Give": 23318, "\u0120transplant": 23319, "redients": 23320, "idable": 23321, "\u0120screenshots": 23322, "\u0120Gund": 23323, "\u0120FS": 23324, "\u0120travelers": 23325, "\u0120sensory": 23326, "\u0120Fiat": 23327, "\u0120Rockets": 23328, "\u0130\u012d": 23329, "_{": 23330, "Friend": 23331, "\u0120charming": 23332, "ALS": 23333, "\u0120enjoyment": 23334, "mph": 23335, "\u01205000": 23336, "\u0120REG": 23337, "\u00d9\u0128": 23338, "bia": 23339, "\u0120compilation": 23340, "rost": 23341, "\u0120VP": 23342, "\u0120Schne": 23343, "2019": 23344, "\u0120copying": 23345, "MORE": 23346, "\u0120Flore": 23347, "falls": 23348, "215": 23349, "total": 23350, "\u0120disciples": 23351, "double": 23352, "\u0120exceeding": 23353, "\u0120smashed": 23354, "\u0120conceptual": 23355, "\u0120Romania": 23356, "\u0120Brent": 23357, "\u0120ICE": 23358, "\u0120Tou": 23359, "\u0120grap": 23360, "\u0120nails": 23361, "189": 23362, "\u00e3\u0125\u013a": 23363, "\u0120procure": 23364, "eur": 23365, "\u0120confirming": 23366, "\u0120Cec": 23367, "awi": 23368, "\u0120Eden": 23369, "\u0120ng": 23370, "\u0120engineered": 23371, "atics": 23372, "\u0120hooked": 23373, "\u0120disgusting": 23374, "\u0120Murder": 23375, "\u00e3\u0124\u00bf": 23376, "Library": 23377, "\u0120168": 23378, "Almost": 23379, "hematic": 23380, "Menu": 23381, "\u0120Notre": 23382, "\u0120Jur": 23383, "\u0120kidnapped": 23384, "\u0120hacker": 23385, "\u0120Jade": 23386, "\u0120creepy": 23387, "\u0120drawings": 23388, "\u0120Sponsor": 23389, "\u0120cyclists": 23390, "\u0120Goblin": 23391, "\u0120optimized": 23392, "\u0120staged": 23393, "\u0120McD": 23394, "between": 23395, "Age": 23396, "eno": 23397, "Sex": 23398, "\u0120Wide": 23399, "nings": 23400, "avis": 23401, "\u0120incapable": 23402, "\u0120Kob": 23403, "\u0120rewarding": 23404, "\u0120Lone": 23405, "olescent": 23406, "\u0120contracted": 23407, "\u0120sticky": 23408, "Jose": 23409, "Ball": 23410, "fest": 23411, "\u0120Input": 23412, "\u0120Recently": 23413, "\u0120tomat": 23414, "square": 23415, "Application": 23416, "\u0120nitrogen": 23417, "\u0120duplicate": 23418, "\u0120Recon": 23419, "\u0120Dear": 23420, "London": 23421, "\u0120intra": 23422, "\u0120dock": 23423, "\u0120outreach": 23424, "\u0120Million": 23425, "\u0120mammals": 23426, "ampton": 23427, "VAL": 23428, "\u0120snaps": 23429, "\u0120dos": 23430, "\u0120Whole": 23431, "\u0120Ready": 23432, "Try": 23433, "\u0120Winnipeg": 23434, "earance": 23435, "\u0120incurred": 23436, "renched": 23437, "\u0120NSW": 23438, "ilot": 23439, "raine": 23440, "\u0120cube": 23441, "got": 23442, "\u0120runway": 23443, "etermined": 23444, "\u0120Hawks": 23445, "\u0120survivor": 23446, "\u0120Wish": 23447, "\u0120Din": 23448, "\u0120DEF": 23449, "\u0120Vault": 23450, "187": 23451, "\u0120mushrooms": 23452, "\u0120crisp": 23453, "bey": 23454, "\u0120Discovery": 23455, "\u0120developmental": 23456, "\u0120paradigm": 23457, "\u0120chaotic": 23458, "\u0120Tsu": 23459, "\u0120333": 23460, "bons": 23461, "\u0120bacterial": 23462, "\u0120commits": 23463, "\u0120cosmic": 23464, "\u0120mega": 23465, "ocative": 23466, "\u0120Paint": 23467, "ophobic": 23468, "\u0120vain": 23469, "\u0120carved": 23470, "\u0120Thief": 23471, "\u0120Gul": 23472, "owship": 23473, "\u0120cites": 23474, "\u0120Edinburgh": 23475, "\u0120diminished": 23476, "\u0120acknowledges": 23477, "\u0120Kills": 23478, "\u0120microw": 23479, "\u0120Hera": 23480, "\u0120seniors": 23481, "\u0120whereby": 23482, "Hop": 23483, "atron": 23484, "\u0120unavailable": 23485, "\u0120Nate": 23486, "\u0120480": 23487, "\u0120slated": 23488, "\u0120Rebecca": 23489, "\u0120Battery": 23490, "\u0120grammar": 23491, "\u0120headset": 23492, "\u0120cursor": 23493, "\u0120excluding": 23494, "anye": 23495, "aundering": 23496, "ebin": 23497, "\u0120feasible": 23498, "\u0120Publishing": 23499, "\u0120Labs": 23500, "\u0120Cliff": 23501, "\u0120Ferrari": 23502, "\u0120pac": 23503, "visible": 23504, "marked": 23505, "pell": 23506, "\u0120polite": 23507, "\u0120staggering": 23508, "\u0120Galactic": 23509, "\u0120superst": 23510, "\u0120paran": 23511, "\u0120Officers": 23512, "\u00e3\u0122\u0123": 23513, "\u0120specifics": 23514, "ulus": 23515, "239": 23516, "\u0120Paste": 23517, "AMP": 23518, "\u0120Panama": 23519, "\u0120Delete": 23520, "anguard": 23521, "restrial": 23522, "\u0120heroic": 23523, "\u0120Dy": 23524, "\u00d8\u00a7\u00d9\u0126": 23525, "\u0120incumbent": 23526, "\u0120crunch": 23527, "tro": 23528, "\u0120scoop": 23529, "\u0120blogger": 23530, "\u0120sellers": 23531, "uren": 23532, "\u0120medicines": 23533, "\u0120Caps": 23534, "\u0120Animation": 23535, "oxy": 23536, "\u0120outward": 23537, "\u0120inquiries": 23538, "229": 23539, "\u0120psychologist": 23540, "\u0120Sask": 23541, "evil": 23542, "\u0120contaminated": 23543, "\u00e3\u0124\u00a8": 23544, "herence": 23545, "\u0120branded": 23546, "\u0120Abdul": 23547, "zh": 23548, "\u0120paragraphs": 23549, "\u0120mins": 23550, "\u0120correlated": 23551, "erb": 23552, "\u0120impart": 23553, "\u0120milestone": 23554, "\u0120Solutions": 23555, "otle": 23556, "\u0120undercover": 23557, "\u0120marched": 23558, "\u0120Chargers": 23559, "fax": 23560, "\u0120Secrets": 23561, "\u0120ruth": 23562, "weather": 23563, "\u0120feminine": 23564, "\u0120sham": 23565, "\u0120prestigious": 23566, "iggins": 23567, "\u0120sung": 23568, "history": 23569, "ettle": 23570, "ggie": 23571, "\u0120outdated": 23572, "oland": 23573, "\u0120perceptions": 23574, "\u0120Session": 23575, "\u0120Dodgers": 23576, "uj": 23577, "\u0120END": 23578, "Doc": 23579, "\u0120deficiency": 23580, "Grand": 23581, "\u0120Joker": 23582, "\u0120retrospect": 23583, "\u0120diagnostic": 23584, "\u0120harmless": 23585, "\u0120rogue": 23586, "\u0120Aval": 23587, "Equ": 23588, "\u0120transc": 23589, "\u0120Robertson": 23590, "\u0120Depending": 23591, "\u0120Burns": 23592, "ivo": 23593, "\u0120hostility": 23594, "Features": 23595, "\u0135\u013a": 23596, "\u0120discomfort": 23597, "\u0120LCD": 23598, "specified": 23599, "\u0120Expect": 23600, "340": 23601, "\u0120imperative": 23602, "\u0120Regular": 23603, "Chinese": 23604, "\u0120statewide": 23605, "\u0120symm": 23606, "\u0120loops": 23607, "\u0120autumn": 23608, "Nick": 23609, "\u0120shaping": 23610, "\u0120quot": 23611, "\u0120cherry": 23612, "\u0120Crossref": 23613, "\u00e8\u00a6\u013c\u00e9\u0128\u0134": 23614, "Standard": 23615, "heed": 23616, "\u0120Dell": 23617, "\u0120Vietnamese": 23618, "\u0120ost": 23619, "\u0120Valkyrie": 23620, "OA": 23621, "Assad": 23622, "\u0120rebound": 23623, "\u0120Traffic": 23624, "places": 23625, "\u00e6\u013a": 23626, "\u0120Buc": 23627, "172": 23628, "\u0120shelters": 23629, "\u0120insisting": 23630, "\u0120Certainly": 23631, "\u0120Kenneth": 23632, "\u0120TCP": 23633, "\u0120penal": 23634, "\u0120Replay": 23635, "heard": 23636, "\u0120dialect": 23637, "iza": 23638, "\u0120FY": 23639, "itcher": 23640, "\u0120DL": 23641, "\u0120spiral": 23642, "\u0120quarterbacks": 23643, "\u0120hull": 23644, "\u0120google": 23645, "\u0120todd": 23646, "\u0120Sterling": 23647, "\u0120Plate": 23648, "\u0120spying": 23649, "mbol": 23650, "\u0120Realm": 23651, "\u0120Proced": 23652, "\u0120Crash": 23653, "\u0120terminate": 23654, "\u0120protesting": 23655, "Center": 23656, "guided": 23657, "\u0120uncover": 23658, "\u0120boycott": 23659, "\u0120realizes": 23660, "sound": 23661, "\u0120pretending": 23662, "\u0120Vas": 23663, "1980": 23664, "\u0120framed": 23665, "\u0120139": 23666, "\u0120descended": 23667, "\u0120rehabilitation": 23668, "\u0120borrowing": 23669, "\u0120Buch": 23670, "\u0120blur": 23671, "Ron": 23672, "\u0120Frozen": 23673, "enza": 23674, "Chief": 23675, "\u0120Poor": 23676, "\u0120translates": 23677, "MIN": 23678, "\u0120212": 23679, "JECT": 23680, "\u0120erupted": 23681, "\u0120successes": 23682, "SEC": 23683, "\u0120plague": 23684, "\u0120gems": 23685, "doms": 23686, "\u0120stretches": 23687, "\u0120Spy": 23688, "\u0120storytelling": 23689, "Credit": 23690, "\u0120Push": 23691, "\u0120traction": 23692, "\u0120ineffective": 23693, "\u0120Luna": 23694, "\u0120tapes": 23695, "\u0120analytics": 23696, "ercise": 23697, "\u0120programmes": 23698, "\u0120Carbon": 23699, "\u0120behold": 23700, "heavy": 23701, "\u0120Conservation": 23702, "\u0120FIR": 23703, "\u0120sack": 23704, "termin": 23705, "ricks": 23706, "\u0120housed": 23707, "\u0120unusually": 23708, "Ice": 23709, "\u0120executing": 23710, "\u0120Moroc": 23711, "eday": 23712, "\u0120editions": 23713, "\u0120smarter": 23714, "\u0120BA": 23715, "\u0120outlaw": 23716, "\u0120vanished": 23717, "iba": 23718, "ALSE": 23719, "\u0120Silva": 23720, "238": 23721, "Could": 23722, "\u0120philosopher": 23723, "\u0120evacuated": 23724, "Secret": 23725, "142": 23726, "\u0120visas": 23727, "\u00e3\u0124\u00ac": 23728, "\u0120Malt": 23729, "\u0120Clearly": 23730, "\u0120Niger": 23731, "\u0120Cairo": 23732, "\u0120Fist": 23733, "380": 23734, "\u0120XML": 23735, "auto": 23736, "itant": 23737, "\u0120reinforced": 23738, "Record": 23739, "\u0120Survivor": 23740, "GHz": 23741, "\u0120screws": 23742, "parents": 23743, "\u0120oceans": 23744, "mares": 23745, "\u0120brakes": 23746, "vasive": 23747, "\u0120hello": 23748, "\u0120SIM": 23749, "rimp": 23750, "\u0120ore": 23751, "\u0120Armour": 23752, "247": 23753, "\u0120terrific": 23754, "\u0120tones": 23755, "141": 23756, "\u0120Minutes": 23757, "Episode": 23758, "\u0120curves": 23759, "\u0120inflammatory": 23760, "\u0120batting": 23761, "\u0120Beautiful": 23762, "Lay": 23763, "\u0120unpop": 23764, "vable": 23765, "\u0120riots": 23766, "\u0120Tactics": 23767, "baugh": 23768, "\u0120Cock": 23769, "\u0120orgasm": 23770, "\u0120Sas": 23771, "\u0120constructor": 23772, "etz": 23773, "Gov": 23774, "\u0120antagon": 23775, "\u0120theat": 23776, "\u0120deeds": 23777, "hao": 23778, "cuts": 23779, "\u0120McCl": 23780, "\u0120um": 23781, "\u0120Scientists": 23782, "\u0120grassroots": 23783, "yssey": 23784, "\"]=>": 23785, "\u0120surfaced": 23786, "\u0120shades": 23787, "\u0120neighbours": 23788, "\u0120advertis": 23789, "oya": 23790, "\u0120merged": 23791, "Upon": 23792, "\u0120gad": 23793, "\u0120anticipate": 23794, "Anyway": 23795, "\u0120slogan": 23796, "\u0120disrespect": 23797, "Iran": 23798, "\u0120TB": 23799, "acted": 23800, "\u0120subpoen": 23801, "mediately": 23802, "OOOO": 23803, "\u0120waiver": 23804, "\u0120vulnerabilities": 23805, "ottesville": 23806, "\u0120Huffington": 23807, "Josh": 23808, "\u0120DH": 23809, "Monday": 23810, "\u0120Ellen": 23811, "Know": 23812, "xon": 23813, "items": 23814, "228": 23815, "\u0120fills": 23816, "\u0120Nike": 23817, "\u0120cumulative": 23818, "andals": 23819, "Ir": 23820, "\u0120\u00ec": 23821, "\u0120friction": 23822, "igator": 23823, "\u0120scans": 23824, "\u0120Vienna": 23825, "ldom": 23826, "\u0120performers": 23827, "Prim": 23828, "\u0120bidding": 23829, "Mur": 23830, "\u0120leaned": 23831, "\u0120Prix": 23832, "alks": 23833, "\u0120[\u00e2\u0122\u00a6]": 23834, "\u0120Twitch": 23835, "\u0120Developer": 23836, "\u0120Gir": 23837, "\u0120callback": 23838, "Abstract": 23839, "\u0120accustomed": 23840, "\u0120freedoms": 23841, "\u0120PG": 23842, "uracy": 23843, "\u0120lump": 23844, "isman": 23845, ",,,,": 23846, "1992": 23847, "\u0120RED": 23848, "\u0120worm": 23849, "Match": 23850, "\u0120Platinum": 23851, "IJ": 23852, "\u0120Owner": 23853, "Trivia": 23854, "compl": 23855, "\u0120newborn": 23856, "\u0120fantas": 23857, "Own": 23858, "\u01201959": 23859, "\u0120sympath": 23860, "\u0120ubiqu": 23861, "\u0120outputs": 23862, "\u0120allev": 23863, "\u0120prag": 23864, "Kevin": 23865, "\u0120favors": 23866, "\u0120burial": 23867, "\u0120nurt": 23868, "solete": 23869, "cache": 23870, "\u0120156": 23871, "\u0120unlocks": 23872, "techn": 23873, "Making": 23874, "\u0120conquer": 23875, "adic": 23876, "\u00e6\u0138": 23877, "\u0120elf": 23878, "\u0120electorate": 23879, "\u0120Kurds": 23880, "\u0120Stack": 23881, "\u0120Samurai": 23882, "\u0120\u00e2\u013a\u0127": 23883, "\u0120{}": 23884, "\u0120Said": 23885, "\u0120Fallout": 23886, "\u0120kindness": 23887, "\u0120Customs": 23888, "\u0120Boulevard": 23889, "\u0120helicopters": 23890, "otics": 23891, "\u0120Veget": 23892, "comment": 23893, "\u0120criticised": 23894, "\u0120polished": 23895, "\u0120Remix": 23896, "\u0120Cultural": 23897, "\u0120recons": 23898, "\u0120doi": 23899, "atem": 23900, "Screen": 23901, "\u0120barred": 23902, "Comments": 23903, "\u0120Generally": 23904, "\u0120slap": 23905, "720": 23906, "Vari": 23907, "pine": 23908, "\u0120empt": 23909, "\u0120hats": 23910, "\u0120Playing": 23911, "lab": 23912, "average": 23913, "forms": 23914, "\u0120Cotton": 23915, "\u0120cans": 23916, "\u0120DON": 23917, "\u0120Somalia": 23918, "Crypt": 23919, "\u0120Increases": 23920, "Ever": 23921, "modern": 23922, "\u0120surgeon": 23923, "3000": 23924, "\u0120randomized": 23925, "================================================================": 23926, "Bern": 23927, "impl": 23928, "\u0120COR": 23929, "\u0120proclaim": 23930, "thouse": 23931, "\u0120toes": 23932, "\u0120ample": 23933, "\u0120preserving": 23934, "\u0120disbel": 23935, "grand": 23936, "Besides": 23937, "\u0120silk": 23938, "\u0120Pattern": 23939, "hm": 23940, "\u0120enterprises": 23941, "\u0120affidavit": 23942, "\u0120Advisory": 23943, "\u0120advertised": 23944, "\u0120Religious": 23945, "sections": 23946, "psych": 23947, "\u0120Fields": 23948, "aways": 23949, "\u0120hashtag": 23950, "\u0120Nightmare": 23951, "\u0120vampire": 23952, "\u0120forensic": 23953, "rossover": 23954, "nar": 23955, "\u0120navy": 23956, "\u0120vacant": 23957, "\u0120Duel": 23958, "\u0120hallway": 23959, "\u0120facebook": 23960, "identally": 23961, "\u0120NRA": 23962, "\u0120matt": 23963, "\u0120hurricane": 23964, "\u0120Kirby": 23965, "\u0120Puzzle": 23966, "\u0120skirt": 23967, "oust": 23968, "dullah": 23969, "\u0120analogy": 23970, "inion": 23971, "\u0120tomatoes": 23972, "\u0120NV": 23973, "\u0120Peak": 23974, "\u0120Meyer": 23975, "\u0120appointments": 23976, "\u0120masc": 23977, "\u0120alley": 23978, "rehend": 23979, "\u0120charities": 23980, "\u0120undo": 23981, "\u0120destinations": 23982, "\u0120Testing": 23983, "\">\"": 24618, "cats": 24619, "*.": 24620, "\u0120gestures": 24621, "general": 24622, "League": 24623, "\u0120packets": 24624, "\u0120Inspector": 24625, "\u0120Berg": 24626, "\u0120fraudulent": 24627, "\u0120criticize": 24628, "Fun": 24629, "\u0120blaming": 24630, "ndra": 24631, "\u0120slash": 24632, "\u0120Eston": 24633, "\u0120proposing": 24634, "\u0120whales": 24635, "\u0120therapist": 24636, "\u0120subset": 24637, "\u0120leisure": 24638, "ELD": 24639, "\u0120CVE": 24640, "\u0120Activity": 24641, "\u0120culmin": 24642, "shop": 24643, "\u0120DAY": 24644, "ischer": 24645, "\u0120Admiral": 24646, "\u0120Attacks": 24647, "\u01201958": 24648, "\u0120memoir": 24649, "\u0120folded": 24650, "\u0120sexist": 24651, "\u0120153": 24652, "\u0120LI": 24653, "\u0120readings": 24654, "\u0120embarrassment": 24655, "\u0120Employment": 24656, "wart": 24657, "chin": 24658, "\u0120continuation": 24659, "lia": 24660, "Recently": 24661, "\u0120duel": 24662, "\u0120evacuation": 24663, "\u0120Kashmir": 24664, "\u0120disposition": 24665, "\u0120Rig": 24666, "\u0120bolts": 24667, "\u0120insurers": 24668, "467": 24669, "Mex": 24670, "\u0120retaliation": 24671, "\u0120misery": 24672, "\u0120unreasonable": 24673, "raining": 24674, "Imm": 24675, "\u0120PU": 24676, "emer": 24677, "\u0120genital": 24678, "\u00e3\u0124\u00b3": 24679, "\u0120Candy": 24680, "\u0120onions": 24681, "\u0120Patt": 24682, "liner": 24683, "\u0120conceded": 24684, "\u0120fa": 24685, "\u0120forc": 24686, "\u0120Hernandez": 24687, "\u0120Geoff": 24688, "debian": 24689, "\u0120Teams": 24690, "\u0120cries": 24691, "\u0120homeowners": 24692, "237": 24693, "ABC": 24694, "\u0120stitch": 24695, "\u0120statistic": 24696, "\u0120headers": 24697, "\u0120Biology": 24698, "\u0120motors": 24699, "\u0120GEN": 24700, "\u0120Lip": 24701, "\u0120hates": 24702, "\u0120heel": 24703, "Self": 24704, "ipl": 24705, "EDIT": 24706, "orting": 24707, "\u0120annot": 24708, "\u0120Speech": 24709, "oldemort": 24710, "\u0120Javascript": 24711, "\u0120LeBron": 24712, "\u0120footprint": 24713, "\u0120fn": 24714, "\u0120seizures": 24715, "nas": 24716, "hide": 24717, "\u01201954": 24718, "\u0120Bee": 24719, "\u0120Declaration": 24720, "\u0120Katie": 24721, "\u0120reservations": 24722, "NR": 24723, "female": 24724, "\u0120saturated": 24725, "\u0120biblical": 24726, "\u0120trolls": 24727, "Device": 24728, "photos": 24729, "\u0120drums": 24730, "\u00e3\u0125\u012b\u00e3\u0125\u00a9\u00e3\u0124\u00b4\u00e3\u0125\u00b3": 24731, "Night": 24732, "fighter": 24733, "\u0120Hak": 24734, "riber": 24735, "\u0120cush": 24736, "\u0120disciplinary": 24737, "baum": 24738, "\u0120GH": 24739, "\u0120Schmidt": 24740, "ilibrium": 24741, "\u0120sixty": 24742, "\u0120Kushner": 24743, "rots": 24744, "\u0120pund": 24745, "\u0120Rac": 24746, "\u0120springs": 24747, "\u0120conve": 24748, "Business": 24749, "Fall": 24750, "\u0120qualifications": 24751, "\u0120verses": 24752, "\u0120narciss": 24753, "\u0120Koh": 24754, "\u0120Wow": 24755, "\u0120Charlottesville": 24756, "edo": 24757, "\u0120interrogation": 24758, "\u0120Wool": 24759, "365": 24760, "Brian": 24761, "\u0120\u00e2\u013e\u0135": 24762, "\u0120alleges": 24763, "onds": 24764, "idation": 24765, "\u0120Jackie": 24766, "yu": 24767, "\u0120lakes": 24768, "\u0120worthwhile": 24769, "\u0120crystals": 24770, "\u0120Juda": 24771, "\u0120comprehend": 24772, "\u0120flush": 24773, "\u0120absorption": 24774, "\u0120OC": 24775, "\u0120frightened": 24776, "\u0120Chocolate": 24777, "Martin": 24778, "\u0120buys": 24779, "\u0120bucks": 24780, "\u0120appell": 24781, "\u0120Championships": 24782, "\u0120listener": 24783, "\u0120Defensive": 24784, "\u0120cz": 24785, "uds": 24786, "\u0120Mate": 24787, "\u0120replay": 24788, "\u0120decorated": 24789, "\u0120sunk": 24790, "\u0120VIP": 24791, "\u0120Ank": 24792, "\u0120195": 24793, "aaaa": 24794, "Nobody": 24795, "\u0120Milk": 24796, "\u0120Gur": 24797, "\u0120Mk": 24798, "\u0120Sara": 24799, "\u0120seating": 24800, "\u0120Wid": 24801, "Track": 24802, "\u0120employs": 24803, "\u0120gigantic": 24804, "APP": 24805, "\u00e3\u0124\u00a7": 24806, "inventory": 24807, "\u0120towel": 24808, "atche": 24809, "lasting": 24810, "\u0120TL": 24811, "\u0120latency": 24812, "\u0120kne": 24813, "Ber": 24814, "meaning": 24815, "\u0120upheld": 24816, "\u0120playground": 24817, "\u0120mant": 24818, "Side": 24819, "\u0120stereo": 24820, "\u0120northwest": 24821, "\u0120exceptionally": 24822, "\u0120rays": 24823, "\u0120recurring": 24824, "Drive": 24825, "\u0120upright": 24826, "\u0120abduct": 24827, "\u0120Marathon": 24828, "\u0120goodbye": 24829, "\u0120alphabet": 24830, "hp": 24831, "\u0120courtroom": 24832, "rington": 24833, "othing": 24834, "Tag": 24835, "\u0120diplomats": 24836, "\u0120barbar": 24837, "\u0120Aqua": 24838, "183": 24839, "3333": 24840, "\u0120maturity": 24841, "\u0120instability": 24842, "\u0120Apache": 24843, "\u0120===": 24844, "\u0120fasting": 24845, "\u0120Grid": 24846, "ModLoader": 24847, "\u0120152": 24848, "Abs": 24849, "\u0120Operating": 24850, "etti": 24851, "\u0120acquaint": 24852, "Donnell": 24853, "\u0120Kem": 24854, "\u0120Forge": 24855, "\u0120armored": 24856, "Mil": 24857, "\u0120philosophers": 24858, "invest": 24859, "Players": 24860, "\u00e2\u012a": 24861, "\u0120myriad": 24862, "\u0120comrades": 24863, "Rot": 24864, "\u0120remembering": 24865, "\u0120corresponds": 24866, "\u0120programmers": 24867, "\u0120Lynn": 24868, "\u0120olig": 24869, "\u0120coherent": 24870, "ynchron": 24871, "\u0120Chemical": 24872, "\u0120jugg": 24873, "pair": 24874, "posts": 24875, "Eye": 24876, "\u0120Inner": 24877, "\u0120semester": 24878, "ottest": 24879, "\u0120Emirates": 24880, "ricanes": 24881, "orously": 24882, "mits": 24883, "\u0120Wis": 24884, "\u0120dodge": 24885, "location": 24886, "\u0120faded": 24887, "Amazon": 24888, "\u0120Proceed": 24889, "\u0120INFO": 24890, "journal": 24891, "\u0120Truck": 24892, "Ten": 24893, "\u0120217": 24894, "\u0120statutes": 24895, "mobile": 24896, "\u0120Types": 24897, "Recomm": 24898, "buster": 24899, "pex": 24900, "\u0120legends": 24901, "\u0120headache": 24902, "faced": 24903, "\u0120WiFi": 24904, "ifty": 24905, "\u0120HER": 24906, "\u0120circuits": 24907, "ERROR": 24908, "226": 24909, "olin": 24910, "\u0120cylinder": 24911, "ospace": 24912, "ikers": 24913, "Prem": 24914, "Quant": 24915, "\u0120conflicting": 24916, "\u0120slightest": 24917, "\u0120forged": 24918, "ionage": 24919, "Stephen": 24920, "\u0120Kub": 24921, "\u0120Opportun": 24922, "\u0120Heal": 24923, "\u0120blo": 24924, "\u0120rulers": 24925, "\u0120huh": 24926, "\u0120submarine": 24927, "fy": 24928, "asser": 24929, "\u0120allowance": 24930, "\u0120Kasich": 24931, "\u0120Tas": 24932, "\u0120Australians": 24933, "ForgeModLoader": 24934, "\u0120\u00e2\u0128\u0133": 24935, "\u0120Matrix": 24936, "amins": 24937, "\u01201200": 24938, "\u0120Acqu": 24939, "236": 24940, "Document": 24941, "\u0120Breaking": 24942, "193": 24943, "\u0120Subst": 24944, "\u0120Roller": 24945, "\u0120Properties": 24946, "\u0120NI": 24947, "tier": 24948, "\u0120crushing": 24949, "\u0120advocating": 24950, "Furthermore": 24951, "keepers": 24952, "\u0120sexism": 24953, "xd": 24954, "\u0120caller": 24955, "\u0120Sense": 24956, "chieve": 24957, "\u0120TF": 24958, "\u0120fueled": 24959, "\u0120reminiscent": 24960, "\u0120obsess": 24961, "urst": 24962, "\u0120uphold": 24963, "\u0120Fans": 24964, "hetics": 24965, "\u0120\u00e2\u0139": 24966, "\u0120Bath": 24967, "\u0120beverage": 24968, "\u0120oscill": 24969, "254": 24970, "\u0120poles": 24971, "\u0120gradual": 24972, "\u0120exting": 24973, "\u0120Suff": 24974, "\u0120Suddenly": 24975, "\u0120liking": 24976, "\u01201949": 24977, "unciation": 24978, "amination": 24979, "\u0120Omar": 24980, "\u0120LV": 24981, "\u0120Consequently": 24982, "\u0120synthes": 24983, "\u0120GIF": 24984, "\u0120pains": 24985, "\u0120interacting": 24986, "uously": 24987, "incre": 24988, "\u0120rumor": 24989, "\u0120Scientology": 24990, "197": 24991, "\u0120Zig": 24992, "\u0120spelling": 24993, "\u0120ASS": 24994, "\u0120extingu": 24995, "mson": 24996, "\u0120gh": 24997, "\u0120remarked": 24998, "\u0120Strategic": 24999, "\u0120MON": 25000, "\u00e5\u00a5": 25001, "gae": 25002, "\u0120WHAT": 25003, "Eric": 25004, "\u0120Campus": 25005, "\u0120methane": 25006, "\u0120imagin": 25007, "JUST": 25008, "\u0120Alm": 25009, "XT": 25010, "iq": 25011, "\u0120RSS": 25012, "\u0120wrongdoing": 25013, "atta": 25014, "\u0120bigot": 25015, "\u0120demonstrators": 25016, "\u0120Calvin": 25017, "\u0120Villa": 25018, "\u0120membrane": 25019, "\u0120Awesome": 25020, "\u0120benefic": 25021, "268": 25022, "\u0120magnificent": 25023, "\u0120Lots": 25024, "Greg": 25025, "\u0120Boris": 25026, "\u0120detainees": 25027, "\u0120Herman": 25028, "\u0120whispered": 25029, "\u0120awe": 25030, "Professor": 25031, "funding": 25032, "\u0120physiological": 25033, "\u0120Destruction": 25034, "\u0120limb": 25035, "\u0120manipulated": 25036, "\u0120bubbles": 25037, "\u0120pseud": 25038, "\u0120hydra": 25039, "\u0120Bristol": 25040, "\u0120stellar": 25041, "\u0120Expansion": 25042, "\u0120Kell": 25043, "\u0120Interestingly": 25044, "\u0120mans": 25045, "\u0120dragging": 25046, "\u0120ecological": 25047, "\u0120Fit": 25048, "\u0120gent": 25049, "\u0120benefited": 25050, "\u0120Haiti": 25051, "\u0120polyg": 25052, "\u00e3\u0125\u0130": 25053, "\u01202030": 25054, "\u0120prow": 25055, "\u0120reconstruction": 25056, "\u0120wast": 25057, "\u0120psychic": 25058, "\u0120Greeks": 25059, "Handler": 25060, "162": 25061, "\u0120Pulse": 25062, "\u0120solicit": 25063, "\u0120sys": 25064, "\u0120influx": 25065, "\u0120Gentle": 25066, "percent": 25067, "\u0120proliferation": 25068, "\u0120taxable": 25069, "\u0120disregard": 25070, "\u0120escaping": 25071, "\u0120ginger": 25072, "\u0120withstand": 25073, "\u0120devastated": 25074, "\u0120Dew": 25075, "series": 25076, "\u0120injected": 25077, "elaide": 25078, "\u0120turnover": 25079, "heat": 25080, "\u013b\u0124": 25081, "Happy": 25082, "\u0120Silent": 25083, "\u00e3\u0124\u0143": 25084, "ivism": 25085, "\u0120irrational": 25086, "AMA": 25087, "\u0120reef": 25088, "rub": 25089, "\u0120162": 25090, "\u0120bankers": 25091, "\u0120Ethics": 25092, "vv": 25093, "\u0120criticisms": 25094, "Kn": 25095, "186": 25096, "Movie": 25097, "\u0120Tories": 25098, "\u0120nood": 25099, "\u0120distortion": 25100, "False": 25101, "odore": 25102, "\u0120tasty": 25103, "Research": 25104, "\u0120UID": 25105, "-)": 25106, "\u0120divorced": 25107, "\u0120MU": 25108, "\u0120Hayes": 25109, "\u0120Isn": 25110, "iani": 25111, "\u0120HQ": 25112, "\u0120\"#": 25113, "ignant": 25114, "\u0120traumatic": 25115, "\u0120Ling": 25116, "Hun": 25117, "\u0120sabot": 25118, "online": 25119, "random": 25120, "\u0120renamed": 25121, "rared": 25122, "KA": 25123, "dead": 25124, "\u00c3\u00a9t": 25125, "\u0120Assistance": 25126, "\u0120seaf": 25127, "++++++++": 25128, "\u0120seldom": 25129, "\u0120Webb": 25130, "\u0120boolean": 25131, "ulet": 25132, "\u0120refrain": 25133, "\u0120DIY": 25134, "rule": 25135, "\u0120shutting": 25136, "\u0120utilizing": 25137, "loading": 25138, "\u0120Param": 25139, "coal": 25140, "ooter": 25141, "\u0120attracting": 25142, "\u0120Dol": 25143, "\u0120hers": 25144, "agnetic": 25145, "\u0120Reach": 25146, "imo": 25147, "\u0120discarded": 25148, "\u0120Pip": 25149, "015": 25150, "\u00c3\u00bcr": 25151, "\u0120mug": 25152, "Imagine": 25153, "COL": 25154, "\u0120cursed": 25155, "\u0120Shows": 25156, "\u0120Curtis": 25157, "\u0120Sachs": 25158, "speaking": 25159, "\u0120Vista": 25160, "\u0120Framework": 25161, "ongo": 25162, "\u0120subreddit": 25163, "\u0120crus": 25164, "\u0120Oval": 25165, "Row": 25166, "growing": 25167, "\u0120installment": 25168, "\u0120glac": 25169, "\u0120Advance": 25170, "ECK": 25171, "\u0120LGBTQ": 25172, "LEY": 25173, "\u0120acet": 25174, "\u0120successive": 25175, "\u0120Nicole": 25176, "\u01201957": 25177, "Quote": 25178, "\u0120circumstance": 25179, "ackets": 25180, "\u0120142": 25181, "ortium": 25182, "\u0120guessed": 25183, "\u0120Frame": 25184, "\u0120perpetrators": 25185, "\u0120Aviation": 25186, "\u0120Bench": 25187, "\u0120handc": 25188, "Ap": 25189, "\u01201956": 25190, "259": 25191, "rand": 25192, "NetMessage": 25193, "din": 25194, "urtles": 25195, "hig": 25196, "\u0120VIII": 25197, "ffiti": 25198, "\u0120Swords": 25199, "bial": 25200, "\u0120kidnapping": 25201, "device": 25202, "\u0120barn": 25203, "\u0120Eli": 25204, "aucas": 25205, "Send": 25206, "Constructed": 25207, "\u0120\u00c2\u00bd": 25208, "\u0120needles": 25209, "\u0120advertisements": 25210, "\u0120vou": 25211, "\u0120exhibited": 25212, "\u0120Fortress": 25213, "Ask": 25214, "Berry": 25215, "TYPE": 25216, "\u0120cancers": 25217, "umping": 25218, "\u0120Territory": 25219, "\u0120prud": 25220, "\u0120nas": 25221, "\u0120atheist": 25222, "\u0120balances": 25223, "\u00e3\u0123\u0141": 25224, "\u0120Shawn": 25225, "&&": 25226, "\u0120landsc": 25227, "\u0120RGB": 25228, "\u0120petty": 25229, "\u0120excellence": 25230, "\u0120translations": 25231, "\u0120parcel": 25232, "\u0120Chev": 25233, "East": 25234, "\u0120Output": 25235, "imi": 25236, "\u0120ambient": 25237, "\u0120Threat": 25238, "\u0120villains": 25239, "\u0120550": 25240, "ICA": 25241, "\u0120taller": 25242, "\u0120leaking": 25243, "cup": 25244, "\u0120polish": 25245, "\u0120infectious": 25246, "\u0120KC": 25247, "\u0120@@": 25248, "background": 25249, "\u0120bureaucracy": 25250, "\u0120Sai": 25251, "unless": 25252, "itious": 25253, "\u0120Skype": 25254, "Atl": 25255, "IDENT": 25256, "008": 25257, "\u0120hypocr": 25258, "\u0120pitchers": 25259, "\u0120guessing": 25260, "\u0120FINAL": 25261, "Between": 25262, "\u0120villagers": 25263, "\u0120252": 25264, "fashion": 25265, "\u0120Tunis": 25266, "Beh": 25267, "\u0120Exc": 25268, "\u0120MID": 25269, "288": 25270, "\u0120Haskell": 25271, "196": 25272, "\u0120NOR": 25273, "\u0120specs": 25274, "\u0120invari": 25275, "\u0120glut": 25276, "\u0120Cars": 25277, "\u0120impulse": 25278, "\u0120honors": 25279, "gel": 25280, "\u0120jurisdictions": 25281, "\u0120Bundle": 25282, "ulas": 25283, "California": 25284, "\u0120Increase": 25285, "\u0120pear": 25286, "\u0120singles": 25287, "\u0120cues": 25288, "\u0120underwent": 25289, "\u0120WS": 25290, "\u0120exaggerated": 25291, "\u0120dubious": 25292, "\u0120flashing": 25293, "LOG": 25294, ")].": 25295, "Journal": 25296, "tg": 25297, "Van": 25298, "\u0120Istanbul": 25299, "\u0120Insp": 25300, "\u0120Franken": 25301, "Draw": 25302, "\u0120sadness": 25303, "\u0120ironic": 25304, "\u0120Fry": 25305, "xc": 25306, "\u0120164": 25307, "isch": 25308, "Way": 25309, "\u0120Protestant": 25310, "horn": 25311, "\u0120unaff": 25312, "\u0120Viv": 25313, "illas": 25314, "\u0120Productions": 25315, "\u0120Hogan": 25316, "\u0120perimeter": 25317, "\u0120Sisters": 25318, "\u0120spontaneous": 25319, "\u0120downside": 25320, "\u0120descendants": 25321, "\u0120orn": 25322, "worm": 25323, "Japanese": 25324, "\u01201955": 25325, "\u0120151": 25326, "\u0120Doing": 25327, "elsen": 25328, "umbles": 25329, "\u0120radically": 25330, "\u0120Drum": 25331, "\u0120Bach": 25332, "\u0120liabilities": 25333, "\u0120OB": 25334, "\u0120Elementary": 25335, "\u0120meme": 25336, "ynes": 25337, "\u0120fingerprint": 25338, "\u0120Grab": 25339, "\u0120undertake": 25340, "Members": 25341, "\u0120Reader": 25342, "\u0120Sims": 25343, "god": 25344, "\u0120hypothetical": 25345, "scient": 25346, "\u0120AJ": 25347, "\u0120charism": 25348, "\u0120admissions": 25349, "\u0120Missile": 25350, "trade": 25351, "\u0120exercising": 25352, "\u0120Background": 25353, "Written": 25354, "\u0120vocals": 25355, "whether": 25356, "\u0120vi": 25357, "\u0120Winner": 25358, "\u0120litter": 25359, "\u0120Shooting": 25360, "STEM": 25361, "\u00e3\u0124\u00a1": 25362, "\u0120AFL": 25363, "\u0120variability": 25364, "\u0120eats": 25365, "\u0120DPS": 25366, "brow": 25367, "\u0120elephants": 25368, "\u0120strat": 25369, "\u0120\u00c5": 25370, "\u0120settlers": 25371, "Matthew": 25372, "\u0120inadvert": 25373, "HI": 25374, "\u0120IMF": 25375, "\u0120Goal": 25376, "\u0120nerves": 25377, "Johnson": 25378, "eye": 25379, "ablishment": 25380, "Thursday": 25381, "BILITY": 25382, "Had": 25383, "amoto": 25384, "hetamine": 25385, "eps": 25386, "\u0120mitochond": 25387, "\u0120compressed": 25388, "\u0120Trevor": 25389, "\u0120Animals": 25390, "Tool": 25391, "Lock": 25392, "\u0120tweak": 25393, "\u0120pinch": 25394, "\u0120cancellation": 25395, "Pot": 25396, "\u0120focal": 25397, "\u0120Astron": 25398, "173": 25399, "\u0120ASC": 25400, "\u0120OTHER": 25401, "umni": 25402, "\u0120demise": 25403, "dl": 25404, "\u00d9\u0127": 25405, "Semitism": 25406, "\u0120cracking": 25407, "\u0120collaborative": 25408, "\u0120explores": 25409, "sql": 25410, "\u0120herbs": 25411, "\u0120configurations": 25412, "mis": 25413, "\u0120Result": 25414, "acey": 25415, "\u0120Smoke": 25416, "\u0120sanct": 25417, "elia": 25418, "\u0120degener": 25419, "\u0120deepest": 25420, "\u0120screamed": 25421, "\u0120nap": 25422, "Software": 25423, "\u0120STAR": 25424, "EF": 25425, "\u0120Xin": 25426, "sponsored": 25427, "manship": 25428, "233": 25429, "\u0120primaries": 25430, "\u0120filtering": 25431, "\u0120assemble": 25432, "mil": 25433, "\u0120Myers": 25434, "bows": 25435, "\u0120punched": 25436, "Mic": 25437, "\u0120innovations": 25438, "\u0120func": 25439, "ando": 25440, "\u0120fracking": 25441, "\u0120Vul": 25442, "\u00d0\u00be\u00d0": 25443, "oshop": 25444, "\u0120Immun": 25445, "\u0120settling": 25446, "\u0120adolescents": 25447, "\u0120rebuilding": 25448, "\u0120transforming": 25449, "\u0120parole": 25450, "\u0120harbor": 25451, "\u0120booking": 25452, "otional": 25453, "ongevity": 25454, "\u0120Yo": 25455, "bug": 25456, "\u0120emerges": 25457, "\u0120Methods": 25458, "\u0120Chu": 25459, "Pres": 25460, "\u0120Dungeons": 25461, "\u0120trailing": 25462, "\u0120Rum": 25463, "\u0120Hugh": 25464, "\u00e5\u00a4\u00a9": 25465, "\u0120Era": 25466, "\u0120Battles": 25467, "Results": 25468, "\u0120Trading": 25469, "\u0120versa": 25470, "css": 25471, "axies": 25472, "heet": 25473, "\u0120greed": 25474, "1989": 25475, "\u0120gardens": 25476, "\u0120contingent": 25477, "Park": 25478, "\u0120Leafs": 25479, "hook": 25480, "robe": 25481, "\u0120diplomacy": 25482, "\u0120Fuel": 25483, "\u0120Invasion": 25484, "\u0120upgrading": 25485, "Male": 25486, "\u0120elic": 25487, "\u0120relentless": 25488, "\u0120Covenant": 25489, "apesh": 25490, "\u0120Trop": 25491, "Ty": 25492, "production": 25493, "arty": 25494, "\u0120punches": 25495, "ako": 25496, "cyclopedia": 25497, "\u0120Rabbit": 25498, "\u0120HDMI": 25499, "\u0120141": 25500, "\u0120foil": 25501, "ItemImage": 25502, "\u0120FG": 25503, "\u0120implementations": 25504, "\u0120Pom": 25505, "ixtures": 25506, "\u0120await": 25507, "\u0120330": 25508, "amus": 25509, "\u0120umbrella": 25510, "\u0120foresee": 25511, "separ": 25512, "\u0120circumcision": 25513, "\u0120peripheral": 25514, "Say": 25515, "\u0120Expert": 25516, "Inc": 25517, "\u0120withdrew": 25518, "\u0120Anders": 25519, "fried": 25520, "\u0120radioactive": 25521, "\u0120Opening": 25522, "\u0120boarding": 25523, "\u0120ND": 25524, "\u0120overthrow": 25525, "Activ": 25526, "WP": 25527, "\u0120Acts": 25528, "\u00d7\u013b": 25529, "\u0120motions": 25530, "vic": 25531, "\u0120Mighty": 25532, "\u0120Defender": 25533, "aer": 25534, "\u0120thankful": 25535, "\u0120Killing": 25536, "\u0120Bris": 25537, "moil": 25538, "\u0120predicting": 25539, "266": 25540, "choice": 25541, "\u0120killers": 25542, "\u0120incub": 25543, "\u0120Chest": 25544, "athering": 25545, "\u0120proclaimed": 25546, "flower": 25547, "ossom": 25548, "umbledore": 25549, "\u0120Cycling": 25550, "\u0120Occupy": 25551, "AGES": 25552, "Pen": 25553, "\u0120Yug": 25554, "\u0120packaged": 25555, "\u0120heightened": 25556, "cot": 25557, "stack": 25558, "Cond": 25559, "\u0120stamps": 25560, "mage": 25561, "\u0120persuaded": 25562, "\u0120ensl": 25563, "\u0120Cardinal": 25564, "\u0120solitary": 25565, "\u0120possessing": 25566, "\u0120Cork": 25567, "\u0120evid": 25568, "\u0120Tay": 25569, "\u0120blues": 25570, "\u0120extremism": 25571, "\u0120lunar": 25572, "\u0120clown": 25573, "Techn": 25574, "\u0120festivals": 25575, "\u0120PvP": 25576, "\u0120Lar": 25577, "\u0120consequently": 25578, "present": 25579, "\u0120someday": 25580, "\u00e7\u0130\u012d": 25581, "\u0120Meteor": 25582, "\u0120touring": 25583, "culture": 25584, "\u0120beaches": 25585, "Ship": 25586, "cause": 25587, "\u0120Flood": 25588, "\u00e3\u0125\u00af": 25589, "\u0120purity": 25590, "those": 25591, "\u0120emission": 25592, "bolt": 25593, "\u0120chord": 25594, "\u0120Scripture": 25595, "Lu": 25596, "\u0120${": 25597, "created": 25598, "Others": 25599, "258": 25600, "\u0120elemental": 25601, "\u0120annoyed": 25602, "\u0120AE": 25603, "dan": 25604, "\u0120Sag": 25605, "Researchers": 25606, "\u0120fairy": 25607, "\u00e2\u0122\u0135\u00e2\u0122\u0135": 25608, "============": 25609, "Smart": 25610, "GGGG": 25611, "\u0120skeletons": 25612, "\u0120pupils": 25613, "linked": 25614, "\u0120urgency": 25615, "enabled": 25616, "\u0120Fuck": 25617, "\u0120councill": 25618, "rab": 25619, "UAL": 25620, "TI": 25621, "\u0120lifes": 25622, "\u0120confessed": 25623, "Bug": 25624, "\u0120harmon": 25625, "\u0120CONFIG": 25626, "\u0120Neutral": 25627, "Double": 25628, "\u0120staple": 25629, "\u0120SHA": 25630, "British": 25631, "\u0120SNP": 25632, "ATOR": 25633, "oco": 25634, "\u0120swinging": 25635, "gex": 25636, "oleon": 25637, "plain": 25638, "\u0120Missing": 25639, "\u0120Trophy": 25640, "vari": 25641, "ranch": 25642, "\u0120301": 25643, "440": 25644, "0000000000000000": 25645, "\u0120restoring": 25646, "\u0120haul": 25647, "ucing": 25648, "nerg": 25649, "\u0120futures": 25650, "\u0120strategist": 25651, "question": 25652, "\u0120lateral": 25653, "\u0120Bard": 25654, "\u0120sor": 25655, "\u0120Rhodes": 25656, "\u0120Downtown": 25657, "?????-": 25658, "\u0120Lit": 25659, "\u0120Bened": 25660, "\u0120coil": 25661, "street": 25662, "\u0120Portal": 25663, "FILE": 25664, "\u0120Gru": 25665, "*,": 25666, "231": 25667, "neum": 25668, "\u0120sucked": 25669, "\u0120rapper": 25670, "\u0120tendencies": 25671, "\u0120Lauren": 25672, "cellaneous": 25673, "267": 25674, "\u0120browse": 25675, "\u0120overc": 25676, "header": 25677, "oise": 25678, "\u0120beet": 25679, "\u0120Gle": 25680, "Stay": 25681, "\u0120mum": 25682, "\u0120typed": 25683, "\u0120discounts": 25684, "Talk": 25685, "\u0120Og": 25686, "existing": 25687, "\u0120Sell": 25688, "uph": 25689, "CI": 25690, "\u0120Austrian": 25691, "\u0120Warm": 25692, "\u0120dismissal": 25693, "\u0120averages": 25694, "camera": 25695, "\u0120allegiance": 25696, "LAN": 25697, "=\"#": 25698, "\u0120commentators": 25699, "\u0120Setting": 25700, "\u0120Midwest": 25701, "\u0120pharmac": 25702, "\u0120EXP": 25703, "\u0120stainless": 25704, "Chicago": 25705, "\u0120tan": 25706, "244": 25707, "\u0120countryside": 25708, "\u0120Vac": 25709, "295": 25710, "\u0120pinned": 25711, "\u0120crises": 25712, "\u0120standardized": 25713, "Task": 25714, "\u0120Jail": 25715, "\u0120Docker": 25716, "colored": 25717, "forth": 25718, "\"},": 25719, "\u0120patrons": 25720, "\u0120spice": 25721, "\u0120mourn": 25722, "\u0120Mood": 25723, "\u0120laundry": 25724, "\u0120equip": 25725, "\u0120Mole": 25726, "yll": 25727, "\u0120THC": 25728, "nation": 25729, "\u0120Sherlock": 25730, "\u0120issu": 25731, "\u0120Kre": 25732, "\u0120Americas": 25733, "\u0120AAA": 25734, "\u0120systematically": 25735, "\u0120contra": 25736, "\u0120Sally": 25737, "\u0120rationale": 25738, "\u0120carriage": 25739, "\u0120peaks": 25740, "\u0120contradiction": 25741, "ensation": 25742, "\u0120Failure": 25743, "\u0120props": 25744, "\u0120namespace": 25745, "\u0120cove": 25746, "fields": 25747, "\u00e3\u0124\u012d": 25748, "\u0120wool": 25749, "\u0120Catch": 25750, "\u0120presumed": 25751, "\u0120Diana": 25752, "ragon": 25753, "igi": 25754, "\u0120hamm": 25755, "\u0120stunt": 25756, "\u0120GUI": 25757, "\u0120Observatory": 25758, "\u0120Shore": 25759, "\u0120smells": 25760, "annah": 25761, "\u0120cockpit": 25762, "\u0120Duterte": 25763, "850": 25764, "\u0120oppressed": 25765, "breaker": 25766, "\u0120Contribut": 25767, "\u0120Peru": 25768, "\u0120Monsanto": 25769, "\u0120Attempt": 25770, "\u0120commanding": 25771, "\u0120fridge": 25772, "\u0120Rin": 25773, "\u0120Chess": 25774, "uality": 25775, "\u0120ol": 25776, "Republican": 25777, "\u0120Glory": 25778, "\u0120WIN": 25779, ".......": 25780, "agent": 25781, "reading": 25782, "\u0120inh": 25783, "Jones": 25784, "\u0120clicks": 25785, "alan": 25786, "\u0120[];": 25787, "\u0120Majesty": 25788, "\u0120Ced": 25789, "opus": 25790, "atel": 25791, "\u00c3\u00aa": 25792, "ARC": 25793, "\u0120Ecuador": 25794, "\u00e3\u0125\u0142": 25795, "\u0120Kuro": 25796, "\u0120rituals": 25797, "\u0120captive": 25798, "\u0120ounce": 25799, "\u0120disagreement": 25800, "\u0120slog": 25801, "fuel": 25802, "Pet": 25803, "Mail": 25804, "\u0120exercised": 25805, "\u0120solic": 25806, "\u0120rainfall": 25807, "\u0120devotion": 25808, "\u0120Assessment": 25809, "\u0120robotic": 25810, "options": 25811, "\u0120RP": 25812, "\u0120Families": 25813, "\u0120Flames": 25814, "\u0120assignments": 25815, "007": 25816, "akedown": 25817, "\u0120vocabulary": 25818, "Reilly": 25819, "\u0120caval": 25820, "gars": 25821, "\u0120suppressed": 25822, "\u0120SET": 25823, "\u0120Johns": 25824, "\u0120warp": 25825, "broken": 25826, "\u0120statues": 25827, "\u0120advocated": 25828, "\u0120275": 25829, "\u0120peril": 25830, "omorph": 25831, "\u0120Femin": 25832, "perfect": 25833, "\u0120hatch": 25834, "Lib": 25835, "512": 25836, "\u0120lifelong": 25837, "313": 25838, "\u0120cheeks": 25839, "\u0120numbered": 25840, "\u0120Mug": 25841, "Body": 25842, "ravel": 25843, "Weight": 25844, "\u0120Jak": 25845, "\u0120Heath": 25846, "\u0120kissing": 25847, "\u0120JUST": 25848, "\u0120waving": 25849, "upload": 25850, "\u0120insider": 25851, "\u0120Progressive": 25852, "\u0120Filter": 25853, "tta": 25854, "\u0120Beam": 25855, "\u0120violently": 25856, "ipation": 25857, "\u0120skepticism": 25858, "\u01201918": 25859, "\u0120Annie": 25860, "\u0120SI": 25861, "\u0120genetics": 25862, "\u0120onboard": 25863, "atl": 25864, "\u0120Friedman": 25865, "\u0120Bri": 25866, "ceptive": 25867, "\u0120pirate": 25868, "\u0120Reporter": 25869, "278": 25870, "\u0120mythology": 25871, "\u0120eclipse": 25872, "\u0120skins": 25873, "\u0120glyph": 25874, "ingham": 25875, "Files": 25876, "Cour": 25877, "women": 25878, "\u0120regimes": 25879, "\u0120photographed": 25880, "Kat": 25881, "\u0120MAX": 25882, "Officials": 25883, "\u0120unexpectedly": 25884, "\u0120impressions": 25885, "Front": 25886, ";;;;;;;;": 25887, "\u0120supremacy": 25888, "\u0120sang": 25889, "\u0120aggravated": 25890, "\u0120abruptly": 25891, "\u0120Sector": 25892, "\u0120excuses": 25893, "\u0120costing": 25894, "idepress": 25895, "Stack": 25896, "\u0120RNA": 25897, "obil": 25898, "\u0120ghosts": 25899, "ldon": 25900, "atibility": 25901, "Topics": 25902, "\u0120reimburse": 25903, "\u0120HM": 25904, "\u0120Deg": 25905, "\u0120thief": 25906, "yet": 25907, "ogenesis": 25908, "leaning": 25909, "\u0120Kol": 25910, "\u0120Basketball": 25911, "\u0120fi": 25912, "\u0120Seeing": 25913, "\u0120recycling": 25914, "\u0120[-": 25915, "Congress": 25916, "\u0120lectures": 25917, "Psy": 25918, "\u0120nep": 25919, "\u0120maid": 25920, "\u0120oriented": 25921, "AX": 25922, "\u0120respectful": 25923, "rene": 25924, "flush": 25925, "\u0120Unloaded": 25926, "request": 25927, "grid": 25928, "\u0120Alternatively": 25929, "\u0120Hugo": 25930, "\u0120decree": 25931, "\u0120Buddhism": 25932, "andum": 25933, "Android": 25934, "\u0120Congo": 25935, "\u0120Joyce": 25936, "\u0120acknowledging": 25937, "hesive": 25938, "\u0120Tomorrow": 25939, "\u0120Hiro": 25940, "thren": 25941, "\u0120Maced": 25942, "\u0120hoax": 25943, "\u0120Increased": 25944, "\u0120Pradesh": 25945, "Wild": 25946, "______": 25947, "161": 25948, "\u0120aunt": 25949, "\u0120distributing": 25950, "\u0120Tucker": 25951, "\u0120SSL": 25952, "\u0120Wolves": 25953, "Building": 25954, "oult": 25955, "\u0120Luo": 25956, "\u0120Yas": 25957, "\u0120Spir": 25958, "\u0120Shape": 25959, "\u0120Cambod": 25960, "\u0120IPv": 25961, "\u0120ml": 25962, "\u0120extrad": 25963, "390": 25964, "\u0120Penny": 25965, "dream": 25966, "\u0120stationed": 25967, "optional": 25968, "eworthy": 25969, ".": 26700, "\u0120Workshop": 26701, "\u0120Retail": 26702, "\u0120Avatar": 26703, "625": 26704, "Na": 26705, "\u0120VC": 26706, "\u0120Secure": 26707, "MY": 26708, "1988": 26709, "ossip": 26710, "\u0120prostate": 26711, "\u0120unden": 26712, "\u0120gamer": 26713, "\u0120Contents": 26714, "\u0120Warhammer": 26715, "\u0120Sentinel": 26716, "310": 26717, "\u0120segregation": 26718, "\u0120Flex": 26719, "\u0120MAY": 26720, "\u0120drills": 26721, "\u0120Drugs": 26722, "Islamic": 26723, "\u0120spur": 26724, "\u0120cafe": 26725, "\u0120imaginary": 26726, "\u0120guiding": 26727, "\u0120swings": 26728, "\u0120Theme": 26729, "oby": 26730, "\u0120nud": 26731, "\u0120begging": 26732, "\u0120strongh": 26733, "\u0120rejecting": 26734, "\u0120pedestrians": 26735, "\u0120Prospect": 26736, "Rare": 26737, "sle": 26738, "\u0120concessions": 26739, "\u0120Constitutional": 26740, "\u0120beams": 26741, "\u0120fibers": 26742, "poon": 26743, "\u0120instincts": 26744, "property": 26745, "\u0120BIG": 26746, "Sanders": 26747, "imates": 26748, "\u0120coating": 26749, "\u0120corpses": 26750, "\u0120TRUE": 26751, "checked": 26752, "\u0120166": 26753, "Ash": 26754, "\u0120JS": 26755, "\u0120Fiction": 26756, "\u0120communal": 26757, "\u0120energetic": 26758, "oooooooo": 26759, "\u0120nowadays": 26760, "ILD": 26761, "ibo": 26762, "\u0120SUV": 26763, "Ren": 26764, "\u0120dwelling": 26765, "Silver": 26766, "\u0120tally": 26767, "\u0120Moving": 26768, "\u0120coward": 26769, "\u0120generals": 26770, "\u0120horns": 26771, "\u0120circulated": 26772, "\u0120robbed": 26773, "\u0120Unlimited": 26774, "\u0120harassed": 26775, "\u0120inhibit": 26776, "\u0120composer": 26777, "\u0120Spotify": 26778, "\u0120spreads": 26779, "364": 26780, "\u0120suicidal": 26781, "\u0120noises": 26782, "\u0120Stur": 26783, "\u0120saga": 26784, "\u0120Kag": 26785, "iso": 26786, "\u0120theoretically": 26787, "Money": 26788, "\u0120similarity": 26789, "\u0120sliced": 26790, "utils": 26791, "inges": 26792, "\"-": 26793, "\u0120anth": 26794, "\u0120imped": 26795, "Module": 26796, "Throughout": 26797, "\u0120menus": 26798, "committee": 26799, "andi": 26800, "obj": 26801, "inav": 26802, "fired": 26803, "\u0120Abdullah": 26804, "\u0120undead": 26805, "\u0120fonts": 26806, "Hold": 26807, "ENG": 26808, "\u0120sustainability": 26809, "\u0120flick": 26810, "\u0120razor": 26811, "\u0120Fest": 26812, "\u0120Characters": 26813, "\u0120wording": 26814, "\u0120populist": 26815, "\u0120criticizing": 26816, "\u0120muse": 26817, "vine": 26818, "\u0120cardboard": 26819, "\u0120kindly": 26820, "\u0120fringe": 26821, "\u0120Theft": 26822, "icultural": 26823, "\u0120governors": 26824, "\u0120\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 26825, "\u0120163": 26826, "\u0120timeout": 26827, "\u0120Auth": 26828, "Children": 26829, "AU": 26830, "\u0120redemption": 26831, "\u0120Alger": 26832, "\u01201914": 26833, "\u0120waved": 26834, "\u0120astronauts": 26835, "ograms": 26836, "\u0120swamp": 26837, "\u0120Finnish": 26838, "\u0120candle": 26839, "\u0120tonnes": 26840, "utm": 26841, "\u0120ray": 26842, "\u0120spun": 26843, "\u0120fearful": 26844, "articles": 26845, "\u0120caus": 26846, "orically": 26847, "\u0120Requires": 26848, "\u0120Gol": 26849, "\u0120pope": 26850, "\u0120inaugural": 26851, "\u0120gle": 26852, "ADA": 26853, "\u0120ISIL": 26854, "\u0120Offensive": 26855, "\u0120watchdog": 26856, "\u0120balcon": 26857, "entity": 26858, "\u0120Hoo": 26859, "\u0120gallon": 26860, "ACC": 26861, "\u0120doubling": 26862, "\u0120implication": 26863, "\u0120Sight": 26864, "\u0120doctr": 26865, "-------": 26866, "\u0120\\\\": 26867, "\u0120malt": 26868, "Roll": 26869, "\u0120\u00e2\u012b\u00a5": 26870, "\u0120recap": 26871, "adding": 26872, "uces": 26873, "\u0120Bend": 26874, "figure": 26875, "\u0120turkey": 26876, "\u0120societal": 26877, "\u0120Tickets": 26878, "\u0120commercially": 26879, "\u0120spicy": 26880, "\u0120216": 26881, "\u0120Ramp": 26882, "\u0120superiority": 26883, "\u00c3\u00af": 26884, "\u0120Tracker": 26885, "Carl": 26886, "\u0120Coy": 26887, "\u0120Patriot": 26888, "\u0120consulted": 26889, "\u0120listings": 26890, "\u0120slew": 26891, "reenshot": 26892, "\u0120Gone": 26893, "\u0120[...]": 26894, "309": 26895, "\u0120hottest": 26896, "\u00d8\u00b1": 26897, "\u0120rocky": 26898, "\u0120Diaz": 26899, "\u0120massage": 26900, "\u0120paraly": 26901, "\u0120pony": 26902, "Az": 26903, "\u0120cartridge": 26904, "\u0120NZ": 26905, "\u0120snack": 26906, "\u0120Lamar": 26907, "plement": 26908, "\u0120Leslie": 26909, "\u0120mater": 26910, "\u0120snipp": 26911, "246": 26912, "\u0120jointly": 26913, "\u0120Brisbane": 26914, "\u0120iPod": 26915, "\u0120pumping": 26916, "\u0120goat": 26917, "\u0120Sharon": 26918, "ealing": 26919, "\u0120coron": 26920, "\u0120anomal": 26921, "rahim": 26922, "\u0120Connection": 26923, "\u0120sculpture": 26924, "\u0120scheduling": 26925, "\u0120Daddy": 26926, "athing": 26927, "\u0120eyebrows": 26928, "\u0120curved": 26929, "\u0120sentiments": 26930, "\u0120drafting": 26931, "Drop": 26932, "([": 26933, "\u0120nominal": 26934, "\u0120Leadership": 26935, "\u0120Grow": 26936, "\u0120176": 26937, "\u0120constructive": 26938, "ivation": 26939, "\u0120corrupted": 26940, "gerald": 26941, "\u0120Cros": 26942, "\u0120Chester": 26943, "\u0120Lap": 26944, "\u00e3\u0123\u00aa": 26945, "OTH": 26946, "DATA": 26947, "\u0120almond": 26948, "probably": 26949, "Imp": 26950, "\u0120feast": 26951, "\u0120Warcraft": 26952, "Flor": 26953, "\u0120checkpoint": 26954, "\u0120transcription": 26955, "\u0120204": 26956, "\u0120tweaks": 26957, "\u0120relieve": 26958, "Science": 26959, "\u0120performer": 26960, "Zone": 26961, "\u0120turmoil": 26962, "igated": 26963, "hibit": 26964, "\u0120Cafe": 26965, "themed": 26966, "\u0120fluor": 26967, "bench": 26968, "\u0120decom": 26969, "\u0120Unt": 26970, "\u0120Barrett": 26971, "\u0120Facts": 26972, "\u0120tasting": 26973, "\u0120PTSD": 26974, "\u0120Seal": 26975, "\u0120Judaism": 26976, "\u0120Dynamic": 26977, "\u0120Cors": 26978, "Ve": 26979, "\u0120Ming": 26980, "\u0120Transform": 26981, "von": 26982, "\u0120Defenders": 26983, "\u0120Tactical": 26984, "\u0120Von": 26985, "\u0120Univers": 26986, "\u0120distorted": 26987, "\u0120Breath": 26988, "?'\"": 26989, "\u0120agon": 26990, "\u0120Deadly": 26991, "\u0120lan": 26992, "\u0120Cycle": 26993, "orned": 26994, "\u0120reliably": 26995, "\u0120glor": 26996, "\u0120Monkey": 26997, "\u00e3\u0125\u00a1": 26998, "\u0120adren": 26999, "\u0120microwave": 27000, "\u0120Alban": 27001, "ircraft": 27002, "digit": 27003, "smart": 27004, "\u0120Dread": 27005, "\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af\u00c2\u00af": 27006, "{{": 27007, "\u0120Rochester": 27008, "\u0120simplified": 27009, "\u0120inflicted": 27010, "\u0120takeover": 27011, "\u0120yourselves": 27012, "aditional": 27013, "\u0120muscular": 27014, "KS": 27015, "\u0120ingen": 27016, "Tax": 27017, "\u0120Feature": 27018, "277": 27019, "\u0120cruc": 27020, "\u0120crate": 27021, "\u0120unidentified": 27022, "\u0120acclaimed": 27023, "\u0120Manga": 27024, "\u0120Frances": 27025, "\u0120Nepal": 27026, "\u0120Gerald": 27027, "\u0120Kuwait": 27028, "\u0120slain": 27029, "\u0120Heb": 27030, "\u0120Goku": 27031, "\u00e3\u0123\u00ae\u00e6": 27032, "286": 27033, "Mrs": 27034, "\u0120Cody": 27035, "\u0120Sanctuary": 27036, "016": 27037, "\u0120dismant": 27038, "\u0120dataset": 27039, "\u0120Hond": 27040, "buck": 27041, "\u0120Patterson": 27042, "\u0120palette": 27043, "\u0120GD": 27044, "icol": 27045, "\u0120Lodge": 27046, "\u0120planetary": 27047, "akin": 27048, "\u0120Registered": 27049, "abwe": 27050, "\u0120Petersburg": 27051, "\u0120hailed": 27052, "\u0120Piece": 27053, "Sche": 27054, "\u0120DOJ": 27055, "\u0120enumer": 27056, "181": 27057, "\u0120Observer": 27058, "\u0120Bold": 27059, "founded": 27060, "commerce": 27061, "\u0120exploits": 27062, "\u0120Finding": 27063, "URN": 27064, "\u0120Sne": 27065, "\u0120Acid": 27066, "ayette": 27067, "\u0120Values": 27068, "\u0120drastic": 27069, "\u0120architectural": 27070, "\u0120\".": 27071, "\u00d7\u0137": 27072, "umped": 27073, "\u0120wrapping": 27074, "\u0120widow": 27075, "\u0120Slayer": 27076, "lace": 27077, "once": 27078, "Germany": 27079, "avoid": 27080, "\u0120temples": 27081, "PAR": 27082, "\u00c3\u00b4": 27083, "\u0120Lucifer": 27084, "\u0120Flickr": 27085, "lov": 27086, "forces": 27087, "\u0120scouting": 27088, "\u0120louder": 27089, "tesy": 27090, "\u0120beforehand": 27091, "\u00c4\u0135": 27092, "\u0120Neon": 27093, "\u0120Wol": 27094, "\u0120Typically": 27095, "\u0120Politico": 27096, "-+-+": 27097, "\u0120builder": 27098, "\u0120derive": 27099, "Kill": 27100, "\u0120poker": 27101, "\u0120ambiguous": 27102, "\u0120lifts": 27103, "\u0120cyt": 27104, "\u0120ribs": 27105, "oodle": 27106, "\u0120Sounds": 27107, "hair": 27108, "\u0120Syndrome": 27109, "tf": 27110, "\u0120proportional": 27111, "uid": 27112, "\u0120pertaining": 27113, "\u0120Kindle": 27114, "\u0120Negro": 27115, "\u0120reiterated": 27116, "\u0120Tonight": 27117, "oths": 27118, "\u0120Cornell": 27119, "\u0120owing": 27120, "\u0120208": 27121, "elfare": 27122, "ocating": 27123, "\u0120Birds": 27124, "Subscribe": 27125, "\u0120essays": 27126, "\u0120burdens": 27127, "\u0120illustrations": 27128, "arious": 27129, "ERAL": 27130, "\u0120Calcul": 27131, "\u0120xen": 27132, "\u0120LinkedIn": 27133, "\u0120Jung": 27134, "\u0120redesign": 27135, "Connor": 27136, "296": 27137, "\u0120reversal": 27138, "\u0120Adelaide": 27139, "\u0120LL": 27140, "\u0120sinking": 27141, "\u0120gum": 27142, "USH": 27143, "capt": 27144, "\u0120Grimm": 27145, "\u0120footsteps": 27146, "\u0120CBD": 27147, "ispers": 27148, "\u0120prose": 27149, "Wednesday": 27150, "\u0120Movies": 27151, "edin": 27152, "\u0120overturned": 27153, "\u0120contentious": 27154, "USB": 27155, "~~~~~~~~~~~~~~~~": 27156, "\u0120Copper": 27157, "\u0120pointless": 27158, "NV": 27159, "values": 27160, "olphin": 27161, "dain": 27162, "\u0120deposited": 27163, "\u0120GW": 27164, "\u0120preceded": 27165, "\u0120Cla": 27166, "\u0120Golem": 27167, "\u0120Nim": 27168, "\u0120\u00ce\u00b2": 27169, "\u0120Engineers": 27170, "middle": 27171, "\u0120flatt": 27172, "operative": 27173, "\u0120councils": 27174, "imbabwe": 27175, "elin": 27176, "\u0120stressful": 27177, "\u0120LD": 27178, "\u0120resh": 27179, "lake": 27180, "\u0120wheelchair": 27181, "\u0120Alternative": 27182, "\u0120optimize": 27183, "operation": 27184, "\u0120peek": 27185, "\u0120oneself": 27186, "igil": 27187, "\u0120transitions": 27188, "opathy": 27189, "blank": 27190, "\u0120169": 27191, "171": 27192, "________________________________________________________________": 27193, "\u0120laundering": 27194, "Enc": 27195, "\u0120DEC": 27196, "\u0120workouts": 27197, "\u0120spikes": 27198, "\u0120dinosaurs": 27199, "\u0120discriminatory": 27200, "Pool": 27201, "Rather": 27202, "385": 27203, "RNA": 27204, "testers": 27205, "eto": 27206, "\u0120Identity": 27207, "\u0120vein": 27208, "\u0120Burton": 27209, "\u0120arcade": 27210, "420": 27211, "Ultimately": 27212, "\u0120Sadly": 27213, "\u00c3\u00b0": 27214, "pill": 27215, "\u0120cubic": 27216, "\u0120Spectrum": 27217, "these": 27218, "states": 27219, "\u0120unofficial": 27220, "hawks": 27221, "\u0120EVERY": 27222, "\u0120rainbow": 27223, "\u0120incarceration": 27224, "anding": 27225, "\u0120syll": 27226, "\u0120Everton": 27227, "\u0120179": 27228, "\u0120Serbia": 27229, "\u0120189": 27230, "meter": 27231, "\u0120Mickey": 27232, "\u0120antiqu": 27233, "\u0120factual": 27234, "neck": 27235, "\u0120Nare": 27236, "norm": 27237, "must": 27238, "\u0120highways": 27239, "\u0120glam": 27240, "\u0120dividing": 27241, "\u0120Squadron": 27242, "\u0120Martha": 27243, "\u0120births": 27244, "Cover": 27245, "////////////////": 27246, "\u0120Wong": 27247, "Phot": 27248, "\u0120ALS": 27249, "rio": 27250, "\u0120Nonetheless": 27251, "\u0120Lemon": 27252, "\u0120206": 27253, "\u0120EE": 27254, "\u0120derivative": 27255, "\u0120WWII": 27256, "vote": 27257, "\u0120therein": 27258, "\u0120separating": 27259, "446": 27260, "sync": 27261, "\u0120Streets": 27262, "\u0120ratt": 27263, "\u0120municipality": 27264, "\u0120Shortly": 27265, "\u0120monk": 27266, "),\"": 27267, "\u0120scrub": 27268, "\u0120operatives": 27269, "Neither": 27270, "Place": 27271, "\u0120Limit": 27272, "Female": 27273, "\u0120Actor": 27274, "Character": 27275, "\u0120constituted": 27276, "357": 27277, "\u0120protested": 27278, "\u0120Straw": 27279, "\u0120Height": 27280, "ilda": 27281, "\u0120Typh": 27282, "\u0120floods": 27283, "\u0120cosmetic": 27284, "WAY": 27285, "perture": 27286, "upon": 27287, "tons": 27288, "essing": 27289, "\u0120Pocket": 27290, "\u0120rooft": 27291, "\u0120Caucas": 27292, "\u0120antidepress": 27293, "\u0120incompatible": 27294, "ECD": 27295, "\u0120opera": 27296, "\u0120Contest": 27297, "\u0120generators": 27298, "lime": 27299, "Defense": 27300, "1987": 27301, "forum": 27302, "\u0120savage": 27303, "\u0120Hungarian": 27304, "nz": 27305, "\u0120metallic": 27306, "\u0120expelled": 27307, "\u0120residency": 27308, "\u0120dresses": 27309, "666": 27310, "\u0120Clement": 27311, "fires": 27312, "Category": 27313, "\u0120geek": 27314, "alis": 27315, "\u0120cemetery": 27316, "educated": 27317, "\u0120crawl": 27318, "\u0120Unable": 27319, "\u0120Tyson": 27320, "akis": 27321, "\u0120pardon": 27322, "\u0120Wra": 27323, "\u0120strengthened": 27324, "\u0120Fors": 27325, "335": 27326, "\u0120HC": 27327, "\u0120Mond": 27328, "\u0120visuals": 27329, "\u0120Beatles": 27330, "ettlement": 27331, "\u0120\u00ef": 27332, "gro": 27333, "\u0120bash": 27334, "\u0120poorest": 27335, "\u0120excel": 27336, "\u0120aspirations": 27337, "\u0120Municip": 27338, "ensible": 27339, "\u0120ceremonies": 27340, "\u0120intimidation": 27341, "\u0120CONTR": 27342, "beck": 27343, "\u0120Kap": 27344, "asu": 27345, "\u0120trademarks": 27346, "\u0120Sew": 27347, "\u0120Competition": 27348, "network": 27349, "\u0120Arri": 27350, "\u0120Tet": 27351, "Roaming": 27352, "WC": 27353, "Dat": 27354, "\u0120sob": 27355, "\u0120pairing": 27356, "\u0120overdose": 27357, "SAY": 27358, "aber": 27359, "\u0120revolt": 27360, "\u0120Fah": 27361, "acting": 27362, "eq": 27363, "estation": 27364, "Fight": 27365, "\u0120Marks": 27366, "273": 27367, "\u0120178": 27368, "Raw": 27369, "\u00e3\u0123\u012d": 27370, "349": 27371, "blocks": 27372, "\u0120verge": 27373, "estine": 27374, "\u0120Podesta": 27375, "\u0120invasive": 27376, "\u0120profoundly": 27377, "\u0120Ao": 27378, "each": 27379, "\u0120lest": 27380, "interpret": 27381, "\u0120shrinking": 27382, "\u0120errone": 27383, "\u0120chees": 27384, "lys": 27385, "\u0120Ivy": 27386, "\u0120Directory": 27387, "\u0120hinted": 27388, "VICE": 27389, "\u0120contacting": 27390, "\u0120Gent": 27391, "hei": 27392, "\u0120labeling": 27393, "\u0120mercury": 27394, "\u0120Lite": 27395, "\u0120expires": 27396, "\u0120destabil": 27397, "ritis": 27398, "cu": 27399, "\u0120feathers": 27400, "\u0120steer": 27401, "\u0120programmed": 27402, "\u0120Vader": 27403, "Going": 27404, "\u0120Elim": 27405, "\u0120yo": 27406, "\u0120Miche": 27407, "\u0120203": 27408, "\u0120sleeves": 27409, "\u0120bully": 27410, "\u0120Humans": 27411, "368": 27412, "\u0120compress": 27413, "\u0120Banner": 27414, "ARS": 27415, "\u0120awhile": 27416, "\u0120calib": 27417, "\u0120sponsorship": 27418, "\u0120Difficulty": 27419, "\u0120Papers": 27420, "\u0120identifier": 27421, "}.": 27422, "\u0120yog": 27423, "\u0120Shia": 27424, "\u0120cleanup": 27425, "\u0120vibe": 27426, "introdu": 27427, "imming": 27428, "Australia": 27429, "\u0120outlines": 27430, "\u0120Youtube": 27431, "train": 27432, "\u0120Makes": 27433, "\u0120deported": 27434, "\u0120centr": 27435, "\u0120Dug": 27436, "\u0120Boulder": 27437, "\u0120Buffy": 27438, "\u0120injunction": 27439, "\u0120Harley": 27440, "\u0120Groups": 27441, "\u0120Dumbledore": 27442, "\u0120Clara": 27443, "\u0120\"-": 27444, "\u0120sacrificed": 27445, "eph": 27446, "Shadow": 27447, "ibling": 27448, "\u0120freelance": 27449, "\u0120evidently": 27450, "phal": 27451, "\u0120retains": 27452, "Mir": 27453, "\u0120finite": 27454, "dar": 27455, "\u0120Cous": 27456, "\u0120repaired": 27457, "\u0120periodic": 27458, "\u0120championships": 27459, "\u0120asteroid": 27460, "blind": 27461, "\u0120expressly": 27462, "\u0120Astros": 27463, "\u0120scaled": 27464, "\u0120geographical": 27465, "\u0120Rapids": 27466, "Enjoy": 27467, "\u0120elastic": 27468, "\u0120Mohamed": 27469, "Market": 27470, "begin": 27471, "\u0120discovers": 27472, "\u0120telecommunications": 27473, "\u0120scanner": 27474, "\u0120enlarge": 27475, "\u0120sharks": 27476, "\u0120psychedel": 27477, "\u0120Rouge": 27478, "\u0120snapshot": 27479, "isine": 27480, "XP": 27481, "\u0120pesticides": 27482, "\u0120LSD": 27483, "\u0120Distribution": 27484, "really": 27485, "\u0120degradation": 27486, "\u0120disguise": 27487, "\u0120biom": 27488, "\u0120EXT": 27489, "\u0120equations": 27490, "\u0120hazards": 27491, "\u0120Compared": 27492, ")*": 27493, "\u0120virtues": 27494, "\u0120elders": 27495, "\u0120enhancing": 27496, "\u0120Across": 27497, "eros": 27498, "angling": 27499, "\u0120combust": 27500, "ucci": 27501, "\u0120concussion": 27502, "\u0120contraception": 27503, "\u0120Kang": 27504, "\u0120expresses": 27505, "\u0120aux": 27506, "\u0120Pione": 27507, "\u0120exhibits": 27508, "Debug": 27509, "OTAL": 27510, "\u0120Already": 27511, "\u0120Wheeler": 27512, "\u0120expands": 27513, "?:": 27514, "\u0120reconciliation": 27515, "\u0120pirates": 27516, "\u0120purse": 27517, "\u0120discourage": 27518, "\u0120spectacle": 27519, "Rank": 27520, "\u0120wraps": 27521, "\u0120Thought": 27522, "\u0120impending": 27523, "Opp": 27524, "\u0120Anglo": 27525, "\u0120EUR": 27526, "\u0120screwed": 27527, "retched": 27528, "\u0120encouragement": 27529, "models": 27530, "\u0120confuse": 27531, "mmm": 27532, "\u0120Vitamin": 27533, "\u00e2\u0138\u0133\u00e2\u0138\u0133": 27534, "Cru": 27535, "\u0120knights": 27536, "\u0120discard": 27537, "\u0120bishops": 27538, "\u0120Wear": 27539, "\u0120Garrett": 27540, "kan": 27541, "\u00e3\u0125\u0141": 27542, "\u0120masculine": 27543, "capital": 27544, "\u0120Aus": 27545, "\u0120fatally": 27546, "thanks": 27547, "\u0120AU": 27548, "\u0120Gut": 27549, "1200": 27550, "\u012000000000": 27551, "\u0120surrog": 27552, "\u0120BIOS": 27553, "raits": 27554, "\u0120Watts": 27555, "\u0120resurrection": 27556, "\u0120Electoral": 27557, "\u0120Tips": 27558, "4000": 27559, "\u0120nutrient": 27560, "\u0120depicting": 27561, "\u0120sprink": 27562, "\u0120muff": 27563, "\u0120LIM": 27564, "\u0120Sample": 27565, "psc": 27566, "ibi": 27567, "generated": 27568, "\u0120specimens": 27569, "\u0120dissatisf": 27570, "\u0120tailored": 27571, "\u0120holdings": 27572, "\u0120Monthly": 27573, "\u0120Eat": 27574, "poons": 27575, "\u0120nec": 27576, "\u0120Cage": 27577, "\u0120Lotus": 27578, "\u0120Lantern": 27579, "\u0120frontier": 27580, "\u0120pensions": 27581, "\u0120joked": 27582, "\u0120Hardy": 27583, "=-=-=-=-": 27584, "rade": 27585, "UID": 27586, "\u0120rails": 27587, "\u0120emit": 27588, "\u0120slate": 27589, "\u0120smug": 27590, "\u0120spit": 27591, "\u0120Calls": 27592, "\u0120Jacobs": 27593, "feat": 27594, "\u0120UE": 27595, "\u0120restruct": 27596, "\u0120regeneration": 27597, "\u0120energies": 27598, "\u0120Connor": 27599, "OHN": 27600, "\u0120Cheese": 27601, "\u0120ger": 27602, "\u0120resurrect": 27603, "management": 27604, "NW": 27605, "\u0120presently": 27606, "\u0120Bruins": 27607, "Member": 27608, "\u0120Mang": 27609, "idan": 27610, "\u0120boosting": 27611, "wyn": 27612, "+.": 27613, "requisite": 27614, "\u0120NYPD": 27615, "\u0120Megan": 27616, "\u0120Conditions": 27617, "\u0120pics": 27618, "nesium": 27619, "\u0120Rash": 27620, "\u0120174": 27621, "\u0120Ducks": 27622, "\u0120embro": 27623, "zu": 27624, "onian": 27625, "religious": 27626, "\u0120craz": 27627, "\u0120ACA": 27628, "\u0120Zucker": 27629, "EMA": 27630, "\u0120Pros": 27631, "Weapon": 27632, "\u0120Knox": 27633, "\u0120Arduino": 27634, "\u0120stove": 27635, "\u0120heavens": 27636, "\u0120Purchase": 27637, "\u0120herd": 27638, "\u0120fundraiser": 27639, "Digital": 27640, "5000": 27641, "\u0120proponents": 27642, "/\u00e2\u0122\u012d": 27643, "\u0120jelly": 27644, "\u0120Visa": 27645, "\u0120monks": 27646, "\u0120advancement": 27647, "\u0120Wer": 27648, "\u0120187": 27649, "eus": 27650, "ertility": 27651, "\u0120fetal": 27652, "\u01201936": 27653, "Lo": 27654, "\u0120outfits": 27655, "\u0120staircase": 27656, "bomb": 27657, "\u0120customized": 27658, "clair": 27659, "Tree": 27660, "\u0120mapped": 27661, "\u0120Considering": 27662, "\u0120Torres": 27663, "\u0120methyl": 27664, "\u0120approximate": 27665, "\u0120doom": 27666, "\u0120Hansen": 27667, "\u0120crossover": 27668, "\u0120standalone": 27669, "\u00e4\u00bc": 27670, "\u0120invites": 27671, "\u0120graveyard": 27672, "\u0120hp": 27673, "DonaldTrump": 27674, "\u0120escort": 27675, "Gar": 27676, "\u0120predecessors": 27677, "\u0120hay": 27678, "\u0120enzyme": 27679, "\u0120Straight": 27680, "visors": 27681, "Ing": 27682, "aneously": 27683, "\u0120Applied": 27684, "\u0120fec": 27685, "\u0120Durant": 27686, "\u0120outspoken": 27687, "orb": 27688, "\u0120zeal": 27689, "\u0120disgrace": 27690, "').": 27691, "\u0120Cheng": 27692, "289": 27693, "\u0120Rena": 27694, "\u0120Suicide": 27695, "294": 27696, "\u0120outraged": 27697, "\u0120Newman": 27698, "\u0120Nvidia": 27699, "\u0120Aber": 27700, "\u0120Bers": 27701, "\u0120recreation": 27702, "Window": 27703, "\u0120DP": 27704, "xe": 27705, "\u0120pedoph": 27706, "\u0120fallout": 27707, "amboo": 27708, "\u0120presentations": 27709, "\u0120Apps": 27710, "\u0120html": 27711, "345": 27712, "\u0120XXX": 27713, "\u0120rubbing": 27714, "\u0120Leather": 27715, "\u0120humidity": 27716, "seys": 27717, "established": 27718, "\u0120Units": 27719, "646": 27720, "\u0120respectable": 27721, "Auto": 27722, "\u0120thriving": 27723, "\u0120Innovation": 27724, "angs": 27725, "Extra": 27726, "regulation": 27727, "298": 27728, "pick": 27729, "Examples": 27730, "\u0120CJ": 27731, "Attack": 27732, "\u0120dracon": 27733, "LT": 27734, "\u0120sticker": 27735, "rers": 27736, "\u0120sunny": 27737, "Iss": 27738, "regulated": 27739, "dim": 27740, "\u0120Abstract": 27741, "\u0120husbands": 27742, "Office": 27743, "omination": 27744, "itars": 27745, "ANGE": 27746, "ascal": 27747, "\u0120Kris": 27748, "\u0120Infantry": 27749, "\u0120malf": 27750, "\u0120Athe": 27751, "\u0120Rally": 27752, "balanced": 27753, "........................": 27754, "OUP": 27755, "\u0120molecule": 27756, "metics": 27757, "\u0120Split": 27758, "\u0120Instructions": 27759, "\u0120Nights": 27760, "cards": 27761, "\u0120tug": 27762, "\u0120cone": 27763, "\u00e5\u0143": 27764, "\u0120tx": 27765, "\u0120Discussion": 27766, "\u0120catastrophe": 27767, "ppe": 27768, "gio": 27769, "\u0120communism": 27770, "\u0120halted": 27771, "\u0120Guant": 27772, "clean": 27773, "\u0120Sched": 27774, "\u0120Kanye": 27775, "\u0120wander": 27776, "\u0120Seriously": 27777, "\u0120188": 27778, "ennial": 27779, "follow": 27780, "productive": 27781, "\u0120Flow": 27782, "\u0120Sail": 27783, "\u0120craw": 27784, "\u0120simulations": 27785, "oru": 27786, "angles": 27787, "\u0120Nolan": 27788, "\u0120menstru": 27789, "470": 27790, "\u0120207": 27791, "aja": 27792, "\u0120casually": 27793, "boarding": 27794, "\u0120222": 27795, "ovy": 27796, "\u0120Numbers": 27797, "umat": 27798, "OE": 27799, "287": 27800, "\u0120Clemson": 27801, "\u0120certs": 27802, "\u0120slid": 27803, "\u0120Tribe": 27804, "\u0120toast": 27805, "\u0120fortunes": 27806, "\u0120fals": 27807, "\u0120Committees": 27808, "\u0120gp": 27809, "\u0120fiery": 27810, "\u0120Nets": 27811, "\u0120Anime": 27812, "Package": 27813, "\u0120Compare": 27814, "laughter": 27815, "infect": 27816, "\u0120atrocities": 27817, "\u0120justices": 27818, "\u0120insults": 27819, "\u0120Vernon": 27820, "\u0120shaken": 27821, "\u0120persona": 27822, "estamp": 27823, "367": 27824, "brain": 27825, "\u0120experimenting": 27826, "Ken": 27827, "\u0120Electronics": 27828, "\u0120161": 27829, "domain": 27830, "\u0120graphical": 27831, "bishop": 27832, "\u0120whopping": 27833, "\u0120Evangel": 27834, "\u0120advertisers": 27835, "\u0120Spear": 27836, "\u0120bids": 27837, "\u0120destroys": 27838, "utz": 27839, "\u0120undersc": 27840, "\u0120ADD": 27841, "\u0120ants": 27842, "\u0120Cum": 27843, "ipples": 27844, "\u0120Fill": 27845, "\u0120glanced": 27846, "\u0120indicted": 27847, "\u0120Eff": 27848, "\u0120miscon": 27849, "\u0120Desktop": 27850, "\u0120abide": 27851, "\u00e3\u0125\u0122": 27852, "\u0120Io": 27853, "\u0120Coul": 27854, "\u0120capsule": 27855, "\u0120Chrys": 27856, "MON": 27857, "\u0120undes": 27858, "\u0120IRA": 27859, "\u0120citation": 27860, "\u0120dictate": 27861, "\u0120Networks": 27862, "\u0120Conflict": 27863, "\u0120Stuff": 27864, "xa": 27865, "isec": 27866, "\u0120Chemistry": 27867, "\u0120quarterly": 27868, "Williams": 27869, "anan": 27870, "Opt": 27871, "\u0120Alexandria": 27872, "outheastern": 27873, "\u0120Springfield": 27874, "\u0120Blacks": 27875, "\u0120geography": 27876, "242": 27877, "\u0120utmost": 27878, "\u0120Exxon": 27879, "abouts": 27880, "EVA": 27881, "\u0120Enable": 27882, "\u0120Barr": 27883, "\u0120disagreed": 27884, "\u0120Cyprus": 27885, "\u0120dementia": 27886, "\u0120labs": 27887, "\u0120ubiquitous": 27888, "\u0120LOVE": 27889, "\u0120consolidated": 27890, "sr": 27891, "\u0120creamy": 27892, "\u0120Timber": 27893, "Regardless": 27894, "\u0120Certificate": 27895, "\u0120\"...": 27896, "ogenous": 27897, "Captain": 27898, "\u0120insulting": 27899, "\u0120Soros": 27900, "\u0120Instr": 27901, "\u0120Bulgaria": 27902, "better": 27903, "\u0120sucking": 27904, "\u0120Davidson": 27905, "atz": 27906, "\u0120collateral": 27907, "gif": 27908, "\u0120plagued": 27909, "\u0120Cancel": 27910, "\u0120Gardner": 27911, "RB": 27912, "\u0120sixteen": 27913, "Remove": 27914, "uristic": 27915, "cook": 27916, "Rod": 27917, "\u0120comprising": 27918, "fle": 27919, ")\u00e2\u0122\u0136": 27920, "\u0120Viking": 27921, "growth": 27922, "agonal": 27923, "\u0120srf": 27924, "afety": 27925, "mot": 27926, "Nearly": 27927, "stown": 27928, "\u0120Factor": 27929, "\u0120automobile": 27930, "\u0120procedural": 27931, "mask": 27932, "ampires": 27933, "\u0120disappears": 27934, "jab": 27935, "315": 27936, "\u01201951": 27937, "needed": 27938, "\u0120daring": 27939, "leader": 27940, "\u0120podium": 27941, "\u0120unhealthy": 27942, "\u0120mund": 27943, "\u0120pyramid": 27944, "ocre": 27945, "\u0120kissed": 27946, "\u0120dreamed": 27947, "\u0120Fantastic": 27948, "\u0120Gly": 27949, "\u00e5\u012c": 27950, "\u0120greatness": 27951, "\u0120spices": 27952, "\u0120metropolitan": 27953, "\u0120compuls": 27954, "iets": 27955, "1016": 27956, "\u0120Sham": 27957, "\u0120Pyr": 27958, "flies": 27959, "\u0120Midnight": 27960, "\u0120swallowed": 27961, "\u0120genres": 27962, "\u0120Lucky": 27963, "\u0120Rewards": 27964, "\u0120dispatch": 27965, "\u0120IPA": 27966, "\u0120Apply": 27967, "\u0120aven": 27968, "alities": 27969, "312": 27970, "things": 27971, "\u0120().": 27972, "\u0120mates": 27973, "\u0120Sz": 27974, "\u0120COP": 27975, "olate": 27976, "OFF": 27977, "\u0120recharge": 27978, "caps": 27979, "\u0120Yorker": 27980, "icone": 27981, "\u0120galaxies": 27982, "ileaks": 27983, "Dave": 27984, "\u0120Puzz": 27985, "\u0120Celtic": 27986, "\u0120AFC": 27987, "276": 27988, "\u0120Sons": 27989, "\u0120affirmative": 27990, "Hor": 27991, "\u0120tutorials": 27992, "\u0120CITY": 27993, "\u0120Rosa": 27994, "\u0120Extension": 27995, "Series": 27996, "\u0120fats": 27997, "\u0120rab": 27998, "lis": 27999, "\u0120unic": 28000, "\u0120eve": 28001, "\u0120Spin": 28002, "\u0120adulthood": 28003, "typ": 28004, "\u0120sectarian": 28005, "\u0120checkout": 28006, "\u0120Cycl": 28007, "Single": 28008, "\u0120martyr": 28009, "\u0120chilling": 28010, "888": 28011, "oufl": 28012, "\u0120];": 28013, "\u0120congestion": 28014, "mk": 28015, "\u0120Whereas": 28016, "\u01201938": 28017, "urrencies": 28018, "erion": 28019, "\u0120boast": 28020, "\u0120Patients": 28021, "\u0120chap": 28022, "\u0120BD": 28023, "realDonaldTrump": 28024, "\u0120examines": 28025, "hov": 28026, "\u0120startling": 28027, "\u0120Babylon": 28028, "wid": 28029, "omew": 28030, "brance": 28031, "\u0120Odyssey": 28032, "wig": 28033, "\u0120torch": 28034, "\u0120Vox": 28035, "\u0120Moz": 28036, "\u0120Troll": 28037, "\u0120Ans": 28038, "Similarly": 28039, "\u0120Ful": 28040, "006": 28041, "Unless": 28042, "\u0120Alone": 28043, "stead": 28044, "\u0120Publisher": 28045, "rights": 28046, "tu": 28047, "\u0120Doesn": 28048, "\u0120professionally": 28049, "\u0120clo": 28050, "icz": 28051, "\u0120steals": 28052, "\u0120\u00e1": 28053, "1986": 28054, "\u0120sturdy": 28055, "\u0120Johann": 28056, "\u0120medals": 28057, "\u0120filings": 28058, "\u0120Fraser": 28059, "done": 28060, "\u0120multinational": 28061, "\u0120feder": 28062, "\u0120worthless": 28063, "\u0120pest": 28064, "Yesterday": 28065, "ankind": 28066, "\u0120gays": 28067, "\u0120borne": 28068, "\u0120POS": 28069, "Picture": 28070, "\u0120percentages": 28071, "251": 28072, "rame": 28073, "\u0120potions": 28074, "AMD": 28075, "\u0120Lebanese": 28076, "\u0120rang": 28077, "\u0120LSU": 28078, "ongs": 28079, "\u0120peninsula": 28080, "\u0120Clause": 28081, "ALK": 28082, "oha": 28083, "\u0120MacBook": 28084, "\u0120unanimous": 28085, "\u0120lenders": 28086, "\u0120hangs": 28087, "\u0120franchises": 28088, "orers": 28089, "\u0120Updates": 28090, "\u0120isolate": 28091, "andro": 28092, "Soon": 28093, "\u0120disruptive": 28094, "\u0120Surve": 28095, "\u0120stitches": 28096, "\u0120Scorp": 28097, "\u0120Dominion": 28098, "\u0120supplying": 28099, "Arg": 28100, "\u0120turret": 28101, "\u0120Luk": 28102, "\u0120brackets": 28103, "*)": 28104, "\u0120Revolutionary": 28105, "\u0120Honest": 28106, "\u0120noticing": 28107, "\u0120Shannon": 28108, "\u0120afforded": 28109, "\u0120tha": 28110, "\u0120Janet": 28111, "!--": 28112, "\u0120Narendra": 28113, "\u0120Plot": 28114, "Hol": 28115, "sever": 28116, "eenth": 28117, "\u0120obstruction": 28118, "\u01201024": 28119, "staff": 28120, "jas": 28121, "orget": 28122, "scenes": 28123, "laughs": 28124, "\u0120Fargo": 28125, "crime": 28126, "\u0120orchestr": 28127, "\u0120delet": 28128, "iliary": 28129, "rieved": 28130, "\u0120militar": 28131, "\u0120Greene": 28132, "\u00e2\u0139\u0131": 28133, "\u00e3\u0123\u00a6": 28134, "\u0120Guards": 28135, "\u0120unleashed": 28136, "\u0120Weber": 28137, "\u0120adjustable": 28138, "\u0120caliber": 28139, "\u0120motivations": 28140, "\u0120\u00c3\u0142": 28141, "mAh": 28142, "\u0120Lanka": 28143, "handle": 28144, "\u0120pent": 28145, "\u0120Rav": 28146, "\u0120Angular": 28147, "\u0120Kau": 28148, "umbing": 28149, "\u0120philanthrop": 28150, "\u0120dehyd": 28151, "\u0120toxicity": 28152, "eer": 28153, "\u0120YORK": 28154, "witz": 28155, "\u00e5\u00bc": 28156, "\u0120IE": 28157, "community": 28158, "\u0120AH": 28159, "\u0120retali": 28160, "\u0120massively": 28161, "\u0120Daniels": 28162, "\u0120DEL": 28163, "\u0120carcin": 28164, "Url": 28165, "\u0120routing": 28166, "\u0120NPCs": 28167, "\u0120RAF": 28168, "ryce": 28169, "\u0120waived": 28170, "\u0120Guatem": 28171, "Everybody": 28172, "\u0120covenant": 28173, "\u0120173": 28174, "\u0120relaxing": 28175, "\u0120quart": 28176, "almost": 28177, "\u0120guarded": 28178, "\u0120Soldiers": 28179, "\u0120PLAY": 28180, "\u0120outgoing": 28181, "LAND": 28182, "\u0120rewrite": 28183, "\u0120MOV": 28184, "\u0120Imper": 28185, "\u0120Solution": 28186, "\u0120phenomenal": 28187, "\u0120longevity": 28188, "\u0120impat": 28189, "\u0120Nissan": 28190, "irie": 28191, "\u0120odor": 28192, "\u0120Zar": 28193, "oks": 28194, "\u0120militias": 28195, "\u0120SPEC": 28196, "\u0120tolerated": 28197, "arser": 28198, "\u0120Bradford": 28199, "+,": 28200, "\u0120surreal": 28201, "sf": 28202, "Canadian": 28203, "\u0120resemblance": 28204, "\u0120carbohydrate": 28205, "VIEW": 28206, "\u0120accessory": 28207, "meal": 28208, "largest": 28209, "iegel": 28210, "Someone": 28211, "\u0120toughest": 28212, "oso": 28213, "\u0120funnel": 28214, "\u0120condemnation": 28215, "luent": 28216, "\u0120wired": 28217, "\u0120Sunset": 28218, "Jesus": 28219, "\u0120PST": 28220, "\u0120Pages": 28221, "\u0120Tycoon": 28222, "\u0120PF": 28223, "\u0120selections": 28224, "\u0120\u00e0\u00a4": 28225, "partisan": 28226, "\u0120highs": 28227, "\u0120Rune": 28228, "\u0120crafts": 28229, "lead": 28230, "\u0120Parents": 28231, "\u0120reclaim": 28232, "eker": 28233, "\u0120Allied": 28234, "aeper": 28235, "\u0120looming": 28236, "\u0120beneficiaries": 28237, "\u0120Hull": 28238, "Students": 28239, "Jewish": 28240, "dj": 28241, "\u0120pact": 28242, "template": 28243, "\u0120Officials": 28244, "\u0120Baylor": 28245, "\u0120hemp": 28246, "\u0120youths": 28247, "\u0120Levels": 28248, "\u0120Xiao": 28249, "\u0120Ches": 28250, "\u0120endeavor": 28251, "\u0120Removed": 28252, "\u0120hippocamp": 28253, "Hell": 28254, "\u00e3\u0124\u012c": 28255, "805": 28256, "\u0120dinosaur": 28257, "\u0120Wrath": 28258, "\u0120Indonesian": 28259, "\u0120calculator": 28260, "\u0120Dictionary": 28261, "\u0120420": 28262, "\u0120MAG": 28263, "(_": 28264, "!,": 28265, "tarians": 28266, "\u0120restricting": 28267, "racuse": 28268, "\u0120weekday": 28269, "OUNT": 28270, "\u0120shrugged": 28271, "leground": 28272, "\u0120bald": 28273, "\u0120Doctors": 28274, "\u0120touted": 28275, "\u0120Maxwell": 28276, "\u0120214": 28277, "\u0120diplomat": 28278, "\u0120repression": 28279, "\u0120constituency": 28280, "vice": 28281, "ranked": 28282, "\u0120Napoleon": 28283, "gang": 28284, "\u0120Forever": 28285, "tun": 28286, "\u0120bulb": 28287, "\u0120PDT": 28288, "\u0120Cisco": 28289, "VEN": 28290, "\u0120resumed": 28291, "Steven": 28292, "\u0120Manitoba": 28293, "\u0120fabulous": 28294, "\u0120Agents": 28295, "1984": 28296, "\u0120amusing": 28297, "\u0120Mysteries": 28298, "\u0120orthodox": 28299, "floor": 28300, "\u0120questionnaire": 28301, "\u0120penetrate": 28302, "\u0120filmmakers": 28303, "\u0120Unc": 28304, "\u0120stamped": 28305, "\u0120thirteen": 28306, "\u0120outfield": 28307, "\u0120forwarded": 28308, "\u0120appra": 28309, "\u0120aided": 28310, "try": 28311, "\u0120unfocused": 28312, "\u0120Liz": 28313, "\u0120Wendy": 28314, "\u0120Scene": 28315, "Charg": 28316, "\u0120rejects": 28317, "\u0120leftist": 28318, "\u0120Providence": 28319, "\u0120Brid": 28320, "regn": 28321, "\u0120prophecy": 28322, "\u0120LIVE": 28323, "499": 28324, "\u0120forge": 28325, "\u0120FML": 28326, "\u0120intrinsic": 28327, "\u0120Frog": 28328, "\u0120wont": 28329, "\u0120Holt": 28330, "\u0120famed": 28331, "CLUS": 28332, "aepernick": 28333, "\u0120Hate": 28334, "\u0120Cay": 28335, "\u0120registering": 28336, "ortality": 28337, "ropy": 28338, "ocalyptic": 28339, "aan": 28340, "nav": 28341, "\u0120fascist": 28342, "IFIED": 28343, "\u0120implicated": 28344, "\u0120Resort": 28345, "\u0120Chandler": 28346, "\u0120Brick": 28347, "Pin": 28348, "ysc": 28349, "Usage": 28350, "\u0120Helm": 28351, "usra": 28352, "\u00e2\u013a\u0127\u00e2\u013a\u0127": 28353, "\u0120Abbas": 28354, "\u0120unanimously": 28355, "\u0120keeper": 28356, "\u0120addicted": 28357, "???": 28358, "\u0120helmets": 28359, "\u0120antioxid": 28360, "apsed": 28361, "808": 28362, "giene": 28363, "\u0120waits": 28364, "\u0120minion": 28365, "raved": 28366, "\u0120Porsche": 28367, "\u0120dreaming": 28368, "\u0120171": 28369, "\u0120Cain": 28370, "\u0120unfor": 28371, "asso": 28372, "\u0120Configuration": 28373, "kun": 28374, "hardt": 28375, "\u0120nested": 28376, "\u0120LDS": 28377, "LES": 28378, "\u0120tying": 28379, "enos": 28380, "\u0120cue": 28381, "\u0120Marqu": 28382, "skirts": 28383, "\u0120clicked": 28384, "\u0120expiration": 28385, "\u0120Accordingly": 28386, "\u0120WC": 28387, "\u0120blessings": 28388, "\u0120addictive": 28389, "\u0120Narr": 28390, "yx": 28391, "\u0120Jaguars": 28392, "\u0120rents": 28393, "\u0120Siber": 28394, "\u0120tipped": 28395, "ousse": 28396, "\u0120Fitzgerald": 28397, "\u0120hierarch": 28398, "outine": 28399, "\u0120wavelength": 28400, ">.": 28401, "chid": 28402, "\u0120Processing": 28403, "/+": 28404, "ranking": 28405, "Easy": 28406, "\u0120Construct": 28407, "\u0120tet": 28408, "insured": 28409, "HUD": 28410, "\u0120quoting": 28411, "\u0120communicated": 28412, "inx": 28413, "\u0120inmate": 28414, "\u0120erected": 28415, "\u0120Absolutely": 28416, "\u0120Surely": 28417, "\u0120unim": 28418, "\u0120Throne": 28419, "heid": 28420, "\u0120claws": 28421, "\u0120superstar": 28422, "\u0120Lenn": 28423, "\u0120Whis": 28424, "Uk": 28425, "abol": 28426, "\u0120sket": 28427, "\u0120Niet": 28428, "\u0120perks": 28429, "\u0120affinity": 28430, "\u0120openings": 28431, "phasis": 28432, "\u0120discriminate": 28433, "Tip": 28434, "vc": 28435, "\u0120grinding": 28436, "\u0120Jenny": 28437, "\u0120asthma": 28438, "holes": 28439, "\u0120Homer": 28440, "\u0120registers": 28441, "\u0120Glad": 28442, "\u0120creations": 28443, "\u0120lithium": 28444, "\u0120applause": 28445, "until": 28446, "Justice": 28447, "\u0120Turks": 28448, "\u0120scandals": 28449, "\u0120bake": 28450, "tank": 28451, "Mech": 28452, "\u0120Means": 28453, "\u0120Maid": 28454, "Republicans": 28455, "isal": 28456, "windows": 28457, "\u0120Santos": 28458, "\u0120vegetation": 28459, "338": 28460, "tri": 28461, "\u0120flux": 28462, "insert": 28463, "\u0120clarified": 28464, "\u0120mortg": 28465, "\u0120Chim": 28466, "\u0120Tort": 28467, "\u0120disclaim": 28468, "metal": 28469, "\u0120Aside": 28470, "\u0120induction": 28471, "\u0120infl": 28472, "\u0120atheists": 28473, "amph": 28474, "\u0120ether": 28475, "\u0120Vital": 28476, "\u0120Built": 28477, "Mind": 28478, "\u0120weaponry": 28479, "SET": 28480, "\u0120186": 28481, "admin": 28482, "gam": 28483, "contract": 28484, "afa": 28485, "\u0120derivatives": 28486, "\u0120snacks": 28487, "\u0120churn": 28488, "Econom": 28489, "\u0120capped": 28490, "\u0120Understanding": 28491, "\u0120Hers": 28492, "\u0120Iz": 28493, "\u0120duct": 28494, "IENT": 28495, "aughty": 28496, "\u0120\u00e2\u013e\u0136": 28497, "\u0120NP": 28498, "\u0120sailing": 28499, "Initialized": 28500, "\u0120ted": 28501, "\u0120reactors": 28502, "\u0120Lomb": 28503, "\u0120choke": 28504, "\u0120Worm": 28505, "\u0120admiration": 28506, "\u0120swung": 28507, "ensibly": 28508, "\u0120rash": 28509, "\u0120Goals": 28510, "\u0120Important": 28511, "Shot": 28512, "\u0120Ras": 28513, "\u0120trainers": 28514, "\u0120Bun": 28515, "Working": 28516, "\u0120harmed": 28517, "\u0120Pandora": 28518, "\u0120LTE": 28519, "\u0120mushroom": 28520, "\u0120CHAR": 28521, "\u0120Fee": 28522, "\u0120Moy": 28523, "Born": 28524, "oliberal": 28525, "\u0120Martial": 28526, "\u0120gentlemen": 28527, "\u0120lingering": 28528, "Official": 28529, "\u0120graffiti": 28530, "\u0120Names": 28531, "Der": 28532, "\u0120quint": 28533, "istrate": 28534, "azeera": 28535, "\u0120NOTICE": 28536, "\u0120Florence": 28537, "\u0120payable": 28538, "\u0120depicts": 28539, "\u0120Species": 28540, "Heart": 28541, "\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122\u00e2\u0136\u0122": 28542, "\u0120enclosed": 28543, "Increases": 28544, "Daily": 28545, "\u0120Lis": 28546, "\u0120enactment": 28547, "\u0120Bacon": 28548, "\u0120Steele": 28549, "demand": 28550, "\u0120183": 28551, "\u0120mouths": 28552, "\u0120stranded": 28553, "\u0120enhancement": 28554, "011": 28555, "\u0120Whats": 28556, "\u0120healed": 28557, "eny": 28558, "\u0120Rab": 28559, "\u0120340": 28560, "\u0120Labyrinth": 28561, "roach": 28562, "\u0120Yosh": 28563, "\u0120Clippers": 28564, "\u0120concerts": 28565, "Internet": 28566, "355": 28567, "\u0120stickers": 28568, "\u0120termed": 28569, "\u0120Axe": 28570, "\u0120grandparents": 28571, "France": 28572, "\u0120Clim": 28573, "\u0120Uh": 28574, "ulic": 28575, "\u0120thrill": 28576, "centric": 28577, "\u0120Overview": 28578, "\u0120Conduct": 28579, "\u0120substantive": 28580, "\u0120182": 28581, "mur": 28582, "\u0120stray": 28583, "\u0120Coff": 28584, "\u0120repetitive": 28585, "\u0120Forgotten": 28586, "\u0120qualification": 28587, "ewitness": 28588, "\u0120Zimbabwe": 28589, "\u0120simulated": 28590, "\u0120JD": 28591, "253": 28592, "\u0120Ware": 28593, "\u0120unsc": 28594, "Times": 28595, "\u0120summons": 28596, "\u0120disconnected": 28597, "\u0120184": 28598, "cius": 28599, "\u0120Gujar": 28600, "odka": 28601, "\u0120erase": 28602, "\u0120Tobacco": 28603, "elected": 28604, "\u0120uncont": 28605, "\u0120Shepard": 28606, "\u0120Lamp": 28607, "\u0120alerted": 28608, "\u0120operative": 28609, "arna": 28610, "uint": 28611, "\u0120negligence": 28612, "acements": 28613, "\u0120supra": 28614, "\u0120prevail": 28615, "\u0120Shark": 28616, "\u0120belts": 28617, "\u00e3\u0123\u00ab": 28618, "\u0120tighter": 28619, "Engineers": 28620, "\u0120inactive": 28621, "\u0120exponent": 28622, "\u0120Willie": 28623, "aples": 28624, "\u0120heir": 28625, "\u0120Hits": 28626, "iann": 28627, "\u0120Says": 28628, "\u0120currents": 28629, "\u0120Bengal": 28630, "\u0120arist": 28631, "Buffer": 28632, "\u0120breeze": 28633, "\u0120Wesley": 28634, "Cola": 28635, "\u0120pronoun": 28636, "\u0120deed": 28637, "\u0120Kling": 28638, "\u0120oft": 28639, "\u0120inflict": 28640, "\u0120punishing": 28641, "\u0120nm": 28642, "iku": 28643, "ODUCT": 28644, "014": 28645, "\u0120subsidy": 28646, "\u0120DEA": 28647, "\u0120Herbert": 28648, "\u0120Jal": 28649, "Bank": 28650, "\u0120deferred": 28651, "\u0120shipment": 28652, "Bott": 28653, "\u0120alle": 28654, "bearing": 28655, "HTML": 28656, "Offline": 28657, "\u0120213": 28658, "\u0120scrolling": 28659, "\u0120scanned": 28660, "\u0120Libyan": 28661, "\u0120TOP": 28662, "chrom": 28663, "dt": 28664, "column": 28665, "PsyNetMessage": 28666, "Zero": 28667, "\u0120torso": 28668, "050": 28669, "\u00e2\u0137\u0132": 28670, "\u0120imperson": 28671, "\u0120Schwartz": 28672, "udic": 28673, "\u0120pissed": 28674, "\u0120Sapp": 28675, "257": 28676, "\u0120ISPs": 28677, "ogl": 28678, "\u0120supervised": 28679, "\u0120adolescent": 28680, "\u0120attained": 28681, "\u0120Delivery": 28682, "\u0120Bunny": 28683, "\u01201937": 28684, "\u0120miniature": 28685, "\u0120os": 28686, "\u0120370": 28687, "608": 28688, "\u0120Mourinho": 28689, "\u0120innate": 28690, "\u0120tempo": 28691, "\u0120NM": 28692, "\u0120Fallen": 28693, "009": 28694, "\u0120provocative": 28695, "Streamer": 28696, "\u0120Benedict": 28697, "\u0120Bolshe": 28698, "\u0120turtle": 28699, "\u0120PCB": 28700, "\u0120Equal": 28701, "Director": 28702, "\u0120Rend": 28703, "\u0120fluids": 28704, "Authorities": 28705, "\u0120cousins": 28706, "requency": 28707, "\u0120Neighbor": 28708, "sets": 28709, "shared": 28710, "Charles": 28711, "password": 28712, "\u0120gears": 28713, "\u0120211": 28714, "\u0120Hardware": 28715, "rika": 28716, "\u0120upstream": 28717, "Hom": 28718, "\u0120disproportionately": 28719, "ivities": 28720, "\u0120undefined": 28721, "\u0120electrons": 28722, "\u0120commemor": 28723, "Eventually": 28724, "\u0120><": 28725, "\u0120irresponsible": 28726, "218": 28727, "\u0120Released": 28728, "\u0120OVER": 28729, "\u0120IGN": 28730, "\u0120Bread": 28731, "stellar": 28732, "\u0120Sage": 28733, "tted": 28734, "damage": 28735, "edition": 28736, "\u0120Prec": 28737, "\u0120lime": 28738, "\u0120confinement": 28739, "\u0120calorie": 28740, "weapon": 28741, "\u0120differing": 28742, "\u0120Sina": 28743, "mys": 28744, "amd": 28745, "\u0120intricate": 28746, "kk": 28747, "\u0120PAT": 28748, "\u00c3\u00a3o": 28749, "stones": 28750, "links": 28751, "\u0120ranch": 28752, "Semitic": 28753, "\u0120differentiate": 28754, "\u0120Singer": 28755, "occupied": 28756, "\u0120fortress": 28757, "cmd": 28758, "\u0120interception": 28759, "\u0120Ankara": 28760, "\u0120rept": 28761, "\u0120Solitaire": 28762, "\u0120remake": 28763, "pred": 28764, "\u0120dared": 28765, "autions": 28766, "\u0120BACK": 28767, "Running": 28768, "\u0120debugging": 28769, "\u0120graphs": 28770, "399": 28771, "\u0120Nigel": 28772, "\u0120bun": 28773, "\u0120pillow": 28774, "\u0120progressed": 28775, "fashioned": 28776, "\u0120obedience": 28777, "ERN": 28778, "\u0120rehears": 28779, "Cell": 28780, "tl": 28781, "Sher": 28782, "\u0120herald": 28783, "\u0120Payment": 28784, "\u0120Cory": 28785, "\u0120Dept": 28786, "\u0120repent": 28787, "\u0120Weak": 28788, "uckland": 28789, "\u0120pleasing": 28790, "\u0120shortages": 28791, "\u0120jurors": 28792, "\u0120Kab": 28793, "qqa": 28794, "Anti": 28795, "\u0120wow": 28796, "\u0120RCMP": 28797, "\u0120tsun": 28798, "\u0120Sic": 28799, "\u0120comprises": 28800, "\u0120spies": 28801, "\u0120precinct": 28802, "nu": 28803, "\u0120urges": 28804, "\u0120timed": 28805, "\u0120stripes": 28806, "\u0120Boots": 28807, "\u0120yen": 28808, "Advanced": 28809, "\u0120discrete": 28810, "\u0120Archangel": 28811, "employment": 28812, "Diff": 28813, "\u0120monuments": 28814, "\u0120209": 28815, "worker": 28816, "\u0120196": 28817, "\u0120Ig": 28818, "utterstock": 28819, "TPS": 28820, "Jac": 28821, "\u0120homelessness": 28822, "\u0120commentator": 28823, "\u0120racially": 28824, "fing": 28825, "seed": 28826, "Ele": 28827, "ellation": 28828, "\u0120ethanol": 28829, "\u0120parish": 28830, "\u0120Dong": 28831, "\u0120Awakening": 28832, "\u0120deviation": 28833, "\u0120Bearing": 28834, "\u0120Tsuk": 28835, "\u0120recess": 28836, "\u0120lymph": 28837, "\u0120Cannabis": 28838, "\u00e5\u013e": 28839, "\u0120NEWS": 28840, "\u0120dra": 28841, "\u0120Stefan": 28842, "\u0120Wrong": 28843, "\u0120SAM": 28844, "\u0120loosely": 28845, "\u0120interpreter": 28846, "\u0120Plain": 28847, "Government": 28848, "\u0120bigotry": 28849, "\u0120grenades": 28850, "avez": 28851, "pictured": 28852, "\u0120mandated": 28853, "\u0120Monk": 28854, "\u0120Pedro": 28855, "\u0120lava": 28856, "274": 28857, "\u0120cynical": 28858, "\u0120Scrolls": 28859, "locks": 28860, "Mp": 28861, "\u0120congregation": 28862, "ornings": 28863, "phil": 28864, "\u0120Ibid": 28865, "\u0120ferv": 28866, "\u0120disappearing": 28867, "\u0120arrogant": 28868, "syn": 28869, "\u0120Maver": 28870, "\u0120Suit": 28871, "241": 28872, "\u0120abbre": 28873, "ackers": 28874, "Pa": 28875, "\u0120Yel": 28876, "Whenever": 28877, "\u0120235": 28878, "\u0120Vine": 28879, "\u0120Anat": 28880, "\u0120extinct": 28881, "LET": 28882, "\u0120executable": 28883, "VERS": 28884, "oxide": 28885, "DNA": 28886, "\u0120Prel": 28887, "\u0120resentment": 28888, "\u0120comprise": 28889, "\u0120Aviv": 28890, "\u0120interceptions": 28891, "\u0120prolific": 28892, "INA": 28893, "\u0120Erin": 28894, "thought": 28895, "219": 28896, "\u0120Psychiatry": 28897, "unky": 28898, "chemist": 28899, "Ho": 28900, "\u0120McCoy": 28901, "\u0120bricks": 28902, "Los": 28903, "rily": 28904, "\u0120USSR": 28905, "\u0120rud": 28906, "\u0120laud": 28907, "\u0120Wise": 28908, "\u0120Emerald": 28909, "\u0120revived": 28910, "\u0120damned": 28911, "\u0120Repair": 28912, "idem": 28913, "ctica": 28914, "\u0120patriarch": 28915, "\u0120Nurs": 28916, "meg": 28917, "\u0120cheapest": 28918, "reements": 28919, "empty": 28920, "\u0120Celebr": 28921, "\u0120deprivation": 28922, "chanted": 28923, "\u0120Thumbnails": 28924, "Energy": 28925, "\u0120Ethan": 28926, "\u0120Qing": 28927, "\u0120opposes": 28928, "WIND": 28929, "vik": 28930, "\u0120Mau": 28931, "\u0120SUB": 28932, "667": 28933, "GRE": 28934, "\u0120Volunte": 28935, "nton": 28936, "Cook": 28937, "\u00e5\u0132": 28938, "esque": 28939, "\u0120plummet": 28940, "\u0120suing": 28941, "\u0120pronounce": 28942, "\u0120resisting": 28943, "\u0120Fishing": 28944, "\u0120Trials": 28945, "\u0120yell": 28946, "\u0120310": 28947, "\u0120induct": 28948, "\u0120personalized": 28949, "often": 28950, "Reb": 28951, "EMBER": 28952, "\u0120viewpoint": 28953, "\u0120existential": 28954, "())": 28955, "remove": 28956, "MENTS": 28957, "lasses": 28958, "\u0120evapor": 28959, "\u0120aisle": 28960, "meta": 28961, "\u0120reflective": 28962, "\u0120entitlement": 28963, "\u0120devised": 28964, "music": 28965, "ascade": 28966, "\u0120winding": 28967, "offset": 28968, "\u0120accessibility": 28969, "kered": 28970, "Better": 28971, "\u0120Johnston": 28972, "thinking": 28973, "Snow": 28974, "\u0120Croatia": 28975, "\u0120Atomic": 28976, "271": 28977, "348": 28978, "\u0120textbook": 28979, "\u0120Sixth": 28980, "\u0120\u00d8\u00a7\u00d9\u0126": 28981, "\u0120slider": 28982, "\u0120Burger": 28983, "bol": 28984, "Sync": 28985, "\u0120grandchildren": 28986, "\u0120cerv": 28987, "+)": 28988, "\u0120eternity": 28989, "\u0120tweeting": 28990, "\u0120speculative": 28991, "\u0120pivotal": 28992, "\u0120WP": 28993, "\u0120TER": 28994, "ynamic": 28995, "\u0120upl": 28996, "\u0120Cats": 28997, "perhaps": 28998, "\u0120classmates": 28999, "\u0120blatant": 29000, "'-": 29001, "\u0120lakh": 29002, "antine": 29003, "\u0120Borg": 29004, "iom": 29005, "/(": 29006, "\u0120Athletic": 29007, "\u0120sar": 29008, "OTA": 29009, "\u0120Hoffman": 29010, "Nevertheless": 29011, "\u0120adorable": 29012, "\u0120spawned": 29013, "Associated": 29014, "\u0120Domestic": 29015, "\u0120implant": 29016, "\u0120Luxem": 29017, "\u0120Kens": 29018, "\u0120pumps": 29019, "\u0120SAT": 29020, "Attributes": 29021, "509": 29022, "avour": 29023, "\u0120centralized": 29024, "\u0120TN": 29025, "\u0120freshly": 29026, "\u0120Achieve": 29027, "\u0120outsiders": 29028, "herty": 29029, "\u0120Ree": 29030, "\u0120Towers": 29031, "\u0120Dart": 29032, "akable": 29033, "\u0120mp": 29034, "\u0120Heavenly": 29035, "\u0120ripe": 29036, "\u0120Caroline": 29037, "ryan": 29038, "\u0120classics": 29039, "\u0120retiring": 29040, "\u0120228": 29041, "\u0120ah": 29042, "\u0120dealings": 29043, "\u0120punching": 29044, "\u0120Chapman": 29045, "Options": 29046, "maxwell": 29047, "volume": 29048, "\u0120stal": 29049, "\u0120exported": 29050, "\u0120Quite": 29051, "\u0120numerical": 29052, "Burn": 29053, "Fact": 29054, "\u0120Keystone": 29055, "\u0120trending": 29056, "\u0120altering": 29057, "\u0120Africans": 29058, "478": 29059, "\u0120MN": 29060, "\u0120Knock": 29061, "\u0120temptation": 29062, "\u0120prestige": 29063, "Overview": 29064, "\u0120Traditional": 29065, "\u0120Bahrain": 29066, "Private": 29067, "\u0120HOU": 29068, "\u0120barr": 29069, "\u0120Tat": 29070, "Cube": 29071, "USD": 29072, "\u0120Grande": 29073, "\u0120Gat": 29074, "\u0120Flo": 29075, "\u0120resides": 29076, "\u0120indec": 29077, "volent": 29078, "\u0120perpetual": 29079, "ubes": 29080, "\u0120worldview": 29081, "\u0120Quantum": 29082, "\u0120filtered": 29083, "\u0120ensu": 29084, "orgetown": 29085, "ERSON": 29086, "\u0120Mild": 29087, "379": 29088, "OTT": 29089, "\u00c3\u00a5": 29090, "\u0120vitamins": 29091, "\u0120ribbon": 29092, "\u0120sincerely": 29093, "\u0120Hin": 29094, "\u0120eighteen": 29095, "\u0120contradictory": 29096, "\u0120glaring": 29097, "\u0120expectancy": 29098, "\u0120conspir": 29099, "\u0120monstrous": 29100, "\u0120380": 29101, "reci": 29102, "\u0120handic": 29103, "\u0120pumped": 29104, "\u0120indicative": 29105, "\u0120rapp": 29106, "\u0120avail": 29107, "\u0120LEGO": 29108, "\u0120Marijuana": 29109, "1985": 29110, "erton": 29111, "\u0120twentieth": 29112, "################################": 29113, "\u0120Swamp": 29114, "\u0120valuation": 29115, "\u0120affiliates": 29116, "adjusted": 29117, "\u0120Facility": 29118, "262": 29119, "\u0120enzymes": 29120, "itudinal": 29121, "\u0120imprint": 29122, "Site": 29123, "\u0120installer": 29124, "\u0120TRA": 29125, "mology": 29126, "linear": 29127, "\u0120Collective": 29128, "igating": 29129, "\u0120Token": 29130, "\u0120speculated": 29131, "KN": 29132, "\u0120Cly": 29133, "ority": 29134, "\u0120defer": 29135, "\u0120inspectors": 29136, "approved": 29137, "RM": 29138, "\u0120Suns": 29139, "\u0120informing": 29140, "\u0120Syracuse": 29141, "ibli": 29142, "765": 29143, "\u0120glove": 29144, "\u0120authorize": 29145, "\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6\u00e2\u0122\u00a6": 29146, "\u0120Cruise": 29147, "\u0120contracting": 29148, "shell": 29149, "IFE": 29150, "\u0120Jewel": 29151, "pract": 29152, "\u0120Photoshop": 29153, "\u0120Knowing": 29154, "harm": 29155, "\u0120attractions": 29156, "adan": 29157, "etus": 29158, "018": 29159, "wagen": 29160, "Alt": 29161, "\u0120multiply": 29162, "\u0120equilibrium": 29163, ":{": 29164, "\u0120Fighters": 29165, "\u0120Edgar": 29166, "\u0120fourteen": 29167, "Govern": 29168, "\u0120misuse": 29169, "\u0120abusing": 29170, "\u0120ancestry": 29171, "ramer": 29172, "644": 29173, "\u0120worms": 29174, "\u0120thicker": 29175, "\u0120Combine": 29176, "\u0120peasants": 29177, "\u0120vind": 29178, "\u0120conquest": 29179, "\u0120mocked": 29180, "\u0120cinnamon": 29181, "\u0120Cald": 29182, "\u0120Gallup": 29183, "\u0120avoidance": 29184, "\u0120incarnation": 29185, "\u0120Strat": 29186, "\u0120tasted": 29187, "enta": 29188, "\u0120Neal": 29189, "pared": 29190, "\u0120terminology": 29191, "jection": 29192, "Scientists": 29193, "\u0120INS": 29194, "\u0120Dee": 29195, "\u0120directories": 29196, "Road": 29197, "\u0120Shap": 29198, "bright": 29199, "\u0120Directors": 29200, "\u0120Column": 29201, "\u0120bob": 29202, "\u0120preferably": 29203, "\u0120glitch": 29204, "furt": 29205, "\u0120eg": 29206, "idis": 29207, "CBC": 29208, "\u0120surrendered": 29209, "\u0120testament": 29210, "336": 29211, "uggest": 29212, "\u0120Nil": 29213, "another": 29214, "\u0120pathetic": 29215, "\u0120Donna": 29216, "\u0120218": 29217, "\u0120Avery": 29218, "\u0120whiskey": 29219, "\u0120fixture": 29220, "\u0120Conquest": 29221, "\u0120bets": 29222, "Occ": 29223, "\u0120Leicester": 29224, "].\"": 29225, "\u0120));": 29226, "\u0120flashes": 29227, "456": 29228, "\u0120masked": 29229, "gebra": 29230, "\u0120computed": 29231, "chel": 29232, "auder": 29233, "\u0120defeats": 29234, "\u0120Liberation": 29235, "\u0120Osama": 29236, "\u0120Vive": 29237, "Changes": 29238, "Channel": 29239, "\u0120tariffs": 29240, "\u0120mage": 29241, "\u0120Sax": 29242, "\u0120inadvertently": 29243, "\u0120CRE": 29244, "\u0120Reaper": 29245, "inky": 29246, "grading": 29247, "\u0120stereotyp": 29248, "\u0120curl": 29249, "\u0120FANT": 29250, "\u0120frameworks": 29251, "Mom": 29252, "\u0120Anch": 29253, "\u0120flavour": 29254, "carbon": 29255, "\u0120permitting": 29256, "letcher": 29257, "\u0120Mozilla": 29258, "\u0120Parking": 29259, "\u0120Champ": 29260, "Scroll": 29261, "\u0120murderer": 29262, "\u0120rested": 29263, "\u0120owes": 29264, "\u0120Poss": 29265, "ADD": 29266, "IFF": 29267, "resolution": 29268, "\u0120Mining": 29269, "\u0120comparative": 29270, "Dim": 29271, "\u0120neighbouring": 29272, "\u0120AST": 29273, "\u0120Toxic": 29274, "\u0120biases": 29275, "\u0120gunfire": 29276, "urous": 29277, "\u0120Moment": 29278, "1983": 29279, "\u0120pervasive": 29280, "ttp": 29281, "\u0120Normally": 29282, "rir": 29283, "Sarah": 29284, "\u0120Albany": 29285, "\u0120unsett": 29286, "\u0120SMS": 29287, "ipers": 29288, "layer": 29289, "\u0120Whites": 29290, "uple": 29291, "\u0120turbo": 29292, "\u0120Leeds": 29293, "\u0120thats": 29294, "\u0120Miner": 29295, "MER": 29296, "\u0120Reign": 29297, "\u0120perme": 29298, "\u0120Blitz": 29299, "\u01201934": 29300, "\u0120intimidating": 29301, "tube": 29302, "\u0120eccentric": 29303, "abolic": 29304, "boxes": 29305, "\u0120Associates": 29306, "votes": 29307, "\u0120simulate": 29308, "umbo": 29309, "astery": 29310, "\u0120shipments": 29311, "FFFF": 29312, "anth": 29313, "\u0120seasoned": 29314, "\u0120experimentation": 29315, "\u00e2\u0138\u0142": 29316, "laws": 29317, "Meet": 29318, "iddles": 29319, "antics": 29320, "Rating": 29321, "ISIS": 29322, "hift": 29323, "\u0120fronts": 29324, "buf": 29325, "017": 29326, "\u0120unatt": 29327, "\u0120Dil": 29328, "leases": 29329, "\u0120Gardens": 29330, "777": 29331, "touch": 29332, "vell": 29333, "458": 29334, "\u0120=====": 29335, "saving": 29336, "\u0120erosion": 29337, "\u0120Quin": 29338, "\u0120earns": 29339, "\u0120accomplishment": 29340, "\u0120Wei": 29341, "\u0120<[": 29342, "_____": 29343, "\u0120irrig": 29344, "\u0120Teddy": 29345, "\u0120conquered": 29346, "\u0120Armored": 29347, "\u0120asserts": 29348, "\u0120manipulating": 29349, "r\u00c3\u00a9": 29350, "\u0120transcripts": 29351, "Gallery": 29352, "\u0120plotting": 29353, "Neil": 29354, "\u0120betrayal": 29355, "loader": 29356, "\u0120Sul": 29357, "\u0120displacement": 29358, "\u0120royalty": 29359, "\u0120WI": 29360, "heit": 29361, "\u0120Devices": 29362, "allel": 29363, "\u0120municipalities": 29364, "\u0120canal": 29365, "Stars": 29366, "\u0120UAE": 29367, "\u0120\"\u00e2\u0122\u00a6": 29368, "\u0120CU": 29369, "above": 29370, "\u0120resonance": 29371, "\u0120guiActiveUn": 29372, "added": 29373, "\u0120Braves": 29374, "\u0120Ibn": 29375, "\u0120hereby": 29376, "\u0120BRE": 29377, "\u0120shareholder": 29378, "\u0120Hir": 29379, "\u0120Ji": 29380, "\u0120strangely": 29381, "\u0120admired": 29382, "\u0120plight": 29383, "\u0120bachelor": 29384, "\u0120Pole": 29385, "ciplinary": 29386, "Tony": 29387, "\u0120Armenian": 29388, "\u0120unman": 29389, "\u0120Zionist": 29390, "Stage": 29391, "iscover": 29392, "\u0120automotive": 29393, "\u0120sidelines": 29394, "\u0120slick": 29395, "\u0120Renaissance": 29396, "\u0120FUN": 29397, "Images": 29398, "\u0120Haj": 29399, "\u0120ping": 29400, "\u0120shortcut": 29401, "\u0120Blvd": 29402, "\u0120Looks": 29403, "\u0120bursts": 29404, "\u0120clamp": 29405, "\u0120mish": 29406, "\u0120sorting": 29407, "\u0120patriot": 29408, "\u0120correctness": 29409, "\u0120Scandinav": 29410, "\u0120Cavaliers": 29411, "python": 29412, "azar": 29413, "\u0120375": 29414, "\u0120Jaune": 29415, "409": 29416, "\u0120detrimental": 29417, "\u0120stabbing": 29418, "\u0120poisoned": 29419, "\u0120fountain": 29420, "ocent": 29421, "orst": 29422, "\u0120Mari": 29423, "\u0120rains": 29424, "\u0120Overs": 29425, "\u0120Institution": 29426, "udget": 29427, "AMY": 29428, "tale": 29429, "\u0120KR": 29430, "\u0120Prices": 29431, "\u0120headaches": 29432, "\u0120landsl": 29433, "\u0120Aura": 29434, "Bonus": 29435, "\u0120Zhao": 29436, "\u0120Hip": 29437, "\u0120hops": 29438, "\u0120Kurdistan": 29439, "\u0120exploiting": 29440, "ryn": 29441, "\u0120hypocrisy": 29442, "opening": 29443, "\u0120gunshot": 29444, "\u0120wed": 29445, "interstitial": 29446, "Interstitial": 29447, "\u0120amen": 29448, "Breaking": 29449, "\u0120marketed": 29450, "Wire": 29451, "\u0120Crowd": 29452, "Continue": 29453, "\u0120Known": 29454, "\u0120Effective": 29455, "orean": 29456, "izons": 29457, "Joseph": 29458, "\u0120escalation": 29459, "username": 29460, "\u0120curtain": 29461, "ATES": 29462, "\u0120PAR": 29463, "\u0120Miy": 29464, "\u0120counterfe": 29465, "lene": 29466, "\u0120contenders": 29467, "daily": 29468, "\u0120Asc": 29469, "\u0120Phillip": 29470, "mostly": 29471, "\u0120filename": 29472, "hene": 29473, "\u0120resembling": 29474, "\u0120staging": 29475, "\u0120Chloe": 29476, "\u0120wiring": 29477, "Hon": 29478, "\u0120Renew": 29479, "ottage": 29480, "\u0120Hybrid": 29481, "much": 29482, "\u0120strokes": 29483, "\u0120policymakers": 29484, "APTER": 29485, "\u0120Arkham": 29486, "plot": 29487, "\u0120assistants": 29488, "\u0120deport": 29489, "\u0120Sega": 29490, "\u0120influenza": 29491, "\u0120Cursed": 29492, "\u0120Kobe": 29493, "\u0120skinny": 29494, "Provider": 29495, "\u0120Rip": 29496, "\u0120incremental": 29497, "products": 29498, "BF": 29499, "\u0120dome": 29500, "\u0120Credits": 29501, "\u0120losers": 29502, "ints": 29503, "\u0120Betty": 29504, "\u0120Talent": 29505, "\u0120DAM": 29506, "Lv": 29507, "Ess": 29508, "\u0120dens": 29509, "temp": 29510, "Judge": 29511, "odic": 29512, "\u0120'(": 29513, "URES": 29514, "etsk": 29515, "VO": 29516, "\u0120retrieved": 29517, "\u0120architects": 29518, "\u00d9\u0129": 29519, "\u0120ethic": 29520, "\u0120Secondary": 29521, "stocks": 29522, "adia": 29523, "\u0120325": 29524, "\u0120Opinion": 29525, "\u0120simultaneous": 29526, "\u0120dizz": 29527, "ulp": 29528, "\u0120smuggling": 29529, "ippery": 29530, "Random": 29531, "facing": 29532, "\u0120Das": 29533, "\u0120stockp": 29534, "\u0120disclosures": 29535, "pointer": 29536, "\u0120coral": 29537, "\u0120Selection": 29538, "\u0120Pike": 29539, "ivalent": 29540, "\u0120ruthless": 29541, "\u0120Rim": 29542, "\u0120ensuing": 29543, "\u0120Experiment": 29544, "\u0120congressman": 29545, "\u0120believer": 29546, "\u0120unspecified": 29547, "\u0120Mord": 29548, "\u0120knowledgeable": 29549, "\u0120VERY": 29550, "TX": 29551, "\u0120straps": 29552, "\u0120turf": 29553, "apeshifter": 29554, "\u0120marital": 29555, "\u0120flock": 29556, "\u00e3\u0123\u0128": 29557, "263": 29558, "AMES": 29559, "\u0120Opposition": 29560, "\u0120treasures": 29561, "\u0120GOD": 29562, "\u0120modeled": 29563, "\u0120WORLD": 29564, "\u0120([": 29565, "\u0120Usage": 29566, "HF": 29567, "\u0120$(": 29568, "ussed": 29569, "\u0120pioneer": 29570, "Eight": 29571, "parse": 29572, "bread": 29573, "ritz": 29574, "\u0120Miranda": 29575, "\u0120Kant": 29576, "++)": 29577, "oren": 29578, "\u0120provoked": 29579, "\u0120breeds": 29580, "\u0120Includes": 29581, "\u0120Pastebin": 29582, "\u0120Flip": 29583, "Java": 29584, "\u0120brink": 29585, "\u0120rumored": 29586, "\u0120unseen": 29587, "\u0120garnered": 29588, "\u0120Defin": 29589, "alted": 29590, "\u0120tattoos": 29591, "\u0120hesitation": 29592, "isitions": 29593, "\u0120Weaver": 29594, "\u0120Reporting": 29595, "\u0120therapies": 29596, "\u0120consultants": 29597, "\u0120residual": 29598, "\u0120Mali": 29599, "\u0120Roma": 29600, "iago": 29601, "\u0120Residents": 29602, "ubi": 29603, "\u0120remedies": 29604, "\u0120adaptive": 29605, "\u0120Alive": 29606, "\u0120Barcl": 29607, "\u0120wallets": 29608, "crypt": 29609, "etermination": 29610, "\u0120Pelosi": 29611, "\u0120slipping": 29612, "otonin": 29613, "\u0120alliances": 29614, "patrick": 29615, "iris": 29616, "\u0120orth": 29617, "\u0120Perkins": 29618, "\u0120DeV": 29619, "\u0120Gets": 29620, "\u0120drying": 29621, "gee": 29622, "forest": 29623, "\u0120Forget": 29624, "orem": 29625, "339": 29626, "\u0120vaguely": 29627, "\u0120Dion": 29628, "\u0120Porn": 29629, "\u0120HOW": 29630, "\u0120pneum": 29631, "\u0120rubble": 29632, "\u0120Taste": 29633, "encia": 29634, "\u0120Gel": 29635, "\u0120dst": 29636, "\u0120245": 29637, "\u0120Morocco": 29638, "inflamm": 29639, "\u0120Twins": 29640, "\u0120bots": 29641, "daughter": 29642, "\u0120Balk": 29643, "\u0120brethren": 29644, "\u0120logos": 29645, "\u0120gobl": 29646, "fps": 29647, "\u0120subdivision": 29648, "\u0120pawn": 29649, "\u0120squeezed": 29650, "\u0120morale": 29651, "\u0120DW": 29652, "'\"": 29653, "\u0120knot": 29654, "ooky": 29655, "\u0120divisive": 29656, "\u0120boosted": 29657, "chy": 29658, "\u00e3\u0125\u0132": 29659, "ifact": 29660, "\u0120newcomers": 29661, "\u0120Wrestling": 29662, "\u0120scouts": 29663, "wolves": 29664, "Rat": 29665, "\u0120nineteenth": 29666, "\u0120Osborne": 29667, "Stats": 29668, "\u0120empowered": 29669, "\u0120psychopath": 29670, "\u0120OEM": 29671, "uggage": 29672, "\u0120PK": 29673, "\u0120Mohammad": 29674, "Pak": 29675, "\u0120anarchists": 29676, "\u0120Extract": 29677, "esthes": 29678, "\u0120Stockholm": 29679, "loo": 29680, "\u0120Graph": 29681, "\u0120deploying": 29682, "\u0120Stranger": 29683, "\u0120Mold": 29684, "\u0120staffer": 29685, "\u0120discounted": 29686, "uckle": 29687, "please": 29688, "\u0120Landing": 29689, "\u00c3\u0143a": 29690, "\u0120193": 29691, "\u0120ante": 29692, "\u0120repetition": 29693, "\u0120+/-": 29694, "\u0120parody": 29695, "\u0120lively": 29696, "AAA": 29697, "\u0120Horus": 29698, "\u0120pits": 29699, "inders": 29700, "LOC": 29701, "\u0120Venice": 29702, "406": 29703, "\u0120Discover": 29704, "\u00e2\u0128": 29705, "ellectual": 29706, "\u0120pens": 29707, "\u0120eyel": 29708, "iguous": 29709, "Impl": 29710, "\u0120joking": 29711, "\u0120inval": 29712, "\u0120Belfast": 29713, "\u0120creditors": 29714, "\u0120Skywalker": 29715, "ovsky": 29716, "\u0120ceasefire": 29717, "\u0120seals": 29718, "isoft": 29719, ")).": 29720, "\u0120Felix": 29721, "ITS": 29722, "\u0120tresp": 29723, "\u0120Blockchain": 29724, "eware": 29725, "\u0120Schwar": 29726, "enne": 29727, "mounted": 29728, "\u0120Beacon": 29729, "lesh": 29730, "\u0120immensely": 29731, "\u0120cheering": 29732, "Employ": 29733, "scene": 29734, "ishly": 29735, "atchewan": 29736, "\u0120Nicolas": 29737, "\u0120drained": 29738, "\u0120Exit": 29739, "\u0120Azerb": 29740, "jun": 29741, "\u0120floated": 29742, "uania": 29743, "Deep": 29744, "\u0120superv": 29745, "\u0120mystical": 29746, "\u0120Dollar": 29747, "\u0120Apostle": 29748, "\u0120REL": 29749, "\u0120Provided": 29750, "\u0120Bucks": 29751, "\u00e3\u0125\u00b4": 29752, "cutting": 29753, "\u0120enhancements": 29754, "\u0120Penguins": 29755, "\u0120Isaiah": 29756, "\u0120jerk": 29757, "\u0120Wyn": 29758, "\u0120stalled": 29759, "\u0120cryptocurrencies": 29760, "\u0120Roland": 29761, "single": 29762, "\u0120lumin": 29763, "\u0120Fellow": 29764, "\u0120Capacity": 29765, "\u0120Kazakh": 29766, "WN": 29767, "\u0120financed": 29768, "389": 29769, "\u0120tid": 29770, "\u0120collusion": 29771, "\u0120Myr": 29772, "\u00ee\u0122": 29773, "Senator": 29774, "\u0120pediatric": 29775, "\u0120neatly": 29776, "\u0120sandwiches": 29777, "\u0120Architecture": 29778, "\u0120tucked": 29779, "\u0120balcony": 29780, "\u0120earthquakes": 29781, "quire": 29782, "Future": 29783, "\u0120hefty": 29784, "\u00e9\u0139": 29785, "\u0120specializes": 29786, "\u0120stresses": 29787, "\u0120sender": 29788, "\u0120misunderstanding": 29789, "\u0120epile": 29790, "\u0120provoke": 29791, "\u0120Colors": 29792, "\u0120dismay": 29793, "uko": 29794, "[_": 29795, "586": 29796, "neutral": 29797, "\u0120donating": 29798, "\u0120Randall": 29799, "Multi": 29800, "\u0120conveniently": 29801, "\u0120Sung": 29802, "\u0120Coca": 29803, "\u0120tents": 29804, "\u0120Acceler": 29805, "\u0120partnered": 29806, "272": 29807, "irming": 29808, "\u0120BAS": 29809, "sometimes": 29810, "\u0120objected": 29811, "ubric": 29812, "posed": 29813, "LCS": 29814, "grass": 29815, "\u0120attributable": 29816, "VIS": 29817, "Israeli": 29818, "\u0120repeats": 29819, "\u0120RM": 29820, "vag": 29821, "uta": 29822, "inous": 29823, "\u0120inert": 29824, "\u0120Miguel": 29825, "\u00e6\u0143": 29826, "\u0120Hawaiian": 29827, "Board": 29828, "\u0120artific": 29829, "\u0120Azerbai": 29830, "asio": 29831, "\u0120Rent": 29832, "AIN": 29833, "\u0120appliances": 29834, "\u0120nationality": 29835, "\u0120asshole": 29836, "\u0120Neb": 29837, "\u0120notch": 29838, "hani": 29839, "\u0120Bride": 29840, "Availability": 29841, "\u0120intercepted": 29842, "\u0120continental": 29843, "\u0120swelling": 29844, "\u0120Perspect": 29845, "bies": 29846, ".<": 29847, "ithmetic": 29848, "\u0120Lara": 29849, "\u0120tempting": 29850, "addr": 29851, "\u0120overseeing": 29852, "clad": 29853, "\u0120DV": 29854, "\u0120Gingrich": 29855, "\u0120mun": 29856, "\u0120Appropri": 29857, "\u0120alterations": 29858, "\u0120Patreon": 29859, "\u0120havoc": 29860, "\u0120disciplines": 29861, "\u0120notoriously": 29862, "akuya": 29863, "ieri": 29864, "?).": 29865, "\u0120Went": 29866, "\u0120silicon": 29867, "\u0120tremb": 29868, "Container": 29869, "Known": 29870, "\u0120mortar": 29871, "este": 29872, "icka": 29873, "Arthur": 29874, "\u0120Previously": 29875, "\u0120Marty": 29876, "\u0120sparse": 29877, "gins": 29878, "\u0120inward": 29879, "\u0120Participant": 29880, "Copy": 29881, "\u0120Misc": 29882, "\u0120antibiotic": 29883, "\u0120Retro": 29884, "\u0120elusive": 29885, "\u0120assail": 29886, "\u0120Battalion": 29887, "\u0120Bought": 29888, "\u0120diminish": 29889, "\u0120Europa": 29890, "session": 29891, "\u0120Dangerous": 29892, "iesel": 29893, "\u0120disbelief": 29894, "\u0120blasts": 29895, "extreme": 29896, "\u0120Boyd": 29897, "\u0120Projects": 29898, "\u0120Guys": 29899, "\u0120undergone": 29900, "\u0120grill": 29901, "\u0120Dwight": 29902, "\u0120197": 29903, "USER": 29904, "\u0120filesystem": 29905, "\u0120clocks": 29906, "Taylor": 29907, "\u0120wrapper": 29908, "\u0120folding": 29909, "ousand": 29910, "\u0120Philippine": 29911, "ATIONAL": 29912, "\u0120Perth": 29913, "\u0120ashes": 29914, "\u0120accumulate": 29915, "\u0120Gateway": 29916, "Shop": 29917, "orkshire": 29918, "Han": 29919, "\u0120Barrel": 29920, "\u0120Leh": 29921, "\u0120XV": 29922, "\u0120whim": 29923, "\u0120repo": 29924, "\u0120CG": 29925, "\u0120Mam": 29926, "\u0120incorporating": 29927, "\u0120bailout": 29928, "\u0120linguistic": 29929, "\u0120disinteg": 29930, "CLE": 29931, "\u0120cinematic": 29932, "\u0120Fiber": 29933, "Syn": 29934, "ilion": 29935, "\u0120Compos": 29936, "chens": 29937, "\u0120neoc": 29938, "\u0120boiled": 29939, "FINE": 29940, "ono": 29941, "uncle": 29942, "iken": 29943, "\u0120BM": 29944, "\u00ce\u00b9": 29945, "\u0120receipts": 29946, "\u0120disposed": 29947, "\u0120Thirty": 29948, "\u0120Rough": 29949, "\u0120ABS": 29950, "\u0120notwithstanding": 29951, "ollen": 29952, "#$": 29953, "\u0120unreliable": 29954, "\u0120bloom": 29955, "\u0120mediocre": 29956, "\u0120tram": 29957, "\u0120Tasman": 29958, "\u0120shakes": 29959, "\u0120manifesto": 29960, "\u0120MW": 29961, "\u0120satisfactory": 29962, "\u0120shores": 29963, "\u0120computation": 29964, "\u0120assertions": 29965, "ormons": 29966, "arag": 29967, "abit": 29968, "Democrats": 29969, "\u0120Loot": 29970, "\u0120Volks": 29971, "haired": 29972, "\u0120gravitational": 29973, "Sing": 29974, "\u0120Miz": 29975, "\u0120throttle": 29976, "\u0120tyranny": 29977, "\u0120Views": 29978, "\u0120robber": 29979, "\u0120Minority": 29980, "\u0120shrine": 29981, "scope": 29982, "purpose": 29983, "\u0120nucleus": 29984, "ourcing": 29985, "\u0120USDA": 29986, "\u0120DHS": 29987, "wra": 29988, "\u0120Bowie": 29989, "Scale": 29990, "\u0120BEL": 29991, "xi": 29992, "Iter": 29993, "\u0120(),": 29994, "wright": 29995, "\u0120sailors": 29996, "oused": 29997, "NASA": 29998, "\u0120Proof": 29999, "\u0120Mineral": 30000, "token": 30001, "\u0120FD": 30002, "Rew": 30003, "\u0120ell": 30004, "630": 30005, "\u0120chancellor": 30006, "\u0120Gos": 30007, "\u0120amounted": 30008, "\u0120Recre": 30009, "omez": 30010, "\u0120Optim": 30011, "\u0120Olive": 30012, "\u0120tracker": 30013, "owler": 30014, "\u0120Unique": 30015, "Root": 30016, "\u0120maritime": 30017, "\u0120Quran": 30018, "\u0120Adapt": 30019, "\u0120ecosystems": 30020, "\u0120Repeat": 30021, "\u0120Soy": 30022, "\u0120IMP": 30023, "\u0120graduating": 30024, "andem": 30025, "Pur": 30026, "\u0120Reset": 30027, "\u0120Trick": 30028, "\u0120Philly": 30029, "\u0120Tue": 30030, "\u0120Malaysian": 30031, "\u0120climax": 30032, "\u0120bury": 30033, "\u0120conspic": 30034, "\u0120Southampton": 30035, "\u0120Flowers": 30036, "\u0120escorted": 30037, "\u0120Educational": 30038, "\u0120IRC": 30039, "\u0120brutally": 30040, "eating": 30041, "\u0120pillar": 30042, "\u0120Sang": 30043, "\u0120Jude": 30044, "arling": 30045, "\u0120Amnesty": 30046, "\u0120reminding": 30047, "\u0120Administrative": 30048, "hesda": 30049, "\u0120flashed": 30050, "\u0120PBS": 30051, "perate": 30052, "feature": 30053, "\u0120swipe": 30054, "\u0120graves": 30055, "oultry": 30056, "261": 30057, "breaks": 30058, "\u0120Guer": 30059, "\u0120shrimp": 30060, "\u0120Voting": 30061, "quist": 30062, "\u0120analytical": 30063, "\u0120tablespoons": 30064, "\u0120SOU": 30065, "\u0120researched": 30066, "\u0120disrupted": 30067, "\u0120jour": 30068, "\u0120replica": 30069, "\u0120cartoons": 30070, "bians": 30071, "})": 30072, "copy": 30073, "Got": 30074, "ouched": 30075, "PUT": 30076, "\u0120swarm": 30077, "notations": 30078, "said": 30079, "\u0120rebuilt": 30080, "\u0120collaborate": 30081, "\u0120raging": 30082, "\u0120nar": 30083, "\u0120demographics": 30084, "\u0120DDR": 30085, "\u0120distrust": 30086, "ossier": 30087, "\u0120Kro": 30088, "\u0120pumpkin": 30089, "\u0120regrets": 30090, "\u0120fatalities": 30091, "\u0120Lens": 30092, "\u0120Ole": 30093, "pd": 30094, "\u0120puppet": 30095, "\u0120Outlook": 30096, "\u0120Stam": 30097, "Ol": 30098, "Fair": 30099, "UU": 30100, "\u0120rewritten": 30101, "\u00c4\u00b1": 30102, "\u0120fascinated": 30103, "\u0120vectors": 30104, "\u0120tribunal": 30105, "uay": 30106, "\u0120Mats": 30107, "\u0120Coins": 30108, "[[": 30109, "\u0120181": 30110, "\u0120renders": 30111, "\u0120Kaepernick": 30112, "\u0120espionage": 30113, "\u0120summ": 30114, "\u0120ditch": 30115, "Account": 30116, "\u0120spreadsheet": 30117, "\u0120mutant": 30118, "past": 30119, "407": 30120, "\u0120dye": 30121, "\u0120initiation": 30122, "\u01204000": 30123, "\u0120punishable": 30124, "\u0120thinner": 30125, "\u0120Khal": 30126, "\u0120intermedi": 30127, "Dun": 30128, "\u0120Gotham": 30129, "\u0120eagerly": 30130, "\u0120vaginal": 30131, "powers": 30132, "VW": 30133, "\u0120WATCHED": 30134, "\u0120predator": 30135, "amsung": 30136, "\u0120disparity": 30137, "\u0120[*": 30138, "\u0120amph": 30139, "\u0120outskirts": 30140, "\u0120Spirits": 30141, "\u0120skeletal": 30142, "\u00d0\u00bb": 30143, "\u0120Rear": 30144, "\u0120issuance": 30145, "\u0120Logic": 30146, "released": 30147, "ZZ": 30148, "\u0120Bound": 30149, "Entry": 30150, "\u0120exits": 30151, "isol": 30152, "\u0120Founder": 30153, "\u0120wre": 30154, "\u0120Greenland": 30155, "\u0120MMO": 30156, "taker": 30157, "INC": 30158, "\u00e3\u0123\u00be": 30159, "\u0120hourly": 30160, "henko": 30161, "\u0120fantasies": 30162, "\u0120disob": 30163, "\u0120demolition": 30164, "\u00e3\u0125\u012d": 30165, "\u0120enlisted": 30166, "ratulations": 30167, "\u0120misguided": 30168, "\u0120ensured": 30169, "\u0120discouraged": 30170, "mort": 30171, "\u0120flank": 30172, "\u0120cess": 30173, "\u0120reacts": 30174, "\u0120Sere": 30175, "sensitive": 30176, "\u0120Serpent": 30177, "assad": 30178, "\u0120247": 30179, "\u0120calmly": 30180, "busters": 30181, "\u0120bleed": 30182, "\u0120Stro": 30183, "\u0120amusement": 30184, "\u0120Antarctica": 30185, "\u0120scept": 30186, "\u0120Gaw": 30187, "aq": 30188, "asonic": 30189, "\u0120sprawling": 30190, "native": 30191, "aturated": 30192, "\u0120Battlefield": 30193, "IVERS": 30194, "EB": 30195, "\u0120Gems": 30196, "\u0120Northwestern": 30197, "\u0120Films": 30198, "\u0120Automatic": 30199, "\u0120apprehend": 30200, "\u00e3\u0123\u00a8": 30201, "\u0120guiName": 30202, "\u0120backend": 30203, "\u0120evidenced": 30204, "geant": 30205, "012": 30206, "\u0120Siege": 30207, "\u0120externalTo": 30208, "\u0120unfocusedRange": 30209, "\u0120guiActiveUnfocused": 30210, "\u0120guiIcon": 30211, "\u0120externalToEVA": 30212, "\u0120externalToEVAOnly": 30213, "Fri": 30214, "chard": 30215, "enaries": 30216, "\u0120chiefs": 30217, "\u0120cf": 30218, "\u0120HUD": 30219, "\u0120corrobor": 30220, "\u0120dB": 30221, "\u0120Taken": 30222, "\u0120Patricia": 30223, "rail": 30224, "\u0120Charm": 30225, "\u0120Libertarian": 30226, "rieve": 30227, "Personal": 30228, "\u0120OUR": 30229, "geries": 30230, "\u0120dumping": 30231, "\u0120neurological": 30232, "itimate": 30233, "\u0120Clintons": 30234, "rafted": 30235, "\u0120Molly": 30236, "\u0120terminals": 30237, "register": 30238, "\u0120flare": 30239, "\u0120encoded": 30240, "\u0120autopsy": 30241, "pel": 30242, "machine": 30243, "\u0120exemptions": 30244, "\u0120Royals": 30245, "distance": 30246, "\u0120drafts": 30247, "\u0120lame": 30248, "\u0120Cunning": 30249, "\u0120spouses": 30250, "\u0120Markets": 30251, "\u0120Carrier": 30252, "\u0120implying": 30253, "\u0120Yak": 30254, "sid": 30255, "\u0120loser": 30256, "\u0120vigilant": 30257, "\u0120impeachment": 30258, "\u0120augmented": 30259, "\u0120Employees": 30260, "\u0120unintended": 30261, "ternally": 30262, "\u0120Watt": 30263, "\u0120recognizable": 30264, "essim": 30265, "\u00e6\u013f": 30266, "\u0120coated": 30267, "rha": 30268, "\u0120lieutenant": 30269, "\u0120Legislation": 30270, "published": 30271, "444": 30272, "013": 30273, "\u0120ideally": 30274, "\u0120Password": 30275, "\u0120simplify": 30276, "\u0120Meta": 30277, "\u0120MRI": 30278, "\u0120pleading": 30279, "organized": 30280, "handler": 30281, "\u0120unravel": 30282, "correct": 30283, "\u0120icy": 30284, "\u0120paranoid": 30285, "\u0120passer": 30286, "\u0120inspections": 30287, "ofer": 30288, "\u0120Healthcare": 30289, "283": 30290, "\u0120Brut": 30291, "iola": 30292, "forge": 30293, "\u0120Medieval": 30294, "MSN": 30295, "ievers": 30296, "\u0120Programming": 30297, "\u00e5\u012b": 30298, "\u0120223": 30299, "mu": 30300, "\u0120CLE": 30301, "uga": 30302, "\u0120shoppers": 30303, "\u0120informative": 30304, "\u0120Plans": 30305, "\u0120supplementation": 30306, "\u0120Tests": 30307, "tyard": 30308, "ocytes": 30309, "\u0120Vega": 30310, "\u0120Gujarat": 30311, "ermanent": 30312, "Except": 30313, "\u0120LOT": 30314, "alla": 30315, "\u0120Cumm": 30316, "\u0120Osw": 30317, "\u0120venom": 30318, "\u0120Debt": 30319, "\u0120DOWN": 30320, "\u0120reunion": 30321, "\u0120muc": 30322, "\u0120Relief": 30323, "\u0120geop": 30324, "\u0120\u00f0\u0141\u013a": 30325, "alogue": 30326, "Anth": 30327, "echo": 30328, "\u0120corros": 30329, "\u0120replication": 30330, "\u0120Blazing": 30331, "\u0120Daughter": 30332, "\u0120inflic": 30333, "\u0120Lindsey": 30334, "\u00d9\u012a": 30335, "284": 30336, "Exit": 30337, "\u0120gloom": 30338, "TAIN": 30339, "\u0120undermining": 30340, "\u0120advising": 30341, "hidden": 30342, "\u0120overflow": 30343, "\u0120gor": 30344, "urdue": 30345, "\u0120echoes": 30346, "enhagen": 30347, "\u0120impuls": 30348, "drug": 30349, "cash": 30350, "\u0120async": 30351, "\u0120mirac": 30352, "atts": 30353, "punk": 30354, "\u0120pivot": 30355, "\u0120Legislative": 30356, "\u0120bloggers": 30357, "\u0120Claw": 30358, "sburg": 30359, "dyl": 30360, "\u0120Recommend": 30361, "\u0120verte": 30362, "\u0120prohibiting": 30363, "\u0120Panther": 30364, "Jonathan": 30365, "\u0120omin": 30366, "\u0120hateful": 30367, "281": 30368, "\u0120Orche": 30369, "\u0120Murdoch": 30370, "downs": 30371, "\u0120asymm": 30372, "GER": 30373, "Always": 30374, "\u0120informs": 30375, "\u0120WM": 30376, "\u0120Pony": 30377, "\u0120Appendix": 30378, "\u0120Arlington": 30379, "Jam": 30380, "\u0120medicinal": 30381, "\u0120Slam": 30382, "ITIES": 30383, "\u0120reaff": 30384, "\u0120Ri": 30385, "FG": 30386, "Spring": 30387, "bool": 30388, "\u0120thighs": 30389, "\u0120markings": 30390, "\u0120Raqqa": 30391, "\u0120Lak": 30392, "poll": 30393, "tsky": 30394, "\u0120Morty": 30395, "\u0120Definition": 30396, "\u0120debunk": 30397, "endered": 30398, "\u0120Leone": 30399, "avers": 30400, "\u0120mortgages": 30401, "Apparently": 30402, "Nic": 30403, "haus": 30404, "\u0120Thousands": 30405, "auld": 30406, "\u0120mash": 30407, "shoot": 30408, "\u0120diarr": 30409, "\u0120consciously": 30410, "Hero": 30411, "eas": 30412, "\u0120Naturally": 30413, "\u0120Destroyer": 30414, "\u0120dashboard": 30415, "services": 30416, "Rog": 30417, "\u0120millennials": 30418, "\u0120invade": 30419, "-(": 30420, "\u0120commissions": 30421, "\u0120Auckland": 30422, "\u0120broadcasts": 30423, "\u0120frontal": 30424, "\u0120crank": 30425, "\u0120Historic": 30426, "\u0120rumours": 30427, "CTV": 30428, "\u0120steril": 30429, "\u0120booster": 30430, "rocket": 30431, "\u00e3\u0124\u00bc": 30432, "utsche": 30433, "\u0120PI": 30434, "\u0120233": 30435, "\u0120Producer": 30436, "\u0120Analytics": 30437, "\u0120invaluable": 30438, "\u0120unintention": 30439, "\u0120CY": 30440, "\u0120scrutin": 30441, "\u0120gigg": 30442, "\u0120engulf": 30443, "\u0120proletariat": 30444, "\u0120hacks": 30445, "\u0120Hew": 30446, "arak": 30447, "\u0120Slime": 30448, "ielding": 30449, "agher": 30450, "\u0120Elliot": 30451, "\u0120telecom": 30452, "\u0120219": 30453, "ultan": 30454, "\u0120Arbor": 30455, "\u0120Scouts": 30456, "Ban": 30457, "\u0120lifespan": 30458, "\u0120blasp": 30459, "388": 30460, "\u0120judiciary": 30461, "\u0120Continental": 30462, "asking": 30463, "McC": 30464, "LED": 30465, "\u0120baggage": 30466, "\u0120Sorcerer": 30467, "\u0120remnants": 30468, "\u0120Griffith": 30469, "etsu": 30470, "\u0120Subaru": 30471, "\u0120Personality": 30472, "designed": 30473, "ushima": 30474, "agnar": 30475, "\u0120recoil": 30476, "\u0120passions": 30477, "\\\":": 30478, "\u0120tee": 30479, "\u0120abolition": 30480, "\u0120Creating": 30481, "jac": 30482, "\u0120194": 30483, "019": 30484, "\u0120pillars": 30485, "riched": 30486, "/\"": 30487, "tk": 30488, "\u0120livelihood": 30489, "\u0120roasted": 30490, "ahon": 30491, "\u0120Hutch": 30492, "assert": 30493, "\u0120dividend": 30494, "\u0120knit": 30495, "\u0120daunting": 30496, "\u0120disturbance": 30497, "\u0120shale": 30498, "\u0120cultivated": 30499, "\u0120refrigerator": 30500, "LB": 30501, "\u0120NET": 30502, "\u0120commercials": 30503, "\u0120thinkers": 30504, "455": 30505, "\u0120chop": 30506, "Broad": 30507, "\u0120suspicions": 30508, "\u0120tagged": 30509, "lifting": 30510, "\u0120stylish": 30511, "\u0120Shields": 30512, "Shortly": 30513, "\u0120tails": 30514, "Auth": 30515, "STE": 30516, "\u0120GAME": 30517, "\u0120seism": 30518, "\u0120Kis": 30519, "ologne": 30520, "\u0120cowork": 30521, "\u0120forcibly": 30522, "\u0120thyroid": 30523, "\u0120PB": 30524, "ANE": 30525, "married": 30526, "horse": 30527, "\u0120polymer": 30528, "\u0120Chal": 30529, "odor": 30530, "DEBUG": 30531, "\u0120Context": 30532, "\u0120bliss": 30533, "\u0120pinpoint": 30534, "\u0120Mathemat": 30535, "legram": 30536, "\u0120Weekend": 30537, "\u0120labelled": 30538, "\u0120bart": 30539, "itles": 30540, "\u0120estrogen": 30541, "\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136\u00e2\u0122\u0136": 30542, "\"'": 30543, "\u0120visibly": 30544, "\u0120outsider": 30545, "aida": 30546, "Area": 30547, "\u0120dissemin": 30548, "\u0120dishonest": 30549, "\u0120Closed": 30550, "\u0120Bulletin": 30551, "\u0120Ramsey": 30552, "sword": 30553, "\u0120XI": 30554, "ourced": 30555, "Same": 30556, "346": 30557, "\u0120Repe": 30558, "\u0120Kou": 30559, "cake": 30560, "emis": 30561, "Cache": 30562, "\u0120Meaning": 30563, "\u0120Enlight": 30564, "onomy": 30565, "\u0120manifestation": 30566, "sworth": 30567, "Jay": 30568, "\u0120chore": 30569, "\u00c3\u00b6r": 30570, "Dream": 30571, "\u0120sanctioned": 30572, "\u0120culturally": 30573, "\u0120Ara": 30574, "Nav": 30575, "\u0120theological": 30576, "\u0120strut": 30577, "\u0120VO": 30578, "\u0120Handbook": 30579, "\u0120constructing": 30580, "\u0120\u00c2\u00b6": 30581, "\u0120Benefits": 30582, "\u0120Psychological": 30583, "sac": 30584, "\u00e5\u00b8": 30585, "policy": 30586, "\u0120Matters": 30587, "\u0120Reported": 30588, "\u0120Byte": 30589, "\u0120vitro": 30590, "\u0120Maiden": 30591, "\u0120lam": 30592, "\u0120Jennings": 30593, "\u0120garment": 30594, "\u0120Rutgers": 30595, "\u0120Stafford": 30596, "\u0120Wellington": 30597, "\u0120intermitt": 30598, "\u0120npm": 30599, "\u0120ordeal": 30600, "\u0120plugged": 30601, "ooming": 30602, "inished": 30603, "framework": 30604, "\u0120timber": 30605, "\u0120cass": 30606, "\u0120850": 30607, "iless": 30608, "\u0120Redux": 30609, "768": 30610, "Stre": 30611, "\u0120surpassed": 30612, "whel": 30613, "\u0120parallels": 30614, "\u0120veil": 30615, "\u0120GI": 30616, "\u0120REST": 30617, "\u0120readiness": 30618, "sort": 30619, "\u0120modifying": 30620, "\u0120Slate": 30621, "ruff": 30622, "\u0120marble": 30623, "\u0120infrared": 30624, "\u0120auditor": 30625, "\u0120FANTASY": 30626, "\u0120Poverty": 30627, "\u0120SPD": 30628, "\u0120\"(": 30629, "Ky": 30630, "RAY": 30631, "\u0120executions": 30632, "\u0120Beverly": 30633, "\u0120Marxism": 30634, "\u0120Burst": 30635, "\u0120Kali": 30636, "estones": 30637, "Clearly": 30638, "Ell": 30639, "\u00e3\u0123\u00a7": 30640, "\u0120Proceedings": 30641, "Token": 30642, "IFIC": 30643, "\u00c3\u00b1a": 30644, "Central": 30645, "\u0120Haley": 30646, "\u0120Drama": 30647, "\u0120formations": 30648, "ORN": 30649, "Books": 30650, "\u0120dominating": 30651, "\u0120Flyers": 30652, "\u0120Companion": 30653, "\u0120disciplined": 30654, "\u0120Yugoslav": 30655, "\u0120Spells": 30656, "\u0120vengeance": 30657, "\u0120landlords": 30658, "Len": 30659, "\u0120Ogre": 30660, "anoia": 30661, "\u0120piercing": 30662, "\u0120congreg": 30663, "\u0120scorer": 30664, "obia": 30665, "\u0120nickel": 30666, "\u0120Learns": 30667, "\u0120rejo": 30668, "\u0120masterpiece": 30669, "Flash": 30670, "\u0120inhabited": 30671, "\u0120OpenGL": 30672, "\u0120Dud": 30673, "\u0120ICO": 30674, "\u0120arter": 30675, "\u0120plur": 30676, "\u0120mastery": 30677, "\u0120longstanding": 30678, "sted": 30679, "\u0120wines": 30680, "\u0120televised": 30681, "\u0120Shrine": 30682, "\u0120Bayern": 30683, "\u0120\u00e2\u0135\u013a": 30684, "\u0120enclosure": 30685, "john": 30686, "\u0120prophets": 30687, "\u0120Resurrection": 30688, "\u0120Orders": 30689, "\u0120uneven": 30690, "rals": 30691, "\u0120dwind": 30692, "\u0120Lah": 30693, "\u0120Sloven": 30694, "378": 30695, "\u0120insistence": 30696, "affle": 30697, "\u0120Clone": 30698, "\u0120hardship": 30699, "\u0120Congressman": 30700, "\u0120plead": 30701, "\u0120reviewers": 30702, "\u0120cured": 30703, "\u01201935": 30704, "asley": 30705, "fake": 30706, "\u0120Thinking": 30707, "ydia": 30708, "PART": 30709, "\u0120Dota": 30710, "oit": 30711, "\u0120whipped": 30712, "\u0120bouncing": 30713, "\u0120Hispanics": 30714, "comings": 30715, "\u0120cannabin": 30716, "\u0120Chambers": 30717, "\u0120Zack": 30718, "Optional": 30719, "\u0120coats": 30720, "\u0120prowess": 30721, "\u0120Norton": 30722, "\u0120plainly": 30723, "\u0120freight": 30724, "\u0120inhibition": 30725, "\u0120clam": 30726, "\u0120303": 30727, "kef": 30728, "aleigh": 30729, "Luke": 30730, "\u0120psycho": 30731, "atorium": 30732, "MED": 30733, "\u0120treaties": 30734, "\u0120indisc": 30735, "\u0120dc": 30736, "OPS": 30737, "\u0120resilient": 30738, "\u0120Interstate": 30739, "\u0120slack": 30740, "\u0120mundane": 30741, "\u0120establishes": 30742, "359": 30743, "\u0120strained": 30744, "\u0120nond": 30745, "Sus": 30746, "\u0120caste": 30747, "arate": 30748, "ieving": 30749, "\u0120unfairly": 30750, "\u0120parser": 30751, "onial": 30752, "ursive": 30753, "Via": 30754, "\u0120Otto": 30755, "\u0120Authorities": 30756, "stroke": 30757, "KR": 30758, "\u0120Mercy": 30759, "\u0120furnished": 30760, "\u0120outset": 30761, "\u0120metic": 30762, "1982": 30763, "olithic": 30764, "\u0120Tent": 30765, "ogical": 30766, "\u0120Aircraft": 30767, "\u0120hides": 30768, "\u0120Became": 30769, "\u0120educators": 30770, "reaching": 30771, "\u0120volatility": 30772, "\u0120toddler": 30773, "\u0120NASCAR": 30774, "\u0120Twelve": 30775, "\u0120Highlights": 30776, "\u0120grape": 30777, "\u0120splits": 30778, "\u0120peasant": 30779, "\u0120reneg": 30780, "\u0120MSI": 30781, "Temp": 30782, "stars": 30783, "\u0120trek": 30784, "\u0120Hyde": 30785, "binding": 30786, "\u0120realism": 30787, "\u0120oxide": 30788, "\u0120Hos": 30789, "\u0120mounts": 30790, "\u0120biting": 30791, "\u0120collapsing": 30792, "\u0120postal": 30793, "\u0120museums": 30794, "\u0120detached": 30795, "\u0120respecting": 30796, "\u0120monopol": 30797, "\u0120workflow": 30798, "\u0120Cake": 30799, "Template": 30800, "\u0120Organisation": 30801, "\u0120persistence": 30802, "369": 30803, "Coming": 30804, "Brad": 30805, "\u0120redundant": 30806, "\u0120GTA": 30807, "\u0120bending": 30808, "\u0120revoked": 30809, "\u0120offending": 30810, "\u0120framing": 30811, "\u0120printf": 30812, "Commun": 30813, "members": 30814, "Outside": 30815, "\u0120construed": 30816, "\u0120coded": 30817, "FORE": 30818, "\u0120chast": 30819, "Chat": 30820, "Indian": 30821, "\u0120Yard": 30822, "?!\"": 30823, "\u0120Ports": 30824, "\u0120Xavier": 30825, "\u0120RET": 30826, "'.\"": 30827, "\u0120Boat": 30828, "ivated": 30829, "icht": 30830, "umerable": 30831, "Ds": 30832, "\u0120Dunn": 30833, "\u0120coffin": 30834, "\u0120securely": 30835, "\u0120Raptors": 30836, "\u0120Bes": 30837, "Installation": 30838, "\u0120inception": 30839, "\u0120Healthy": 30840, "endants": 30841, "\u0120psychologists": 30842, "\u0120Sheikh": 30843, "cultural": 30844, "\u0120BlackBerry": 30845, "shift": 30846, "Fred": 30847, "oche": 30848, "\u0120cakes": 30849, "\u0120SEO": 30850, "\u0120Gian": 30851, "\u0120Asians": 30852, "ogging": 30853, "element": 30854, "\u0120pundits": 30855, "\u0120Vaugh": 30856, "\u0120Gavin": 30857, "\u0120hitter": 30858, "\u0120drowned": 30859, "\u0120chalk": 30860, "\u0120Zika": 30861, "\u0120measles": 30862, "802": 30863, "\u00e2\u0122\u00a6..": 30864, "\u0120AWS": 30865, "]\"": 30866, "\u0120distort": 30867, "\u0120Mast": 30868, "\u0120antibodies": 30869, "\u0120Mash": 30870, "Memory": 30871, "\u0120Uganda": 30872, "\u0120Prob": 30873, "\u0120vomiting": 30874, "\u0120Turns": 30875, "\u0120occupying": 30876, "\u0120evasion": 30877, "\u0120Therapy": 30878, "\u0120promo": 30879, "\u0120electr": 30880, "\u0120blueprint": 30881, "\u0120Dre": 30882, "priced": 30883, "\u0120Depot": 30884, "\u0120alleviate": 30885, "\u0120Somali": 30886, "marg": 30887, "nine": 30888, "\u0120nostalgia": 30889, "\u0120Shepherd": 30890, "\u0120cavalry": 30891, "\u0120torped": 30892, "\u0120Bloody": 30893, "xb": 30894, "\u0120sank": 30895, "\u0120goalt": 30896, "reportprint": 30897, "embedreportprint": 30898, "cloneembedreportprint": 30899, "\u0120Initially": 30900, "\u0120Fischer": 30901, "\u0120noteworthy": 30902, "cern": 30903, "\u0120inefficient": 30904, "rawdownload": 30905, "rawdownloadcloneembedreportprint": 30906, "cation": 30907, "\u0120Dynasty": 30908, "lag": 30909, "DES": 30910, "\u0120distinctly": 30911, "\u0120Estonia": 30912, "\u0120openness": 30913, "\u0120gossip": 30914, "ruck": 30915, "Width": 30916, "\u0120Ibrahim": 30917, "\u0120petroleum": 30918, "\u0120avatar": 30919, "\u0120Hed": 30920, "atha": 30921, "\u0120Hogwarts": 30922, "\u0120caves": 30923, "678": 30924, "\u0120safeguard": 30925, "\u0120Mog": 30926, "isson": 30927, "\u0120Durham": 30928, "slaught": 30929, "\u0120Graduate": 30930, "\u0120subconscious": 30931, "\u0120Excellent": 30932, "\u0120Dum": 30933, "-----": 30934, "\u0120piles": 30935, "\u0120WORK": 30936, "\u0120Garn": 30937, "\u0120Fol": 30938, "\u0120ATM": 30939, "\u0120avoids": 30940, "\u0120Tul": 30941, "\u0120bleak": 30942, "ELY": 30943, "ivist": 30944, "lightly": 30945, "Pers": 30946, "\u0120Dob": 30947, "\u0120LS": 30948, "\u0120insanity": 30949, "\u00ce\u00b5": 30950, "atalie": 30951, "Enlarge": 30952, "\u0120twists": 30953, "\u0120faulty": 30954, "\u0120piracy": 30955, "\u0120impover": 30956, "\u0120rugged": 30957, "\u0120Fashion": 30958, "\u0120sands": 30959, "'?": 30960, "swick": 30961, "\u0120natives": 30962, "\u0120hen": 30963, "\u0120Noise": 30964, "\u00e3\u0125\u0139": 30965, "\u0120greens": 30966, "\u0120freezer": 30967, "\u0120dynasty": 30968, "\u0120Fathers": 30969, "\u0120Newark": 30970, "\u0120archaeological": 30971, "\u0120ot": 30972, "obar": 30973, "\u0120blockade": 30974, "\u0120allerg": 30975, "LV": 30976, "\u0120debit": 30977, "\u0120RFC": 30978, "\u0120Milton": 30979, "\u0120Pressure": 30980, "\u0120willingly": 30981, "\u0120disproportionate": 30982, "\u0120oppressive": 30983, "\u0120diamonds": 30984, "\u0120belongings": 30985, "1970": 30986, "\u0120bells": 30987, "\u0120imperialism": 30988, "\u0120227": 30989, "\u0120exploding": 30990, "\u0120Eclipse": 30991, "\u01201919": 30992, "\u0120rant": 30993, "\u0120nominations": 30994, "347": 30995, "\u0120peacefully": 30996, "rica": 30997, "\u0120FUCK": 30998, "\u0120vibration": 30999, "malink": 31000, "\u0120ropes": 31001, "\u0120Ivanka": 31002, "\u0120Brewery": 31003, "\u0120Booker": 31004, "\u0120Owens": 31005, "goers": 31006, "Services": 31007, "\u0120Snape": 31008, "\u0120191": 31009, "395": 31010, "\u0120299": 31011, "justice": 31012, "\u0120bri": 31013, "\u0120discs": 31014, "\u0120prominently": 31015, "\u0120vulgar": 31016, "\u0120skipping": 31017, "lves": 31018, "\u0120tsunami": 31019, "374": 31020, "\u0120Urug": 31021, "\u0120Eid": 31022, "recated": 31023, "phen": 31024, "\u0120faults": 31025, "\u0120Started": 31026, "950": 31027, "\u0120pi": 31028, "\u0120detector": 31029, "\u0120bastard": 31030, "\u0120validated": 31031, "SpaceEngineers": 31032, "OURCE": 31033, "\u0120(~": 31034, "\u0120unsur": 31035, "\u0120affirmed": 31036, "\u0120fascism": 31037, "\u0120resolving": 31038, "\u0120Chavez": 31039, "\u0120Cyn": 31040, "\u0120detract": 31041, "Lost": 31042, "\u0120rigged": 31043, "\u0120homage": 31044, "\u0120Bruno": 31045, "555": 31046, "eca": 31047, "\u0120presses": 31048, "\u0120humour": 31049, "\u0120spacing": 31050, "\u0120'/": 31051, "olkien": 31052, "Coun": 31053, "OPER": 31054, "Tre": 31055, "Son": 31056, "\u0120Cambodia": 31057, "ierre": 31058, "mong": 31059, "ozy": 31060, "\u0120liquidity": 31061, "\u0120Soviets": 31062, "\u0120Fernando": 31063, "\u0120229": 31064, "\u0120slug": 31065, "\u0120Catalan": 31066, "electric": 31067, "\u0120scenery": 31068, "\u0120Hearth": 31069, "\u0120constrained": 31070, "\u0120goalie": 31071, "\u0120Guidelines": 31072, "\u0120Ammo": 31073, "\u0120Pearson": 31074, "\u0120taxed": 31075, "\u0120fetus": 31076, "Response": 31077, "\u0120Alexis": 31078, "thia": 31079, "Guy": 31080, "\u0120reconstruct": 31081, "\u0120extremes": 31082, "\u0120concluding": 31083, "\u0120Peg": 31084, "ooks": 31085, "\u0120deductions": 31086, "Rose": 31087, "\u0120groundbreaking": 31088, "\u0120Targ": 31089, "\u00e3\u0125\u0123": 31090, "\u0120Reve": 31091, "resource": 31092, "\u0120moons": 31093, "\u0120electromagnetic": 31094, "\u0120amidst": 31095, "\u0120Viktor": 31096, "NESS": 31097, "BACK": 31098, "\u0120commute": 31099, "\u0120Anaheim": 31100, "\u0120fluctuations": 31101, "640": 31102, "\u0120noodles": 31103, "\u0120Copenhagen": 31104, "\u0120Tide": 31105, "\u0120Grizz": 31106, "\u0120SEE": 31107, "\u0120pipelines": 31108, "\u0120scars": 31109, "endo": 31110, "agus": 31111, "\u0120ETF": 31112, "/#": 31113, "\u0120Become": 31114, "448": 31115, "\u0120visc": 31116, "\u0120Recommended": 31117, "\u0120jumper": 31118, "\u0120cognition": 31119, "\u0120assassin": 31120, "\u0120witnessing": 31121, "\u0120Setup": 31122, "\u0120lac": 31123, "vim": 31124, "ISM": 31125, "pages": 31126, "SSL": 31127, "358": 31128, "\u0120adject": 31129, "industrial": 31130, "lore": 31131, "chery": 31132, "\u0120glitter": 31133, "\u0120calf": 31134, "Florida": 31135, "\u0120spoilers": 31136, "\u0120succeeds": 31137, "\u0120chanting": 31138, "\u0120slogans": 31139, "\u0120Tracy": 31140, "Visit": 31141, "rology": 31142, "\u0120mornings": 31143, "\u0120lineage": 31144, "\u0120sip": 31145, "\u0120intensely": 31146, "\u0120flourish": 31147, "\u0120Sleeping": 31148, "\u0120Fem": 31149, "orpor": 31150, "\u0120Klan": 31151, "\u0120Darth": 31152, "hack": 31153, "\u0120Nielsen": 31154, "\u0120tumors": 31155, "\u0120procurement": 31156, "\u0120Yorkshire": 31157, "\u0120raided": 31158, "KY": 31159, "Anna": 31160, "\u0120//[": 31161, "\u0120Disorder": 31162, "\u0120Mustang": 31163, "\u0120Wen": 31164, "\u0120Trying": 31165, "sq": 31166, "\u0120deliveries": 31167, "\u0120shutter": 31168, "\u0120cerebral": 31169, "\u0120bipolar": 31170, "\u0120CN": 31171, "lass": 31172, "jet": 31173, "\u0120debating": 31174, ">:": 31175, "\u0120eagle": 31176, "grades": 31177, "\u0120Dixon": 31178, "UGC": 31179, "MAS": 31180, "\u0120Draco": 31181, "\u0120Machines": 31182, "affer": 31183, "\u0120eman": 31184, "\u00c2\u00b2": 31185, "pron": 31186, "\u0120Gym": 31187, "\u0120comparatively": 31188, "\u0120Tribunal": 31189, "PRO": 31190, "\u0120lex": 31191, "\u0120fertile": 31192, "\u0120depressing": 31193, "\u0120superficial": 31194, "essential": 31195, "\u0120Hunters": 31196, "gp": 31197, "\u0120prominence": 31198, "Liber": 31199, "\u0120Ancest": 31200, "otechnology": 31201, "\u0120mocking": 31202, "\u0120Traff": 31203, "\u0138\u013c": 31204, "Medium": 31205, "Iraq": 31206, "\u0120psychiatrist": 31207, "Quantity": 31208, "\u0120Lect": 31209, "\u0120noisy": 31210, "520": 31211, "GY": 31212, "\u0120slapped": 31213, "\u0120MTV": 31214, "\u0120para": 31215, "pull": 31216, "Multiple": 31217, "asher": 31218, "\u0120nour": 31219, "\u0120Seg": 31220, "Spell": 31221, "vous": 31222, "ordial": 31223, "Senior": 31224, "\u0120Goldberg": 31225, "\u0120Plasma": 31226, "need": 31227, "\u0120messenger": 31228, "eret": 31229, "\u0120teamed": 31230, "\u0120literacy": 31231, "\u0120Leah": 31232, "\u0120Doyle": 31233, "\u0120emitted": 31234, "UX": 31235, "\u0120evade": 31236, "\u0120maze": 31237, "\u0120wrongly": 31238, "\u0120Lars": 31239, "\u0120stereotype": 31240, "\u0120pledges": 31241, "\u0120aroma": 31242, "\u0120MET": 31243, "\u0120acre": 31244, "\u0120OD": 31245, "\u0120ff": 31246, "\u0120breweries": 31247, "\u0120Hilton": 31248, "undle": 31249, "\u0120Kak": 31250, "\u0120Thankfully": 31251, "\u0120Canucks": 31252, "inctions": 31253, "\u0120Appears": 31254, "\u0120coer": 31255, "\u0120undermined": 31256, "rovers": 31257, "Andre": 31258, "\u0120blaze": 31259, "umers": 31260, "\u0120famine": 31261, "amphetamine": 31262, "ulkan": 31263, "Amount": 31264, "\u0120desperation": 31265, "wikipedia": 31266, "development": 31267, "\u0120Corinth": 31268, "ussia": 31269, "Jackson": 31270, "LI": 31271, "Native": 31272, "Rs": 31273, "Ohio": 31274, "\u0120Kathleen": 31275, "Fortunately": 31276, "\u0120attendant": 31277, "\u0120Preferred": 31278, "\u0120Didn": 31279, "\u0120Vs": 31280, "Mis": 31281, "\u0120respondent": 31282, "\u0120boun": 31283, "stable": 31284, "\u0120paved": 31285, "\u0120unexpl": 31286, "\u0120Cheney": 31287, "LM": 31288, "\u0120Cull": 31289, "blown": 31290, "\u0120confronting": 31291, "ocese": 31292, "serving": 31293, "Wi": 31294, "\u0120Lithuania": 31295, "anni": 31296, "\u0120stalk": 31297, "hd": 31298, "\u0120vener": 31299, "APH": 31300, "ynchronous": 31301, "URR": 31302, "umably": 31303, "historic": 31304, "Half": 31305, "Hay": 31306, "\u0120resilience": 31307, "spection": 31308, "\u0120abandoning": 31309, "Obs": 31310, "\u0120Debbie": 31311, "\u0120gradient": 31312, "\u0120Plaint": 31313, "\u0120Canal": 31314, "ARCH": 31315, "\u0120expansive": 31316, "\u0120fung": 31317, "\u0120bounced": 31318, "Und": 31319, "\u0120precautions": 31320, "\u0120clarification": 31321, "\u0120dagger": 31322, "\u0120grips": 31323, "\u0120\u00c2\u00b5": 31324, "\u0120Rivera": 31325, "\u0120Undead": 31326, "isites": 31327, "\u0120FIRST": 31328, "\u00c3\u00b1o": 31329, "audi": 31330, "\u0120hostages": 31331, "\u0120compliant": 31332, "\u0120alumni": 31333, "Seven": 31334, "\u0120cybersecurity": 31335, "either": 31336, "Collect": 31337, "\u0120invariably": 31338, "\u0120Soci": 31339, "\u0120lawmaker": 31340, "\u0120ale": 31341, "\u0120Personally": 31342, "Nazi": 31343, "\u0120customization": 31344, "\u0120Proc": 31345, "\u0120Saskatchewan": 31346, "eaturing": 31347, "\u0120spared": 31348, "\u0120discontinued": 31349, "\u0120computational": 31350, "\u0120Motorola": 31351, "\u0120supremacist": 31352, "governmental": 31353, "\u0120paradise": 31354, "\u0120Downing": 31355, "\u0120Nikon": 31356, "\u0120catalyst": 31357, "berra": 31358, "Toronto": 31359, "875": 31360, "beta": 31361, "\u0120Macron": 31362, "\u0120unrealistic": 31363, "vector": 31364, "\u0120Vehicles": 31365, "itiveness": 31366, "\u0120RV": 31367, "\u0120Colbert": 31368, "sin": 31369, "oji": 31370, "entin": 31371, "\u0120Krish": 31372, "hello": 31373, "ffield": 31374, "oky": 31375, "\u0120Tate": 31376, "\u0120maple": 31377, "\u0120aids": 31378, "chemical": 31379, "334": 31380, "nuts": 31381, "\u0120Warp": 31382, "\u0120xx": 31383, "\u0120Robb": 31384, "umerous": 31385, "_-_": 31386, "ftime": 31387, "\u0120VW": 31388, "\u0120winger": 31389, "\u0120Dome": 31390, "tools": 31391, "\u0120PV": 31392, "\u0120Georgetown": 31393, "\u0120geared": 31394, "\u0120jihadists": 31395, "\u0120cp": 31396, "\u0120steroids": 31397, "Mother": 31398, "clerosis": 31399, "\u0120DRM": 31400, "nesia": 31401, "\u0120linger": 31402, "\u0120immersive": 31403, "\u0120COUN": 31404, "\u0120outweigh": 31405, "ensual": 31406, "Band": 31407, "\u0120transforms": 31408, "matched": 31409, "psons": 31410, "\u0120Judicial": 31411, "factor": 31412, "\u0120referral": 31413, "\u0120oddly": 31414, "\u0120Wenger": 31415, "Bring": 31416, "\u0120Bows": 31417, "602": 31418, "ICLE": 31419, "\u0120lions": 31420, "\u0120Academic": 31421, "\u0120Thorn": 31422, "\u0120Raider": 31423, "kefeller": 31424, "Storage": 31425, "Lower": 31426, "\u0120Ort": 31427, "\u0120Equality": 31428, "ALT": 31429, "\u0120SOC": 31430, "Types": 31431, "\u0120lyn": 31432, "\u0120Asset": 31433, "coat": 31434, "TPP": 31435, "CVE": 31436, "\u0120Pioneer": 31437, "application": 31438, "Modern": 31439, "\u0120HK": 31440, "Environment": 31441, "Alright": 31442, "Rain": 31443, "IPP": 31444, "\u0120Shiite": 31445, "\u0120mound": 31446, "\u0120Abilities": 31447, "condition": 31448, "Staff": 31449, "\u0120competence": 31450, "\u0120Moor": 31451, "\u0120Diablo": 31452, "\u0120withheld": 31453, "\u0120ostensibly": 31454, "\u0120Brom": 31455, "\u0120msg": 31456, "\u0120denomin": 31457, "\u0120References": 31458, "\u0120FP": 31459, "\u0120plunged": 31460, "\u0120pamph": 31461, "moving": 31462, "central": 31463, "\u0120downright": 31464, "\u0120fading": 31465, "Tal": 31466, "Typ": 31467, "\u0120Thy": 31468, "ukes": 31469, "ithe": 31470, "\u0120ove": 31471, "\u0120battled": 31472, "\u0120seafood": 31473, "\u0120figur": 31474, "\u0120RD": 31475, "crop": 31476, "\u0120squads": 31477, "{\\": 31478, "\u00e0\u00b9": 31479, "\u0120Eh": 31480, "\u0120interviewing": 31481, "\u0120Qin": 31482, "\u0120aspiring": 31483, "PLIC": 31484, "\u0120clauses": 31485, "\u0120Gast": 31486, "\u0120Nir": 31487, "\u0120luggage": 31488, "\u0120hose": 31489, "\u0120systemd": 31490, "\u0120descending": 31491, "\u0120Revised": 31492, "\u0120Rails": 31493, "align": 31494, "709": 31495, "337": 31496, "\u0120fug": 31497, "charging": 31498, "tags": 31499, "\u0120uter": 31500, "kish": 31501, "WARNING": 31502, "490": 31503, "profits": 31504, "\u0120voyage": 31505, "\u0120ace": 31506, "\u0120Vanguard": 31507, "\u0120Tanks": 31508, "\u0120Muk": 31509, "\u0120226": 31510, "Safe": 31511, "Armor": 31512, "\u0120volcanic": 31513, "\u0120womb": 31514, "\u0120MIL": 31515, "\u0120beginner": 31516, "\u0120Recogn": 31517, "\u0120AAP": 31518, "PLAY": 31519, ")!": 31520, "\u0120detecting": 31521, "cn": 31522, "\u0120breaches": 31523, "Basically": 31524, "\u0120Pag": 31525, "\u0120Municipal": 31526, "\u0120Indie": 31527, "\u0120Laf": 31528, "\u0120Disable": 31529, "\u0120Olson": 31530, "\u0120restrained": 31531, "\u0120rulings": 31532, "\u0120humane": 31533, "events": 31534, "\u0120Cinema": 31535, "displayText": 31536, "\u0120Hatch": 31537, "actionDate": 31538, "onnaissance": 31539, "\u0120assaulting": 31540, "\u0120Lug": 31541, "CHAT": 31542, "\u0120vigorous": 31543, "\u0120Perse": 31544, "\u0120intolerance": 31545, "\u0120Snapchat": 31546, "\u0120Sharks": 31547, "\u0120dummy": 31548, "\u0120Diagn": 31549, "\u0120Guitar": 31550, "imeters": 31551, "403": 31552, "REG": 31553, "Ax": 31554, "\u0120separates": 31555, "\u0120Mahm": 31556, "\u0120tv": 31557, "jah": 31558, "OOL": 31559, "Circ": 31560, "\u0120Windsor": 31561, "ussian": 31562, "\u0120intuition": 31563, "\u0120disdain": 31564, "\u0120Donovan": 31565, "\u0120221": 31566, "Emb": 31567, "\u0120condemning": 31568, "\u0120generosity": 31569, "zzy": 31570, "\u0120panties": 31571, "\u0120Prevent": 31572, "ActionCode": 31573, "ANA": 31574, "342": 31575, "externalActionCode": 31576, "\u0120specifying": 31577, "\u0120crystall": 31578, "Jere": 31579, "\u0120rupt": 31580, "\u0120Apprentice": 31581, "\u0120profiling": 31582, "\u00d0\u00ba": 31583, "Strike": 31584, "\u0120sideline": 31585, "\u0120obligated": 31586, "\u0120occult": 31587, "\u0120bureaucratic": 31588, "antically": 31589, "rupted": 31590, "negative": 31591, "\u0120Ethiopia": 31592, "\u0120Civic": 31593, "\u0120insiders": 31594, "eligible": 31595, "\u0120TVs": 31596, "\u0120BAR": 31597, "\u0120TI": 31598, "iologist": 31599, "\u0120AIR": 31600, "\u0120substituted": 31601, "Arab": 31602, "\u0120Saul": 31603, "\u0120Yog": 31604, "prem": 31605, "\u0120builders": 31606, "\u0120stationary": 31607, "\u0120doubtful": 31608, "\u0120vigorously": 31609, "\u0120thrilling": 31610, "Physical": 31611, "\u0120Carey": 31612, "\u0120Hydra": 31613, "geoning": 31614, "\u0120Sly": 31615, "yton": 31616, "\u0120borrowers": 31617, "\u0120Parkinson": 31618, "\u0120\u00eb": 31619, "\u0120Jamaica": 31620, "\u0120satir": 31621, "\u0120insurgents": 31622, "\u0120Firm": 31623, "\u0120isot": 31624, "\u0120Karn": 31625, "ourning": 31626, "akens": 31627, "docs": 31628, "little": 31629, "\u0120Monaco": 31630, "CLASS": 31631, "Turkey": 31632, "Ly": 31633, "\u0120Conan": 31634, "assic": 31635, "\u0120starred": 31636, "\u0120Pacers": 31637, "eties": 31638, "\u0120tipping": 31639, "Moon": 31640, "\u0120Rw": 31641, "same": 31642, "\u0120cavity": 31643, "\u0120goof": 31644, "\u0120Zo": 31645, "Shock": 31646, "ummer": 31647, "\u0120emphasizes": 31648, "\u0120regrett": 31649, "\u0120novelty": 31650, "\u0120envy": 31651, "\u0120Passive": 31652, "rw": 31653, "505": 31654, "\u0120indifferent": 31655, "\u0120Rica": 31656, "\u0120Himself": 31657, "\u0120Freddie": 31658, "\u0120adip": 31659, "\u00e4\u00b8\u0122": 31660, "\u0120breakout": 31661, "\u0120hurried": 31662, "\u0120Huang": 31663, "\u0120Disk": 31664, "\u0120roaming": 31665, "?????-?????-": 31666, "UV": 31667, "\u0120Ricky": 31668, "\u0120Sigma": 31669, "\u0120marginalized": 31670, "\u0120edits": 31671, "\u0120304": 31672, "memory": 31673, "\u0120specimen": 31674, "293": 31675, "\u00e3\u0123\u00af": 31676, "\u0120vertically": 31677, "\u0120audition": 31678, "\u0120Heck": 31679, "\u0120caster": 31680, "\u0120Holdings": 31681, "adal": 31682, "\u0120Cron": 31683, "\u0120Liam": 31684, "\u0120deflect": 31685, "Pick": 31686, "\u0120Debug": 31687, "REF": 31688, "\u0120versatility": 31689, "othes": 31690, "classified": 31691, "\u0120Mahar": 31692, "\u0120Hort": 31693, "Counter": 31694, "stasy": 31695, "noticed": 31696, "331": 31697, "\u0120Shim": 31698, "fuck": 31699, "\u0120Bie": 31700, "\u0120airing": 31701, "\u0120Protein": 31702, "\u0120Holding": 31703, "\u0120spectators": 31704, "iliated": 31705, "\u0120Thatcher": 31706, "nosis": 31707, "\u00e3\u0125\u00bc\u00e3\u0125\u00b3": 31708, "Tele": 31709, "Boston": 31710, "\u0120Templ": 31711, "stay": 31712, "\u0120declarations": 31713, "479": 31714, "Volume": 31715, "\u0120Designer": 31716, "\u0120Overwatch": 31717, "idae": 31718, "\u0120onwards": 31719, "\u0120nets": 31720, "\u0120Manila": 31721, "particularly": 31722, "\u0120politic": 31723, "oother": 31724, "\u0120portraits": 31725, "\u0120pavement": 31726, "cffff": 31727, "\u0120saints": 31728, "\u0120beginners": 31729, "ESPN": 31730, "\u0120shortcomings": 31731, "\u00e2\u0137\u0132\u00e2\u0137\u0132": 31732, "\u0120comet": 31733, "\u0120Organic": 31734, "quel": 31735, "\u0120hospitalized": 31736, "Break": 31737, "\u0120peel": 31738, "dylib": 31739, "aspx": 31740, "urances": 31741, "\u0120TIM": 31742, "Pg": 31743, "\u0120readable": 31744, "\u0120Malik": 31745, "\u0120muzzle": 31746, "\u0120benchmarks": 31747, "dal": 31748, "\u0120Vacc": 31749, "\u0120Hicks": 31750, "609": 31751, "\u0120Biblical": 31752, "heng": 31753, "\u0120overload": 31754, "\u0120Civilization": 31755, "\u0120immoral": 31756, "\u0120fries": 31757, "\u00e3\u0124\u0134": 31758, "\u0120reproduced": 31759, "\u0120formulation": 31760, "jug": 31761, "irez": 31762, "gear": 31763, "\u0120coached": 31764, "MpServer": 31765, "\u0120SJ": 31766, "\u0120Kw": 31767, "Init": 31768, "deal": 31769, "\u0120Oro": 31770, "\u0120Loki": 31771, "\u0120Songs": 31772, "\u0120232": 31773, "\u0120Louise": 31774, "asionally": 31775, "\u0120uncond": 31776, "ollywood": 31777, "\u0120progressives": 31778, "\u0120Enough": 31779, "\u0120Doe": 31780, "\u0120wreckage": 31781, "\u0120brushed": 31782, "\u0120BaseType": 31783, "\u0120zoning": 31784, "ishable": 31785, "hetically": 31786, "\u0120Caucus": 31787, "\u0120Hue": 31788, "\u0120karma": 31789, "\u0120Sporting": 31790, "\u0120trader": 31791, "\u0120seeming": 31792, "\u0120Capture": 31793, "430": 31794, "bish": 31795, "\u0120tunes": 31796, "\u0120indoors": 31797, "\u0120Sphere": 31798, "\u0120Dancing": 31799, "TERN": 31800, "\u0120nob": 31801, "\u0120GST": 31802, "maps": 31803, "\u0120peppers": 31804, "Fit": 31805, "\u0120oversees": 31806, "\u0120Rabbi": 31807, "\u0120Ruler": 31808, "vertising": 31809, "office": 31810, "xxx": 31811, "\u0120raft": 31812, "Changed": 31813, "\u0120textbooks": 31814, "Links": 31815, "\u0120Omn": 31816, "\u00e3\u0122\u0133": 31817, "\u0120inconvenience": 31818, "\u0120Donetsk": 31819, "=~": 31820, "\u0120implicitly": 31821, "\u0120boosts": 31822, "\u0120Bones": 31823, "\u0120Boom": 31824, "Courtesy": 31825, "\u0120sensational": 31826, "ANY": 31827, "\u0120greedy": 31828, "eden": 31829, "\u0120inexper": 31830, "\u0120Ler": 31831, "\u0120Vale": 31832, "\u0120tighten": 31833, "\u0120EAR": 31834, "\u0120Num": 31835, "\u0120ancestor": 31836, "Sent": 31837, "\u0120Horde": 31838, "urgical": 31839, "allah": 31840, "\u0120sap": 31841, "amba": 31842, "\u0120Spread": 31843, "twitch": 31844, "\u0120grandson": 31845, "\u0120fracture": 31846, "\u0120moderator": 31847, "\u0120Seventh": 31848, "\u0120Reverse": 31849, "\u0120estimation": 31850, "Choose": 31851, "\u0120parach": 31852, "\u0120barric": 31853, "\u00e3\u0122\u0132": 31854, "\u0120compass": 31855, "\u0120allergic": 31856, "\u00e2\u0122\u0137": 31857, "OTHER": 31858, "errilla": 31859, "\u0120wagon": 31860, "\u0120zinc": 31861, "\u0120rubbed": 31862, "\u0120Fuller": 31863, "\u0120Luxembourg": 31864, "\u0120Hoover": 31865, "\u0120liar": 31866, "\u0120Evening": 31867, "\u0120Cobb": 31868, "esteem": 31869, "\u0120selector": 31870, "\u0120Brawl": 31871, "isance": 31872, "\u0120Ek": 31873, "\u0120troop": 31874, "\u0120guts": 31875, "\u0120Appeal": 31876, "\u0120Tibetan": 31877, "\u0120routines": 31878, "\u0120Ment": 31879, "\u0120summarized": 31880, "steamapps": 31881, "\u0120tranqu": 31882, "\u01201929": 31883, "oran": 31884, "\u0120Authent": 31885, "\u0120gmaxwell": 31886, "\u0120apprehens": 31887, "\u0120poems": 31888, "\u0120sausage": 31889, "\u0120Webster": 31890, "urus": 31891, "\u0120themed": 31892, "\u0120lounge": 31893, "\u0120charger": 31894, "Spoiler": 31895, "\u0120spilled": 31896, "hog": 31897, "\u0120Sunder": 31898, "\u0120Ain": 31899, "\u0120Angry": 31900, "\u0120disqual": 31901, "\u0120Frequency": 31902, "\u0120Ethernet": 31903, "\u0120helper": 31904, "Percent": 31905, "\u0120horrifying": 31906, "\u0120ail": 31907, "\u0120Allan": 31908, "EEE": 31909, "\u0120Crossing": 31910, "449": 31911, "\u0120holog": 31912, "\u0120Puzzles": 31913, "\u0120Goes": 31914, "erenn": 31915, "604": 31916, "\u00e3\u0123\u0131": 31917, "\u0120Rafael": 31918, "\u0120atten": 31919, "\u0120Emanuel": 31920, "\u0120upro": 31921, "\u0120Susp": 31922, "Psych": 31923, "\u0120Trainer": 31924, "\u0120NES": 31925, "\u0120Hunts": 31926, "becue": 31927, "\u0120counselor": 31928, "Rule": 31929, "\u0120toxins": 31930, "\u0120banners": 31931, "rifice": 31932, "\u0120greeting": 31933, "\u0120frenzy": 31934, "\u0120allocate": 31935, "\u0120*)": 31936, "expr": 31937, "503": 31938, "\u0120Chick": 31939, "\u0120Torn": 31940, "\u0120consolidation": 31941, "\u0120Fletcher": 31942, "switch": 31943, "frac": 31944, "clips": 31945, "\u0120McKin": 31946, "\u0120Lunar": 31947, "Month": 31948, "ITCH": 31949, "\u0120scholarly": 31950, "raped": 31951, "398": 31952, "\u01201910": 31953, "\u0120egreg": 31954, "\u0120insecure": 31955, "\u0120victorious": 31956, "cffffcc": 31957, "\u0120singled": 31958, "\u0120elves": 31959, "\u0120Wond": 31960, "burst": 31961, "\u0120camoufl": 31962, "\u0120BLACK": 31963, "\u0120conditioned": 31964, "\u00e7\u012b": 31965, "answered": 31966, "\u0120compulsory": 31967, "ascist": 31968, "\u0120podcasts": 31969, "\u0120Frankfurt": 31970, "bnb": 31971, "\u0120neoliberal": 31972, "\u0120Keyboard": 31973, "\u0120Belle": 31974, "warm": 31975, "\u0120trusts": 31976, "\u0120insured": 31977, "\u0120Bucc": 31978, "usable": 31979, "607": 31980, "\u0120Plains": 31981, "\u01201890": 31982, "\u0120sabotage": 31983, "\u0120lodged": 31984, "felt": 31985, "\u0120ga": 31986, "\u0120Narc": 31987, "\u0120Salem": 31988, "\u0120seventy": 31989, "\u0120Blank": 31990, "pocket": 31991, "\u0120whisper": 31992, "\u0120mating": 31993, "omics": 31994, "\u0120Salman": 31995, "\u0120Kad": 31996, "\u0120angered": 31997, "\u0120collisions": 31998, "\u0120extraordinarily": 31999, "\u0120coercion": 32000, "Ghost": 32001, "birds": 32002, "\u00e8\u0122": 32003, "kok": 32004, "\u0120permissible": 32005, "avorable": 32006, "\u0120pointers": 32007, "\u0120dissip": 32008, "aci": 32009, "\u0120theatrical": 32010, "\u0120Cosmic": 32011, "\u0120forgetting": 32012, "\u0120finalized": 32013, "\u00e5\u00a4\u00a7": 32014, "yout": 32015, "library": 32016, "\u0120booming": 32017, "\u0120Believe": 32018, "\u0120Teacher": 32019, "\u0120Liv": 32020, "\u0120GOODMAN": 32021, "\u0120Dominican": 32022, "ORED": 32023, "\u0120Parties": 32024, "\u0120precipitation": 32025, "\u0120Slot": 32026, "Roy": 32027, "\u0120Combined": 32028, "\u0120integrating": 32029, "\u0120chrome": 32030, "\u0120intestinal": 32031, "\u0120Rebell": 32032, "\u0120matchups": 32033, "\u0120blockbuster": 32034, "\u0120Loren": 32035, "\u0120Levy": 32036, "\u0120preaching": 32037, "\u0120Sending": 32038, "\u0120Purpose": 32039, "rax": 32040, "fif": 32041, "\u0120authoritative": 32042, "\u0120PET": 32043, "astical": 32044, "\u0120dishon": 32045, "\u0120chatting": 32046, "\u0120\"$:/": 32047, "Connection": 32048, "\u0120recreate": 32049, "\u0120delinqu": 32050, "\u0120broth": 32051, "\u0120Dirty": 32052, "\u0120Admin": 32053, "zman": 32054, "\u0120scholarships": 32055, "\u0120253": 32056, "contact": 32057, "alsa": 32058, "767": 32059, "creen": 32060, "abbage": 32061, "\u01201915": 32062, "\u0120blended": 32063, "\u0120alarmed": 32064, "Language": 32065, "356": 32066, "\u0120blends": 32067, "\u0120Changed": 32068, "Wolf": 32069, "\u0120hepat": 32070, "Creating": 32071, "\u0120persecut": 32072, "\u0120sweetness": 32073, "arte": 32074, "\u0120forfeiture": 32075, "\u0120Roberto": 32076, "impro": 32077, "NFL": 32078, "\u0120Magnet": 32079, "Detailed": 32080, "\u0120insignificant": 32081, "\u0120POLIT": 32082, "\u0120BBQ": 32083, "\u0120CPS": 32084, "\u0120seaw": 32085, "aminer": 32086, "mL": 32087, "endif": 32088, "finals": 32089, "\u0120265": 32090, "uish": 32091, "\u0120})": 32092, "\u0120Problems": 32093, "\u0120emblem": 32094, "\u0120seriousness": 32095, "\u0120parsing": 32096, "\u0120substitution": 32097, "\u0120pressured": 32098, "\u0120recycled": 32099, "aleb": 32100, "Ruby": 32101, "\u0120proficiency": 32102, "Driver": 32103, "\u0120Wester": 32104, ":'": 32105, "AFTA": 32106, "\u0120mantle": 32107, "\u0120Clayton": 32108, "flag": 32109, "\u0120practitioner": 32110, "covered": 32111, "\u0120Struct": 32112, "addafi": 32113, "425": 32114, "\u0120Township": 32115, "\u0120Hydro": 32116, "Louis": 32117, "343": 32118, "\u0120condo": 32119, "\u0120Tao": 32120, "\u0120utilization": 32121, "\u0120nausea": 32122, "\u0120Dems": 32123, "ridges": 32124, "pause": 32125, "\u0120formulas": 32126, "\u0120challenger": 32127, "376": 32128, "\u0120defective": 32129, "\u0120Railway": 32130, "\u0120PubMed": 32131, "\u0120yogurt": 32132, "lbs": 32133, "\u0120Norfolk": 32134, "OPE": 32135, "\u0120Moody": 32136, "\u0120distributor": 32137, "\u0120scrolls": 32138, "\u0120extracts": 32139, "Stan": 32140, "\u0120viability": 32141, "\u0120exposes": 32142, "\u0120starvation": 32143, "\u0120Steps": 32144, "\u0120Dodd": 32145, "few": 32146, "STD": 32147, "332": 32148, "\u0120closures": 32149, "\u0120complementary": 32150, "\u0120Sasha": 32151, "umpy": 32152, "\u0120monet": 32153, "\u0120articulate": 32154, "\u0120Doct": 32155, "killer": 32156, "\u0120scrim": 32157, "\u0120264": 32158, "\u0120prostitutes": 32159, "\u0120severed": 32160, "\u0120attachments": 32161, "\u0120cooled": 32162, "Lev": 32163, "\u0120Falk": 32164, "fail": 32165, "\u0120policeman": 32166, "\u0120Dag": 32167, "\u0120prayed": 32168, "\u0120Kernel": 32169, "\u0120clut": 32170, "\u0120cath": 32171, "\u0120anomaly": 32172, "Storm": 32173, "emaker": 32174, "\u0120Breakfast": 32175, "uli": 32176, "oire": 32177, "JJ": 32178, "hz": 32179, "Operation": 32180, "\u0120Sick": 32181, "354": 32182, "\u0120Guatemala": 32183, "Rate": 32184, "\u0120exposures": 32185, "faces": 32186, "\u0120Archae": 32187, "raf": 32188, "\u0120Mia": 32189, "\u01202025": 32190, "\u0120opaque": 32191, "\u0120disguised": 32192, "\u0120Headquarters": 32193, "Sah": 32194, "\u0120pots": 32195, "978": 32196, "\u0120Malf": 32197, "\u0120frowned": 32198, "\u0120poisonous": 32199, "\u0120Convers": 32200, "eeks": 32201, "\u0120crab": 32202, ".\"\"": 32203, "\u0120treason": 32204, "\u0120ranc": 32205, "\u0120escalating": 32206, "\u0120warr": 32207, "\u0120mobs": 32208, "\u0120lamps": 32209, "\u0120Sunshine": 32210, "\u0120Brunswick": 32211, "Phones": 32212, "\u0120spelled": 32213, "\u0120Skip": 32214, "\u01202050": 32215, "\u01201911": 32216, "\u0120Pluto": 32217, "\u0120Amend": 32218, "\u0120meats": 32219, "387": 32220, "\u0120stomp": 32221, "\u0120Zhou": 32222, "\u0120Leviathan": 32223, "\u0120Hazard": 32224, "adv": 32225, "\u0120Orwell": 32226, "\u0120aloud": 32227, "\u0120bumper": 32228, "\u0120Anarch": 32229, "ubuntu": 32230, "\u0120Serious": 32231, "fitting": 32232, "\u0120Optional": 32233, "\u0120Cecil": 32234, "REAM": 32235, "\u0120serotonin": 32236, "\u0120cultivate": 32237, "agogue": 32238, "}\\": 32239, "\u0120mosques": 32240, "\u0120Sunny": 32241, "\u0120reactive": 32242, "revolution": 32243, "\u0120Lup": 32244, "\u0120Fedora": 32245, "\u0120defenseman": 32246, "\u0120VID": 32247, "istine": 32248, "\u0120drowning": 32249, "\u0120Broadcasting": 32250, "\u0120thriller": 32251, "\u0120Scy": 32252, "\u0120accelerating": 32253, "\u0120directs": 32254, "odied": 32255, "bike": 32256, "duration": 32257, "\u0120painfully": 32258, "Redd": 32259, "\u0120productions": 32260, "\u0120gag": 32261, "\u0120whist": 32262, "\u0120sock": 32263, "\u0120infinitely": 32264, "\u0120Concern": 32265, "\u0120Citadel": 32266, "\u0120lieu": 32267, "\u0120candles": 32268, "ogeneous": 32269, "arger": 32270, "\u0120heavenly": 32271, "inflammatory": 32272, "Performance": 32273, "Cs": 32274, "ructose": 32275, "azaki": 32276, "\u0120pessim": 32277, "\u0120inference": 32278, "\u0120powd": 32279, "\u0120Zoe": 32280, "\u0120paints": 32281, "\u0120dazz": 32282, "pta": 32283, "-----------": 32284, "\u0120inspir": 32285, "\u0120Experimental": 32286, "\u0120Knife": 32287, "regor": 32288, "bors": 32289, "\u0120showers": 32290, "romeda": 32291, "\u0120saint": 32292, "\u0120benign": 32293, "\u0120Jiang": 32294, "\u0120envisioned": 32295, "\u0120shroud": 32296, "IFT": 32297, "HO": 32298, "\u0120shuff": 32299, "\u0120ICC": 32300, "\u0120segreg": 32301, "\u0120revisit": 32302, "ighthouse": 32303, "Li": 32304, "\u0120substrate": 32305, "\u0120Seas": 32306, "\u0120Reward": 32307, "\u0120Hep": 32308, "\u0120Brass": 32309, "sbm": 32310, "\u0120eliminates": 32311, "\u0120stamina": 32312, "\u0120VAT": 32313, "\u0120Loan": 32314, "\u0120constraint": 32315, "\u0120appropriated": 32316, "\u0120pes": 32317, "\u0120ALE": 32318, "ranging": 32319, "\u0120404": 32320, "392": 32321, "\u0120intellectuals": 32322, "achu": 32323, "\u0120restructuring": 32324, "\u0120Levin": 32325, "\u0120runes": 32326, "\u0120delightful": 32327, "\u0120carbohydrates": 32328, "\u0120Models": 32329, "\u0120Expo": 32330, "\u0120transporting": 32331, "alloc": 32332, "\u0120ringing": 32333, "Samsung": 32334, "\u0120scarcely": 32335, "\u0120URLs": 32336, "\u0120MAS": 32337, "\u0120prototypes": 32338, "\u0120narrator": 32339, "\u0120CPUs": 32340, "cdn": 32341, "\u0120Barton": 32342, "\u0120decidedly": 32343, "\u0120Shu": 32344, "ixir": 32345, "ocious": 32346, "\u0120Myst": 32347, "Nintendo": 32348, "\u0120reuse": 32349, "\u0120forgiven": 32350, "Few": 32351, "inical": 32352, "nat": 32353, "\u0120seamless": 32354, "\u0120Eva": 32355, "\u0120EVE": 32356, "\u0120JO": 32357, "landers": 32358, "\u0120softer": 32359, "negie": 32360, "\u0120transient": 32361, "\u0120orbital": 32362, "\u0120fulfil": 32363, "\u0120Kom": 32364, "Hopefully": 32365, "\u0120dynamically": 32366, "\u0120Hunger": 32367, "\u00e5\u013d": 32368, "\u0120Armenia": 32369, "elman": 32370, "berto": 32371, "\u0120pige": 32372, "\u0120IDs": 32373, "limit": 32374, "\u0120veins": 32375, "\u0120soaring": 32376, "packs": 32377, "Golden": 32378, "\u0120Crab": 32379, "istor": 32380, "\u0120RPM": 32381, "\u0120$$": 32382, "gression": 32383, "\u0120jihadist": 32384, "\u0120gamble": 32385, "\u0120careg": 32386, "\u0120inflated": 32387, "Face": 32388, "\u0120Firearms": 32389, "\u0120Emmanuel": 32390, "\u00e2\u013f": 32391, "\u0120shocks": 32392, "grab": 32393, "\u0120splend": 32394, "\u0120HPV": 32395, "abortion": 32396, "Above": 32397, "Entity": 32398, "players": 32399, "\u0120commenced": 32400, "ulence": 32401, "\u0120fulfillment": 32402, "\u0120embodiments": 32403, "\u0120Welfare": 32404, "\u0120hail": 32405, "\u0120<@": 32406, "tten": 32407, "\u0120catcher": 32408, "\u0120Jazeera": 32409, "\u0120volcano": 32410, "\u0120stabilize": 32411, "\u0120Handler": 32412, "\u0120intensified": 32413, "\u0120Abrams": 32414, "\u0120humiliation": 32415, "paced": 32416, "605": 32417, "\u0120CentOS": 32418, "Specific": 32419, "\u0120heed": 32420, "\u0120CAM": 32421, "\u0120Galile": 32422, "Die": 32423, "\u0120abolished": 32424, "\u0120Thomson": 32425, "\u0120Teachers": 32426, "\u0120Wass": 32427, "jong": 32428, "\u0120ISBN": 32429, "\u0120Allies": 32430, "shake": 32431, "\u00e5\u00b7": 32432, "vict": 32433, "Howard": 32434, "\u0120deem": 32435, "\u0120exceedingly": 32436, "\u0120Smartstocks": 32437, "ibe": 32438, "\u0120doorway": 32439, "\u0120competed": 32440, "igmat": 32441, "\u0120nationalists": 32442, "\u0120groom": 32443, "\u0120Keen": 32444, "\u0120disposable": 32445, "decl": 32446, "\u0120Tolkien": 32447, "\u0120Scheme": 32448, "\u0120biod": 32449, "\u0120avid": 32450, "\u0120Elon": 32451, "agar": 32452, "\u0120TSA": 32453, "Roman": 32454, "\u0120artificially": 32455, "\u0120advisors": 32456, "XL": 32457, "\u0120Inferno": 32458, "366": 32459, "\u0120tedious": 32460, "\u0120Photography": 32461, "\u0120Carrie": 32462, "\u0120trope": 32463, "\u0120Sandra": 32464, "\u0120decimal": 32465, "Queen": 32466, "\u0120Gundam": 32467, "\u0120OM": 32468, "otech": 32469, "NBA": 32470, "\u01201932": 32471, "\u0120entrenched": 32472, "\u0120Marion": 32473, "\u0120fraternity": 32474, "Labour": 32475, "Henry": 32476, "\u0120latitude": 32477, "Either": 32478, "\u0120enhances": 32479, "\u0120Potential": 32480, "\u0120shines": 32481, "idad": 32482, "\u0120breadth": 32483, "\u0120capacities": 32484, "\u0120\u00f0\u0141\u013b\u0124": 32485, "\u0120Bronx": 32486, "\u0120sexes": 32487, "\u0120differentiation": 32488, "\u0120heavyweight": 32489, "\u0120Taj": 32490, "dra": 32491, "\u0120migrate": 32492, "\u0120exhaustion": 32493, "\u0120RUN": 32494, "elsius": 32495, "\u0120Cuomo": 32496, "\u0120guitars": 32497, "\u0120clones": 32498, "\u0120Somew": 32499, "\u0120Pry": 32500, "-------------": 32501, "\u0120warranted": 32502, "cycles": 32503, "\u0120salvage": 32504, "\u0120disks": 32505, "RANT": 32506, "\u0120NGOs": 32507, "\u0120Martian": 32508, "\":[{\"": 32509, "\u0120addicts": 32510, "ojure": 32511, "illet": 32512, "\u0120amazingly": 32513, "artments": 32514, "pixel": 32515, "\u0120GPUs": 32516, "Layout": 32517, "\u00e8\u00a3": 32518, "\u0120Tamil": 32519, "\u0120Basil": 32520, "\u0120impartial": 32521, "\u0120Structure": 32522, "fork": 32523, "bryce": 32524, "\u0120ridge": 32525, "\u0120Hamburg": 32526, "rious": 32527, "\u0120blitz": 32528, "cigarettes": 32529, "\u0120canned": 32530, "402": 32531, "\u0120ironically": 32532, "\u0120compassionate": 32533, "\u0120Hawkins": 32534, ".#": 32535, "\u0120Cathedral": 32536, "\u0120rallied": 32537, "internal": 32538, "\u0120quota": 32539, "stakes": 32540, "TEXT": 32541, "mom": 32542, "\u0120completes": 32543, "\u0120238": 32544, "\u0120shrug": 32545, "\u00e3\u0125\u0133": 32546, "\u0120Ninth": 32547, "\u0120revise": 32548, "\u0120Provider": 32549, "\u0120treacher": 32550, "\u0120quasi": 32551, "\u0120PRES": 32552, "\u0120deposition": 32553, "\u0120confidentiality": 32554, "issors": 32555, "\u0120imbalance": 32556, "\u0120spanning": 32557, "\u0120angular": 32558, "\u0120Cul": 32559, "communication": 32560, "\u0120Nora": 32561, "\u0120Genius": 32562, "opter": 32563, "\u0120sacked": 32564, "Spot": 32565, "\u0120finely": 32566, "\u0120CHR": 32567, "282": 32568, "waves": 32569, "Palest": 32570, "\u0120Rohing": 32571, "NL": 32572, "\u00e8\u00bf": 32573, "\u0120shitty": 32574, "\u0120Scalia": 32575, "475": 32576, "Progress": 32577, "\u0120referencing": 32578, "\u0120classrooms": 32579, "abee": 32580, "\u0120sod": 32581, "hesion": 32582, "708": 32583, "\u0120Zuckerberg": 32584, "\u0120Finish": 32585, "\u0120Scotia": 32586, "\u0120Savior": 32587, "\u0120Installation": 32588, "antha": 32589, "(-": 32590, "\u0120302": 32591, "\u0120Punk": 32592, "\u0120crater": 32593, "youtu": 32594, "\u0120roast": 32595, "\u0120influencing": 32596, "\u0120dup": 32597, "\u0120JR": 32598, "\u0120Grav": 32599, "\u0120stature": 32600, "\u0120bathrooms": 32601, "Aside": 32602, "Wiki": 32603, "mean": 32604, "\u0120Zak": 32605, "\u0120Ones": 32606, "\u0120Nath": 32607, "\u0120hypert": 32608, "\u0120commencement": 32609, "Civil": 32610, "\u0120moderately": 32611, "\u0120distributors": 32612, "\u0120breastfeeding": 32613, "\u0120980": 32614, "\u0120Sik": 32615, "\u0120Cig": 32616, "\u0120AMER": 32617, "RIP": 32618, "\u0120Career": 32619, "usting": 32620, "\u0120messed": 32621, "\u0120eh": 32622, "\u0120Jensen": 32623, "/$": 32624, "\u0120blackmail": 32625, "\u0120conversions": 32626, "\u0120scientifically": 32627, "\u0120mantra": 32628, "paying": 32629, "\u0120ivory": 32630, "\u0120Courts": 32631, "OUGH": 32632, "auntlet": 32633, "Serial": 32634, "Brow": 32635, "\u0120Hundreds": 32636, "323": 32637, "\u0120pee": 32638, "\u0120linux": 32639, "\u0120submer": 32640, "\u0120Principal": 32641, "485": 32642, "\u0120DSL": 32643, "\u0120Cousins": 32644, "\u0120doctrines": 32645, "\u0120Athletics": 32646, "\u0120315": 32647, "\u0120Karma": 32648, "\u0120attent": 32649, "urger": 32650, "\u0120prescribe": 32651, "\u0120encaps": 32652, "\u0120Came": 32653, "\u0120secretive": 32654, "\u0120Crimes": 32655, "dn": 32656, "Clean": 32657, "\u0120Egyptians": 32658, "\u0120Carpenter": 32659, "\u0120ll": 32660, "Hum": 32661, "\u0120Milo": 32662, "\u0120capitalists": 32663, "\u0120briefed": 32664, "Twe": 32665, "\u0120Basin": 32666, "elvet": 32667, "Mos": 32668, "\u0120plunge": 32669, "\u0120Kaiser": 32670, "\u0120Fuj": 32671, "illin": 32672, "\u0120safeguards": 32673, "\u0120oste": 32674, "\u0120Opportunity": 32675, "\u0120Mafia": 32676, "\u0120Calling": 32677, "apa": 32678, "urban": 32679, "brush": 32680, "illard": 32681, "c\u00c3\u00a9": 32682, "intelligence": 32683, "\u0120Lob": 32684, "\u0120Druid": 32685, "\u0120smoother": 32686, "\u0120footing": 32687, "\u0120motorists": 32688, "arcity": 32689, "\u0120masculinity": 32690, "\u0120mism": 32691, "\u0120abdominal": 32692, "\u0120Tavern": 32693, "\u0120Roh": 32694, "\u0120escapes": 32695, "signed": 32696, "Anthony": 32697, "\u0120sacrificing": 32698, "\u0120intimacy": 32699, "\u0120anterior": 32700, "\u0120Kod": 32701, "\u0120motif": 32702, "\u0120graz": 32703, "\u0120visualization": 32704, "\u0120guitarist": 32705, "\u0120Trotsky": 32706, "magic": 32707, "Dar": 32708, "\u0120Mori": 32709, "\u0120wards": 32710, "\u0120toilets": 32711, "lest": 32712, "\u0120teleport": 32713, "\u0120Sundays": 32714, "\u0120Plat": 32715, "ETS": 32716, "\u0120eSports": 32717, "Patrick": 32718, "\u0120Katherine": 32719, "enko": 32720, "\u0120hassle": 32721, "\u0120Mick": 32722, "ggles": 32723, "\u0120hob": 32724, "aintain": 32725, "\u0120airborne": 32726, "\u0120spans": 32727, "\u0120chili": 32728, "\u0120aperture": 32729, "\u0120volunteered": 32730, "\u0120Incident": 32731, "\u0120Fres": 32732, "\u0120Veteran": 32733, "aughtered": 32734, "ingo": 32735, "\u0120uninsured": 32736, "CLOSE": 32737, "\u0120fuse": 32738, "\u0120erotic": 32739, "\u0120advertise": 32740, "raising": 32741, "Texture": 32742, "\u0120attends": 32743, "\u0120REAL": 32744, "uddled": 32745, "\u0120smoot": 32746, "\u0120305": 32747, "\u0120Willis": 32748, "\u0120blond": 32749, "Analysis": 32750, "\u0120VT": 32751, "onica": 32752, "\u0120stronghold": 32753, "RF": 32754, "NM": 32755, ".>>": 32756, "\u0120prosperous": 32757, "\u0120boasted": 32758, "292": 32759, "\u0120Manufacturing": 32760, "PRESS": 32761, "gren": 32762, "\u0120pharmacy": 32763, "\u0120Rockefeller": 32764, "kai": 32765, "\u0120thumbs": 32766, "\u0120Hut": 32767, "\u0120motherboard": 32768, "\u0120guardians": 32769, "\u0120Alter": 32770, "llular": 32771, "\u0120shack": 32772, "\u0120wisely": 32773, "\u0120backbone": 32774, "erva": 32775, "\u0120suicides": 32776, "\u0120McGregor": 32777, "ijah": 32778, "Emer": 32779, "\u0120Brav": 32780, "\u0120designate": 32781, "POST": 32782, "produced": 32783, "\u0120cleansing": 32784, "irlwind": 32785, "existent": 32786, "\u0120Humph": 32787, "\u0120Payne": 32788, "\u0120vested": 32789, "\u00c5\u00a1": 32790, "\u0120stringent": 32791, "iona": 32792, "\u0120unsub": 32793, "\u0120summed": 32794, "\u0120Hercules": 32795, "subject": 32796, "\u0120Ragnar": 32797, "\u0120Nos": 32798, "\u0120characterization": 32799, "\u0120savvy": 32800, "\u0120Dawson": 32801, "\u0120Casino": 32802, "\u0120fri": 32803, "\u0120Barrier": 32804, "\u0120misinformation": 32805, "\u0120insulation": 32806, "\u0120corridors": 32807, "\u0120airplanes": 32808, "\u0120Noct": 32809, "ahi": 32810, "\u01201916": 32811, "kb": 32812, "armac": 32813, "\u0120shun": 32814, "\u0120schema": 32815, "\u0120horrified": 32816, "\u0120239": 32817, "aunders": 32818, "NB": 32819, "iates": 32820, "erity": 32821, "\u0120Shard": 32822, "\u0120rarity": 32823, "\u0120grouped": 32824, "\u0120Ghana": 32825, "against": 32826, "\u0120Biological": 32827, "\u0120Aware": 32828, "owell": 32829, "\u00cf\u0126": 32830, "\u0120Beau": 32831, "shaw": 32832, "Hack": 32833, "\u0120Julius": 32834, "USS": 32835, "olson": 32836, "auna": 32837, "cru": 32838, "\u0120Maurice": 32839, "\u0120Ik": 32840, "\u0120sequencing": 32841, "\u0120radicals": 32842, "\u0120(?,": 32843, "virtual": 32844, "\u0120anyways": 32845, "\u0120reperc": 32846, "\u0120handlers": 32847, "\u0120hesitant": 32848, "\u00e9\u0125": 32849, "\u0120MF": 32850, "plementation": 32851, "associated": 32852, "\u0120campaigned": 32853, "\u0120Yue": 32854, "utations": 32855, "\u0120Yoga": 32856, "\u0120simmer": 32857, "\u0120rods": 32858, "\u0120melody": 32859, "\u0120convoy": 32860, "videos": 32861, "\u0120screened": 32862, "Neg": 32863, "ochemical": 32864, "\u0120())": 32865, "\u0120ultras": 32866, "\u0120antip": 32867, "\u0120Islanders": 32868, "704": 32869, "\u0120fetish": 32870, "\u0120ridiculously": 32871, "\u0120Kart": 32872, "\u0120mitochondrial": 32873, "\u0120interfering": 32874, "Builder": 32875, "\u0120overfl": 32876, "\u0120acne": 32877, "\u0120Mud": 32878, "\u0120Kerr": 32879, "flex": 32880, "\u0120Postal": 32881, "\u0120Baltic": 32882, "477": 32883, "\u0120Persons": 32884, "ourage": 32885, "HB": 32886, "\u0120Muse": 32887, "\u0120Immortal": 32888, "\u0120Driving": 32889, "\u0120petitions": 32890, "\u0120subscript": 32891, "\u0120sorce": 32892, "\u0120Processor": 32893, "uton": 32894, "Sony": 32895, "\u0120phon": 32896, "\u0120raced": 32897, "\u0120Anthrop": 32898, "\u0120daytime": 32899, "\u0120Exercise": 32900, "Adding": 32901, "\u0120engages": 32902, "\u0120Qualcomm": 32903, "\u0120miracles": 32904, "\u0120memes": 32905, "\u0120Drink": 32906, "\u0120Orioles": 32907, "\u0120hairs": 32908, "\u0120Polar": 32909, "athom": 32910, "\u0120slippery": 32911, "\u0120Remy": 32912, "\u0120caramel": 32913, "\u0120YEAR": 32914, "\u0120alk": 32915, "Ign": 32916, "aution": 32917, "\u0120Merlin": 32918, "\u0120Cran": 32919, "\u0120apologies": 32920, "\u0120410": 32921, "\u0120outing": 32922, "\u0120Memories": 32923, "appointed": 32924, "\u0120countered": 32925, "uld": 32926, "posing": 32927, "\u0120firewall": 32928, "\u0120Wast": 32929, "\u0120Wet": 32930, "worked": 32931, "seller": 32932, "\u0120repealed": 32933, "ereo": 32934, "assuming": 32935, "BLIC": 32936, "mite": 32937, "\u0120CEOs": 32938, "\u0120Chapel": 32939, "elligent": 32940, "________________________": 32941, "Dog": 32942, "\u0120wart": 32943, "\u0120subscriber": 32944, "sports": 32945, "\u0120begged": 32946, "\u0120MV": 32947, "\u0120semif": 32948, "ethical": 32949, "\u0120preach": 32950, "\u0120revital": 32951, "\u0120punitive": 32952, "\u0120shortcuts": 32953, "\u0120instituted": 32954, "\u0120Warsaw": 32955, "\u0120abdomen": 32956, "\u0120KING": 32957, "\u0120superintendent": 32958, "\u0120fry": 32959, "\u0120Geo": 32960, "TOR": 32961, "\u0120contradictions": 32962, "aptic": 32963, "\u0120landscapes": 32964, "bugs": 32965, "\u0120clust": 32966, "\u0120volley": 32967, "cribed": 32968, "\u0120tandem": 32969, "\u0120robes": 32970, "WHAT": 32971, "\u0120promoter": 32972, "\u0120eloqu": 32973, "reviewed": 32974, "\u0120DK": 32975, "\u0120Plato": 32976, "\u0120fps": 32977, "Tank": 32978, "\u0120Derrick": 32979, "\u0120prioritize": 32980, "asper": 32981, "\u0120Honduras": 32982, "\u0120Completed": 32983, "nec": 32984, "\u0120mog": 32985, "nir": 32986, "\u0120Mayo": 32987, "DEF": 32988, "stall": 32989, "inness": 32990, "\u0120Volkswagen": 32991, "\u0120precaution": 32992, "\u0120Mell": 32993, "iak": 32994, "istries": 32995, "\u0120248": 32996, "\u0120overlapping": 32997, "Senate": 32998, "\u0120Enhance": 32999, "resy": 33000, "racial": 33001, "ORTS": 33002, "\u0120Mormons": 33003, "Strong": 33004, "\u0120Coch": 33005, "Mexico": 33006, "\u0120Maduro": 33007, "\u0120jars": 33008, "\u0120cane": 33009, "Wik": 33010, "olla": 33011, "ifference": 33012, "\u0120physicist": 33013, "\u0120Maggie": 33014, "\u0120285": 33015, "\u0120depiction": 33016, "\u0120McLaren": 33017, "Ju": 33018, "\u0120slows": 33019, "\u0120commissioners": 33020, "\u0120Willow": 33021, "\u0120Explos": 33022, "hovah": 33023, "\u0120technician": 33024, "\u0120homicides": 33025, "\u0120Flav": 33026, "\u0120Truman": 33027, "\u012010000": 33028, "uctor": 33029, "\u0120shader": 33030, "Newsletter": 33031, "457": 33032, "\u0120rever": 33033, "\u0120hardened": 33034, "\u0120whereabouts": 33035, "\u0120redevelop": 33036, "\u0120carbs": 33037, "\u0120travers": 33038, "\u0120squirrel": 33039, "\u0120follower": 33040, "\u0120sings": 33041, "508": 33042, "\u0120rabbits": 33043, "emonium": 33044, "\u0120documenting": 33045, "\u0120misunderstood": 33046, ")'": 33047, "Rick": 33048, "ggies": 33049, "\u0120premie": 33050, "\u0120skating": 33051, "\u0120passports": 33052, "\u0120fists": 33053, "ageddon": 33054, "Haw": 33055, "ACP": 33056, "080": 33057, "\u0120Thoughts": 33058, "\u0120Carlson": 33059, "\u0120priesthood": 33060, "hua": 33061, "\u0120dungeons": 33062, "\u0120Loans": 33063, "\u0120antis": 33064, "\u0120familiarity": 33065, "\u0120Sabb": 33066, "opal": 33067, "\u0120Ink": 33068, "strike": 33069, "\u0120cram": 33070, "\u0120legalized": 33071, "\u0120cuisine": 33072, "\u0120fibre": 33073, "Travel": 33074, "\u0120Monument": 33075, "ODY": 33076, "ethy": 33077, "\u0120interstate": 33078, "\u0120PUR": 33079, "emporary": 33080, "\u0120Arabian": 33081, "developed": 33082, "\u0120saddle": 33083, "\u0120github": 33084, "\u0120Offer": 33085, "\u0120ISP": 33086, "rolet": 33087, "\u0120SUPER": 33088, "\u0120Denis": 33089, "\u0120multiplier": 33090, "\u0120stirred": 33091, "Interestingly": 33092, "\u0120customary": 33093, "\u0120billed": 33094, "hex": 33095, "\u0120multiplied": 33096, "\u0120flipping": 33097, "\u0120Crosby": 33098, "\u0120fundamentals": 33099, "iae": 33100, "\u0120Played": 33101, "\u0120Atom": 33102, "amazon": 33103, "\u0120Flam": 33104, "eez": 33105, "activated": 33106, "\u0120tablespoon": 33107, "\u0120liberalism": 33108, "\u0120Palin": 33109, "\u0120Patel": 33110, "Num": 33111, "\u0120TAM": 33112, "\u0120surn": 33113, "\u0120Reloaded": 33114, "\u0120coined": 33115, "\"],": 33116, "\u0120Clash": 33117, "\u0120Agu": 33118, "\u0120pragmatic": 33119, "\u0120Activate": 33120, "\u0120802": 33121, "\u0120trailers": 33122, "\u0120silhou": 33123, "\u0120probes": 33124, "\u0120circus": 33125, "\u0120Bain": 33126, "\u0120Lindsay": 33127, "\u0120Abbey": 33128, "Delivery": 33129, "\u0120concession": 33130, "\u0120gastro": 33131, "\u0120Sprite": 33132, "\u00c4\u0141": 33133, "andel": 33134, "\u0120gimm": 33135, "\u0120autobi": 33136, "\u0120Turtle": 33137, "\u0120wonderfully": 33138, "\u0120Haram": 33139, "\u0120Worldwide": 33140, "\u0120Handle": 33141, "\u0120theorists": 33142, "\u0120sleek": 33143, "\u0120Zhu": 33144, "ographically": 33145, "EGA": 33146, "\u0120Owners": 33147, "aths": 33148, "\u0120Antarctic": 33149, "natal": 33150, "=\"\"": 33151, "flags": 33152, "````": 33153, "\u0120sul": 33154, "Kh": 33155, "\u0120potassium": 33156, "\u0120lineman": 33157, "\u0120cereal": 33158, "\u0120Seasons": 33159, "\u01202022": 33160, "\u0120mathematic": 33161, "\u0120astronomers": 33162, "professional": 33163, "\u0120fares": 33164, "cknowled": 33165, "\u0120chi": 33166, "\u0120youngsters": 33167, "\u0120mistakenly": 33168, "\u0120hemisphere": 33169, "\u0120Divinity": 33170, "rone": 33171, "\u0120\",": 33172, "rings": 33173, "\u0120attracts": 33174, "vana": 33175, "\u00e5\u00b9": 33176, "CAP": 33177, "\u0120playlist": 33178, "\u0120porch": 33179, "\u00e3\u0123\u00a3": 33180, "\u0120incorporates": 33181, "\u0120soak": 33182, "\u0120asserting": 33183, "\u0120Terrorism": 33184, "\u0120Pablo": 33185, "Ja": 33186, "cester": 33187, "\u0120fearing": 33188, "\u0120Prayer": 33189, "\u0120escalated": 33190, "GW": 33191, "\u0120robe": 33192, "\u0120Brighton": 33193, "acists": 33194, "\u0120Symphony": 33195, "\u0120Dwarf": 33196, "\u0120Parade": 33197, "\u0120Lego": 33198, "\u0120inexpl": 33199, "\u0120lords": 33200, "leaf": 33201, "RAG": 33202, "liber": 33203, "\u0120cigars": 33204, "\u0120Jehovah": 33205, "606": 33206, "WINDOWS": 33207, "\u0120Liberia": 33208, "ebus": 33209, "Heavy": 33210, "\u0120lubric": 33211, "\u0120RW": 33212, "anguages": 33213, "\u0120narrowed": 33214, "computer": 33215, "\u0120Ember": 33216, "\u0120murdering": 33217, "\u0120downstream": 33218, "\u0120Tuls": 33219, "\u0120Tables": 33220, "Topic": 33221, "\u0120Accuracy": 33222, "=/": 33223, "lost": 33224, "\u0120Rei": 33225, "\u0120progresses": 33226, "bear": 33227, "\u0120establishments": 33228, "Justin": 33229, "\u0120Peach": 33230, "\u0120Gomez": 33231, "\u00e5\u00bf": 33232, "\u0120Triangle": 33233, "Ident": 33234, "\u0120Hive": 33235, "Resources": 33236, "\u0120mixes": 33237, "\u0120Assuming": 33238, "Mu": 33239, "\u0120hypoc": 33240, "\u0120sane": 33241, "\u0120Wan": 33242, "idious": 33243, "Success": 33244, "\u0120io": 33245, "Angel": 33246, "\u0120dangerously": 33247, "\u0120Creature": 33248, "WORK": 33249, ":[": 33250, "\u0120Katrina": 33251, "Listener": 33252, "Miller": 33253, "\u0120Idlib": 33254, "hang": 33255, "\u0120circumvent": 33256, "href": 33257, "\u0120celestial": 33258, "\u0120Weeks": 33259, "\u0120Pug": 33260, "\u0120Dalton": 33261, "\u0120subpoena": 33262, "uku": 33263, "\u0120persisted": 33264, "pei": 33265, "olding": 33266, "\u0120Documents": 33267, "\u0120Hast": 33268, "\u0120CENT": 33269, "\u0120primer": 33270, "\u0120synonymous": 33271, "\u0120nib": 33272, "ombs": 33273, "\u0120notation": 33274, "\u0120Dish": 33275, "\u0120Atmosp": 33276, "\u0120forbid": 33277, "\u0120ANG": 33278, "pattern": 33279, "los": 33280, "\u0120projectiles": 33281, "brown": 33282, ".\",": 33283, "\u0120Venom": 33284, "\u0120fiercely": 33285, "ublished": 33286, "\u0120Uran": 33287, "\u0120Nicarag": 33288, "410": 33289, "\u0120CAL": 33290, "OTOS": 33291, "\u0120Miracle": 33292, "\u0120Enchant": 33293, "\u0120guarding": 33294, "append": 33295, "Attach": 33296, "\u0120leveled": 33297, "\u0120condoms": 33298, "ihilation": 33299, "649": 33300, "\u0120nightmares": 33301, "\u0120THEY": 33302, "\u0120START": 33303, "\u0120Kinn": 33304, "\u0120roommate": 33305, "\u0120hygiene": 33306, "opping": 33307, "Job": 33308, "\u0120lvl": 33309, "\u0120VER": 33310, "\u0120Keeping": 33311, "abetic": 33312, "\u0120formatting": 33313, "erala": 33314, "\u0120revisions": 33315, "\u0120resurg": 33316, "Tel": 33317, "\u0120Goodman": 33318, "353": 33319, "pod": 33320, "\u0120indisp": 33321, "\u0120Translation": 33322, "\u0120gown": 33323, "\u0120Mund": 33324, "\u0120cis": 33325, "\u0120bystand": 33326, "collect": 33327, "\u0120Punjab": 33328, "actively": 33329, "\u0120Gamb": 33330, "tell": 33331, "\u0120importing": 33332, "gencies": 33333, "\u0120locom": 33334, "\u0120Brill": 33335, "Holy": 33336, "\u0120Berger": 33337, "\u0120showdown": 33338, "\u0120responders": 33339, "ILY": 33340, "\u0120takedown": 33341, "leted": 33342, "\u0120mattered": 33343, "\u0120predictive": 33344, "\u0120overlay": 33345, "GPU": 33346, "\u0120Vick": 33347, "\u0120conveyed": 33348, "Tab": 33349, "peer": 33350, "Scan": 33351, "\u0120defensively": 33352, "vae": 33353, "\u0120approving": 33354, "\u0120tiers": 33355, "\u0120Via": 33356, "querade": 33357, "\u0120Saudis": 33358, "\u0120demolished": 33359, "\u0120Prophe": 33360, "\u0120mono": 33361, "\u0120hospitality": 33362, "HAM": 33363, "\u0120Ariel": 33364, "MOD": 33365, "\u0120Torah": 33366, "\u0120blah": 33367, "\u0120Belarus": 33368, "erential": 33369, "\u0120Tuc": 33370, "\u0120banker": 33371, "397": 33372, "\u0120mosquit": 33373, "\u0120Scientist": 33374, "\u0120Musical": 33375, "\u0120hust": 33376, "Shift": 33377, "\u0120torment": 33378, "\u0120standoff": 33379, "Educ": 33380, "\u0120Fog": 33381, "\u0120amplifier": 33382, "Shape": 33383, "Instance": 33384, "\u0120Critics": 33385, "\u0120daemon": 33386, "Houston": 33387, "\u0120mattress": 33388, "\u0120IDF": 33389, "\u0120obscene": 33390, "\u0120Amer": 33391, "hetti": 33392, "\u0120compiling": 33393, "352": 33394, "verett": 33395, "\u0120Reduction": 33396, "istration": 33397, "\u0120Blessed": 33398, "\u0120Bachelor": 33399, "316": 33400, "\u0120prank": 33401, "\u0120Vulcan": 33402, "dding": 33403, "\u0120mourning": 33404, "\u0120Quint": 33405, "\u0120Blaster": 33406, "testing": 33407, "\u0120sediment": 33408, ">>>": 33409, "\u0120Eternity": 33410, "\u0120WHERE": 33411, "\u0120Maze": 33412, "\u0120reacting": 33413, "\u0120Alv": 33414, "omsday": 33415, "\u0120CRA": 33416, "\u0120translator": 33417, "\u0120bogus": 33418, "atu": 33419, "Website": 33420, "olls": 33421, "\u0120baptism": 33422, "\u0120sibling": 33423, "\u0120Autumn": 33424, "vez": 33425, "\u00e3\u0123\u00ae\u00e9": 33426, "guards": 33427, "Georg": 33428, "assadors": 33429, "\u0120Freud": 33430, "\u0120continents": 33431, "\u0120Registry": 33432, "Bernie": 33433, "\u0138\u013c\u00e5\u00a3\u00ab": 33434, "\u0120tolerant": 33435, "\u0120UW": 33436, "\u0120horribly": 33437, "995": 33438, "\u0120MIDI": 33439, "\u0120impatient": 33440, "ocado": 33441, "eri": 33442, "\u0120Worst": 33443, "\u0120Norris": 33444, "\u0120Talking": 33445, "\u0120defends": 33446, "ensable": 33447, "\u01202021": 33448, "\u0120anatomy": 33449, "Lew": 33450, "\u0120drawer": 33451, "\u0120Canberra": 33452, "\u0120patriotic": 33453, "\u00e9\u00be\u012f\u00e5\u0138\u013c\u00e5\u00a3\u00ab": 33454, "\u0120Avg": 33455, "ARM": 33456, "\u0120undisclosed": 33457, "\u0120farewell": 33458, "459": 33459, "bable": 33460, "\u0120Allison": 33461, "OLOG": 33462, "\u0120conco": 33463, "tight": 33464, "\u0120ACPI": 33465, "\u0120Mines": 33466, "lich": 33467, "\u0120\u00e2\u0136\u013e": 33468, "represented": 33469, "200000": 33470, "\u0120enthusiast": 33471, "OTS": 33472, "bil": 33473, "\u0120Ingredients": 33474, "\u0120inventor": 33475, "\u0120MySQL": 33476, "\u00c2\u0142\u00c2\u0142\u00c2\u0142": 33477, "\u0120ABOUT": 33478, "within": 33479, "\u0120mk": 33480, "Bul": 33481, "\u0120Fake": 33482, "\u0120draconian": 33483, "Wa": 33484, "helm": 33485, "\u0120Terran": 33486, "erville": 33487, "\u0120commonplace": 33488, "SIZE": 33489, "\u0120\"<": 33490, "replace": 33491, "ographs": 33492, "\u0120SELECT": 33493, "incible": 33494, "\u0120Mostly": 33495, "\u0120Sheffield": 33496, "\u0120IDE": 33497, "uggle": 33498, "\u0120citations": 33499, "hurst": 33500, "\u0120Unix": 33501, "\u0120unleash": 33502, "\u0120Piper": 33503, "\u0120Nano": 33504, "\u0120succumb": 33505, "\u0120reluctance": 33506, "\u01202500": 33507, "\u0120Merchant": 33508, "\u0120wiret": 33509, "\u0120combos": 33510, "\u0120Birthday": 33511, "\u0120charcoal": 33512, "\u0120UPS": 33513, "\u0120Fairfax": 33514, "\u0120driveway": 33515, "\u0120Tek": 33516, "\u0120Pitch": 33517, "overe": 33518, "\u0120technicians": 33519, "\u0120Actual": 33520, "flation": 33521, "\u0120Fiscal": 33522, "\u0120Empty": 33523, "anamo": 33524, "\u0120magnesium": 33525, "\u0120slut": 33526, "\u0120growers": 33527, "Investigators": 33528, "():": 33529, "\u0120Satellite": 33530, "\u0120Keynes": 33531, "missive": 33532, "lane": 33533, "\u0120borough": 33534, "344": 33535, "\u0120TEAM": 33536, "\u0120Bethesda": 33537, "CV": 33538, "hower": 33539, "\u0120RAD": 33540, "\u0120chant": 33541, "\u0120Riy": 33542, "\u0120compositions": 33543, "\u0120mildly": 33544, "\u0120meddling": 33545, "\u0120agility": 33546, "aneers": 33547, "501": 33548, "\u0120synth": 33549, "linger": 33550, "291": 33551, "\u0120exclaimed": 33552, "Party": 33553, "\u0120contamin": 33554, "\u0120Manor": 33555, "\u0120Respond": 33556, "\u0120praising": 33557, "\u0120manners": 33558, "fleet": 33559, "Summer": 33560, "\u0120Lynd": 33561, "\u0120Definitely": 33562, "grim": 33563, "\u0120bowling": 33564, "stri": 33565, "\u00e7\u013d": 33566, "ynt": 33567, "\u0120mandates": 33568, "DIV": 33569, "\u0120reconcile": 33570, "views": 33571, "\u0120Damon": 33572, "vette": 33573, "Flo": 33574, "\u0120Greatest": 33575, "ilon": 33576, "icia": 33577, "\u0120portrayal": 33578, "\u0120cushion": 33579, "504": 33580, "1979": 33581, "ossal": 33582, "Applic": 33583, "scription": 33584, "\u0120mitigation": 33585, "ATS": 33586, "pac": 33587, "\u0120erased": 33588, "\u0120deficiencies": 33589, "\u0120Hollande": 33590, "\u0120Xu": 33591, "\u0120bred": 33592, "\u0120pregnancies": 33593, "femin": 33594, "\u0120emph": 33595, "\u0120planners": 33596, "\u0120outper": 33597, "uttering": 33598, "\u0120perpetrator": 33599, "\u0120motto": 33600, "\u0120Ellison": 33601, "\u0120NEVER": 33602, "\u0120admittedly": 33603, "ARI": 33604, "\u0120Azerbaijan": 33605, "\u0120millisec": 33606, "\u0120combustion": 33607, "\u0120Bottle": 33608, "\u0120Lund": 33609, "\u0120Ps": 33610, "\u0120Dress": 33611, "\u0120fabricated": 33612, "\u0120battered": 33613, "\u0120sidel": 33614, "\u0120Notting": 33615, "Foreign": 33616, "\u0120Jerome": 33617, "020": 33618, "\u0120Arbit": 33619, "\u0120knots": 33620, "\u0120RIGHT": 33621, "Moving": 33622, "\u00e3\u0123\u013b": 33623, "\u0120surgeries": 33624, "\u0120courthouse": 33625, "\u0120mastered": 33626, "\u0120hovering": 33627, "\u0120Bran": 33628, "\u0120Alison": 33629, "\u0120safest": 33630, "military": 33631, "\u0120bullied": 33632, "\u0120barrage": 33633, "Reader": 33634, "ESE": 33635, "\u0120Geographic": 33636, "Tools": 33637, "314": 33638, "\u0120Geek": 33639, "roth": 33640, "glers": 33641, "\u0120FIN": 33642, "\u00cf\u0123": 33643, "\u0120Aston": 33644, "altern": 33645, "488": 33646, "\u0120veterin": 33647, "Gamer": 33648, "\u0120intel": 33649, "renches": 33650, "Shield": 33651, "\u0120amnesty": 33652, "\u0120Bhar": 33653, "\u0120piled": 33654, "\u0120honorable": 33655, "\u0120Institutes": 33656, "\u0120soaked": 33657, "\u0120coma": 33658, "\u0120EFF": 33659, "341": 33660, "bytes": 33661, "\u0120Gmail": 33662, "lein": 33663, "\u0120Canadiens": 33664, "material": 33665, "Il": 33666, "\u0120instructors": 33667, "\u0120KY": 33668, "\u0120conceive": 33669, "ubb": 33670, "\u0120Possible": 33671, "\u0120easing": 33672, "\u0120Christina": 33673, "\u0120caric": 33674, "\u0120HDR": 33675, "ROM": 33676, "\u0120shovel": 33677, "delete": 33678, "\u0120puff": 33679, "\u0120Changing": 33680, "\u0120seamlessly": 33681, "Attribute": 33682, "\u0120acquisitions": 33683, "akery": 33684, "\u0120EF": 33685, "\u0120autistic": 33686, "\u0120Takes": 33687, "\u0120Powder": 33688, "\u0120Stir": 33689, "510": 33690, "\u0120Bubble": 33691, "settings": 33692, "\u0120Fowler": 33693, "\u0120mustard": 33694, "\u0120moreover": 33695, "\u0120copyrighted": 33696, "\u0120LEDs": 33697, "1500": 33698, "\u00e6\u012b": 33699, "\u0120HIS": 33700, "enf": 33701, "\u0120custod": 33702, "\u0120Huck": 33703, "Gi": 33704, "\u0120img": 33705, "Answer": 33706, "Ct": 33707, "jay": 33708, "\u0120Infrastructure": 33709, "\u0120federally": 33710, "Loc": 33711, "\u0120microbes": 33712, "\u0120overrun": 33713, "dds": 33714, "otent": 33715, "adiator": 33716, ">>>>>>>>": 33717, "\u0120tornado": 33718, "\u0120adjud": 33719, "\u0120intrigued": 33720, "\u0120si": 33721, "\u0120Revelation": 33722, "progress": 33723, "\u0120burglary": 33724, "\u0120Saiyan": 33725, "\u0120Kathy": 33726, "\u0120serpent": 33727, "\u0120Andreas": 33728, "\u0120compel": 33729, "essler": 33730, "\u0120Plastic": 33731, "\u0120Advent": 33732, "\u0120Positive": 33733, "\u0120Qt": 33734, "\u0120Hindus": 33735, "registered": 33736, "ularity": 33737, "\u0120righteousness": 33738, "\u0120demonic": 33739, "uitive": 33740, "\u0120BDS": 33741, "\u0120Gregg": 33742, "cia": 33743, "\u0120Crusade": 33744, "\u0120Sinai": 33745, "WARE": 33746, "+(": 33747, "\u0120mell": 33748, "\u0120derail": 33749, "yards": 33750, "Ast": 33751, "\u0120noticeably": 33752, "\u0120Ober": 33753, "Ram": 33754, "\u0120unnoticed": 33755, "\u0120seq": 33756, "avage": 33757, "Ts": 33758, "\u0120640": 33759, "\u0120concede": 33760, "\u0120])": 33761, "Fill": 33762, "\u0120captivity": 33763, "\u0120Improvement": 33764, "\u0120Crusader": 33765, "araoh": 33766, "MAP": 33767, "\u00e6\u0139": 33768, "\u0120stride": 33769, "always": 33770, "Fly": 33771, "Nit": 33772, "\u0120algae": 33773, "\u0120Cooking": 33774, "\u0120Doors": 33775, "Malley": 33776, "\u0120policemen": 33777, "\u00e3\u0123\u012f": 33778, "\u0120astronaut": 33779, "accessible": 33780, "495": 33781, "\u0120RAW": 33782, "cliffe": 33783, "udicrous": 33784, "\u0120depended": 33785, "alach": 33786, "\u0120ventures": 33787, "rake": 33788, "\u0120tits": 33789, "\u0120Hou": 33790, "\u0120condom": 33791, "ormonal": 33792, "\u0120indent": 33793, "\u0120uploading": 33794, "Footnote": 33795, "Important": 33796, "\u0120271": 33797, "\u0120mindful": 33798, "\u0120contends": 33799, "Cra": 33800, "\u0120calibr": 33801, "\u0120OECD": 33802, "plugin": 33803, "Fat": 33804, "\u0120ISS": 33805, "\u0120Dynamics": 33806, "ansen": 33807, "686": 33808, "'),": 33809, "\u0120sprite": 33810, "\u0120handheld": 33811, "\u0120Hipp": 33812, "=~=~": 33813, "Trust": 33814, "\u0120semantics": 33815, "\u0120Bundes": 33816, "\u0120Reno": 33817, "\u0120Literature": 33818, "sense": 33819, "Gary": 33820, "\u0120Aeg": 33821, "\u0120Trin": 33822, "EEK": 33823, "\u0120cleric": 33824, "\u0120SSH": 33825, "\u0120christ": 33826, "\u0120invading": 33827, "ibu": 33828, "\u0120enum": 33829, "aura": 33830, "\u0120allege": 33831, "\u0120Incredible": 33832, "BBC": 33833, "\u0120thru": 33834, "\u0120sailed": 33835, "\u0120emulate": 33836, "\u0120insecurity": 33837, "\u0120crou": 33838, "\u0120accommodations": 33839, "\u0120incompetent": 33840, "\u0120slips": 33841, "\u0120Earthqu": 33842, "sama": 33843, "ILLE": 33844, "\u0120iPhones": 33845, "asaki": 33846, "\u0120bye": 33847, "\u0120ard": 33848, "\u0120extras": 33849, "\u0120slaughtered": 33850, "\u0120crowdfunding": 33851, "resso": 33852, "\u0120filib": 33853, "\u0120ERROR": 33854, "\u0120TLS": 33855, "egg": 33856, "\u0120Ital": 33857, "\u0120enlist": 33858, "\u0120Catalonia": 33859, "\u0120Scots": 33860, "\u0120sergeant": 33861, "\u0120dissolve": 33862, "NH": 33863, "\u0120standings": 33864, "rique": 33865, "IQ": 33866, "\u0120beneficiary": 33867, "\u0120aquarium": 33868, "YouTube": 33869, "\u0120PowerShell": 33870, "\u0120brightest": 33871, "\u0120Warrant": 33872, "Sold": 33873, "Writing": 33874, "\u0120beginnings": 33875, "\u0120Reserved": 33876, "\u0120Latinos": 33877, "heading": 33878, "\u0120440": 33879, "\u0120rooftop": 33880, "ATING": 33881, "\u0120390": 33882, "VPN": 33883, "Gs": 33884, "kernel": 33885, "turned": 33886, "\u0120preferable": 33887, "\u0120turnovers": 33888, "\u0120Hels": 33889, "Sa": 33890, "\u0120Shinji": 33891, "veh": 33892, "\u0120MODULE": 33893, "Viol": 33894, "\u0120exiting": 33895, "\u0120jab": 33896, "\u0120Vanilla": 33897, "\u0120acron": 33898, "\u0120Gap": 33899, "bern": 33900, "Ak": 33901, "\u0120McGu": 33902, "\u0120endlessly": 33903, "\u0120Farage": 33904, "\u0120Noel": 33905, "Va": 33906, "MK": 33907, "\u0120brute": 33908, "\u0120Kru": 33909, "\u0120ESV": 33910, "\u0120Olivia": 33911, "\u00e2\u0122\u0142": 33912, "\u0120Kaf": 33913, "\u0120trusting": 33914, "\u0120hots": 33915, "324": 33916, "\u0120malaria": 33917, "\u0120json": 33918, "\u0120pounding": 33919, "ortment": 33920, "Country": 33921, "\u0120postponed": 33922, "\u0120unequiv": 33923, "?),": 33924, "\u0120Rooney": 33925, "udding": 33926, "\u0120Leap": 33927, "urrence": 33928, "shapeshifter": 33929, "\u0120HAS": 33930, "osate": 33931, "\u0120cavern": 33932, "\u0120conservatism": 33933, "\u0120BAD": 33934, "\u0120mileage": 33935, "\u0120arresting": 33936, "Vaults": 33937, "\u0120mixer": 33938, "Democratic": 33939, "\u0120Benson": 33940, "\u0120authored": 33941, "8000": 33942, "\u0120proactive": 33943, "\u0120Spiritual": 33944, "tre": 33945, "\u0120incarcerated": 33946, "\u0120Sort": 33947, "\u0120peaked": 33948, "\u0120wielding": 33949, "reciation": 33950, "\u00d7\u013b\u00d7": 33951, "Patch": 33952, "\u0120Emmy": 33953, "\u0120exqu": 33954, "tto": 33955, "\u0120Ratio": 33956, "\u0120Picks": 33957, "\u0120Gry": 33958, "phant": 33959, "\u0120fret": 33960, "\u0120ethn": 33961, "\u0120archived": 33962, "%-": 33963, "cases": 33964, "\u0120Blaze": 33965, "\u0120imb": 33966, "cv": 33967, "yss": 33968, "imony": 33969, "\u0120countdown": 33970, "\u0120awakening": 33971, "\u0120Tunisia": 33972, "\u0120Refer": 33973, "\u0120MJ": 33974, "\u0120unnatural": 33975, "\u0120Carnegie": 33976, "izen": 33977, "\u0120Nuggets": 33978, "hess": 33979, "\u0120evils": 33980, "647": 33981, "\u0120introductory": 33982, "loving": 33983, "\u0120McMahon": 33984, "\u0120ambiguity": 33985, "Label": 33986, "\u0120Almighty": 33987, "\u0120coloring": 33988, "\u0120Claus": 33989, "setting": 33990, "NULL": 33991, "\u0120Favorite": 33992, "\u0120SIG": 33993, ">(": 33994, "\u0120Shiva": 33995, "\u0120Mayer": 33996, "\u0120stormed": 33997, "\u0120Coverage": 33998, "weapons": 33999, "igham": 34000, "\u0120unanswered": 34001, "\u0120leve": 34002, "\u0120coy": 34003, "cas": 34004, "bags": 34005, "asured": 34006, "Seattle": 34007, "\u0120Santorum": 34008, "serious": 34009, "\u0120courageous": 34010, "\u0120Soup": 34011, "\u0120confiscated": 34012, "\u0120///": 34013, "\u0120unconventional": 34014, "\u0120moms": 34015, "\u0120Rohingya": 34016, "\u0120Orchestra": 34017, "\u0120Potion": 34018, "\u0120discredit": 34019, "\u0120FIL": 34020, "fixed": 34021, "\u0120Deer": 34022, "doi": 34023, "\u0120Dimension": 34024, "\u0120bureaucrats": 34025, "eteen": 34026, "\u0120actionGroup": 34027, "ohm": 34028, "\u0120bumps": 34029, "\u0120Utility": 34030, "\u0120submarines": 34031, "renheit": 34032, "research": 34033, "\u0120Shapiro": 34034, "\u0120sketches": 34035, "\u0120deceptive": 34036, "\u0120Vil": 34037, "esame": 34038, "\u0120Essentially": 34039, "\u0120rampage": 34040, "isky": 34041, "\u0120muttered": 34042, "thritis": 34043, "\u0120236": 34044, "fet": 34045, "bars": 34046, "\u0120pupil": 34047, "\u0120Thou": 34048, "oS": 34049, "song": 34050, "\u0120fractured": 34051, "\u0120revert": 34052, "picture": 34053, "\u0120criterion": 34054, "usher": 34055, "\u0120repercussions": 34056, "\u0120Vintage": 34057, "\u0120Superintendent": 34058, "Officers": 34059, "\u0120flagged": 34060, "\u0120blames": 34061, "\u0120inverse": 34062, "ographers": 34063, "\u0120makeshift": 34064, "\u0120devoid": 34065, "\u0120fossils": 34066, "\u0120Aristotle": 34067, "\u0120Funds": 34068, "\u0120depleted": 34069, "\u0120Flu": 34070, "\u0120Yuan": 34071, "\u0120woes": 34072, "\u0120lipid": 34073, "\u0120situ": 34074, "requisites": 34075, "\u0120furnish": 34076, "\u0120Samar": 34077, "\u0120shameful": 34078, "\u0120adversely": 34079, "\u0120adept": 34080, "\u0120remorse": 34081, "\u0120murderous": 34082, "uckles": 34083, "\u0120ESL": 34084, "\u0120314": 34085, "sent": 34086, "\u0120redef": 34087, "\u0120Cache": 34088, "\u0120Purs": 34089, "igans": 34090, "\u0120460": 34091, "\u0120prescriptions": 34092, "\u0120fres": 34093, "Fuck": 34094, "ocrates": 34095, "Twenty": 34096, "\u0120Weird": 34097, "\u0120Toggle": 34098, "\u0120Called": 34099, "itizens": 34100, "\u0120poultry": 34101, "\u0120harvesting": 34102, "\u00e3\u0124\u00a6\u00e3\u0124\u00b9": 34103, "Bottom": 34104, "\u0120cautioned": 34105, "tn": 34106, "396": 34107, "\u0120Nikki": 34108, "\u0120evaluations": 34109, "\u0120harassing": 34110, "\u0120bindings": 34111, "\u0120Monetary": 34112, "\u0120hitters": 34113, "\u0120adversary": 34114, "unts": 34115, "\u0120setback": 34116, "\u0120encrypt": 34117, "\u0120Cait": 34118, "\u0120lows": 34119, "enges": 34120, "\u0120Norn": 34121, "\u0120bulbs": 34122, "\u0120bottled": 34123, "\u0120Voyager": 34124, "317": 34125, "\u0120spheres": 34126, "politics": 34127, "\u0120subtract": 34128, "\u0120sensations": 34129, "\u0120appalling": 34130, "\u0120316": 34131, "\u0120environmentally": 34132, "\u0120STEM": 34133, "\u0120publishes": 34134, "560": 34135, "\u0120diligence": 34136, "484": 34137, "\u0120advises": 34138, "\u0120petrol": 34139, "\u0120imagining": 34140, "\u0120patrols": 34141, "\u0120Integer": 34142, "\u0120Ashes": 34143, "actus": 34144, "\u0120Radiant": 34145, "\u0120LT": 34146, "itability": 34147, "htaking": 34148, "Setting": 34149, "\u0120nuanced": 34150, "\u0120Reef": 34151, "\u0120Developers": 34152, "Ni": 34153, "pieces": 34154, "990": 34155, "License": 34156, "\u0120lowers": 34157, "\u0120Ottoman": 34158, "327": 34159, "ooo": 34160, "\u0120quitting": 34161, "markets": 34162, "Behind": 34163, "\u0120basin": 34164, "\u0120docs": 34165, "anie": 34166, "flash": 34167, "ctl": 34168, "\u0120civilized": 34169, "\u0120Fukushima": 34170, "\"],\"": 34171, "\u0120KS": 34172, "\u0120Honestly": 34173, "arat": 34174, "\u0120constructs": 34175, "\u0120Lans": 34176, "\u0120Dire": 34177, "\u0120LIKE": 34178, "\u0120Trouble": 34179, "\u0120withholding": 34180, "\u0120Oblivion": 34181, "\u0120sanity": 34182, "anya": 34183, "Const": 34184, "\u0120grocer": 34185, "\u0120Celsius": 34186, "\u0120recounted": 34187, "\u0120Wife": 34188, "Border": 34189, "atered": 34190, "happy": 34191, "\u0120spoiler": 34192, "\u0120logically": 34193, "Hall": 34194, "\u0120succeeding": 34195, "\u0120polymorph": 34196, "\u0120axes": 34197, "\u0120Shotgun": 34198, "\u0120Slim": 34199, "\u0120Principles": 34200, "\u0120Leth": 34201, "arta": 34202, "\u0120scor": 34203, "Screenshot": 34204, "\u0120relaxation": 34205, "#$#$": 34206, "\u0120deterrent": 34207, "iddy": 34208, "\u0120powerless": 34209, "\u0120lesbians": 34210, "\u0120chords": 34211, "\u0120Edited": 34212, "selected": 34213, "\u0120separatists": 34214, "0002": 34215, "\u0120airspace": 34216, "\u0120turnaround": 34217, "\u0120cunning": 34218, "PATH": 34219, "Poly": 34220, "\u0120bombed": 34221, "\u0120tion": 34222, "xs": 34223, "\u0120withhold": 34224, "\u0120waged": 34225, "\u0120Liberties": 34226, "Flag": 34227, "\u0120comforting": 34228, "454": 34229, "\u0120Iris": 34230, "arers": 34231, "\u0120rag": 34232, "\u0120relocated": 34233, "\u0120Guarant": 34234, "\u0120strategically": 34235, "\u0120gamma": 34236, "uberty": 34237, "\u0120Lockheed": 34238, "gres": 34239, "\u0120grilled": 34240, "\u0120Lowe": 34241, "stats": 34242, "\u0120Rocks": 34243, "\u0120sensing": 34244, "\u0120renting": 34245, "\u0120Geological": 34246, "\u00d8\u00a7\u00d8": 34247, "otrop": 34248, "\u0120sew": 34249, "\u0120improperly": 34250, "486": 34251, "\u0120\u00e2\u0138\u0142": 34252, "\u0120starving": 34253, "\u0120Bj": 34254, "Discussion": 34255, "328": 34256, "\u0120Combo": 34257, "\u0120Fixes": 34258, "NAT": 34259, "\u0120striving": 34260, "thora": 34261, "\u0120harvested": 34262, "\u0120Ping": 34263, "\u0120playful": 34264, "\u0120avenues": 34265, "\u0120occupational": 34266, "\u0120wakes": 34267, "\u0120Courier": 34268, "\u0120drummer": 34269, "\u0120Browser": 34270, "\u0120Houth": 34271, "itu": 34272, "\u0120apparel": 34273, "paste": 34274, "\u0120hunted": 34275, "\u0120Secondly": 34276, "lain": 34277, "XY": 34278, "\u0120PIN": 34279, "icons": 34280, "\u0120cocktails": 34281, "\u0120sizable": 34282, "\u0120hurdles": 34283, "estinal": 34284, "\u0120Recreation": 34285, "\u0120eco": 34286, "648": 34287, "\u0120Died": 34288, "mint": 34289, "\u0120fingerprints": 34290, "\u0120dispose": 34291, "\u0120Bosnia": 34292, "tsy": 34293, "2200": 34294, "\u0120inspected": 34295, "\u0120Fou": 34296, "\u0120fuss": 34297, "\u0120ambush": 34298, "\u0120Rak": 34299, "\u0120manifested": 34300, "Prosecut": 34301, "\u0120suffice": 34302, "rences": 34303, "\u0120compensated": 34304, "\u0120Cyrus": 34305, "\u0120genus": 34306, "\u0120Wolverine": 34307, "\u0120Trends": 34308, "\u0120hikes": 34309, "\u0120Seen": 34310, "\u0120enrol": 34311, "Cold": 34312, "\u0120politely": 34313, "\u0120Slav": 34314, "\u0120Rupert": 34315, "\u0120eyewitness": 34316, "\u0120Alto": 34317, "\u0120uncomp": 34318, "\u0120posterior": 34319, "Must": 34320, "\u0120Herz": 34321, "\u0120progressively": 34322, "\u0120234": 34323, "\u0120indifference": 34324, "\u0120Cunningham": 34325, "\u0120academia": 34326, "\u0120sewer": 34327, "\u0120astounding": 34328, "\u0120AES": 34329, "rather": 34330, "\u0120eldest": 34331, "\u0120climbs": 34332, "\u0120Adds": 34333, "\u0120outcry": 34334, "\u0120contag": 34335, "\u0120Houses": 34336, "\u0120pept": 34337, "\u0120Melania": 34338, "interested": 34339, "\u0120UCH": 34340, "\u0120Roots": 34341, "\u0120Hubbard": 34342, "\u0120TBD": 34343, "\u0120Romanian": 34344, "filename": 34345, "Stone": 34346, "\u0120Impl": 34347, "\u0120chromosome": 34348, "Cle": 34349, "dx": 34350, "\u0120scrambled": 34351, "\u0120Pt": 34352, "\u0120242": 34353, "OPLE": 34354, "\u0120tremendously": 34355, "Street": 34356, "\u0120craving": 34357, "\u0120bundled": 34358, "\u0120RG": 34359, "pipe": 34360, "\u0120injuring": 34361, "\u0120arcane": 34362, "Particip": 34363, "\u0120Heroic": 34364, "sty": 34365, "\u0120topping": 34366, "\u0120Tempest": 34367, "rentices": 34368, "bh": 34369, "\u0120paranoia": 34370, "\u0120Unicode": 34371, "\u0120egregious": 34372, "\u0120\\'": 34373, "\u0120Oswald": 34374, "\u0120gravel": 34375, "\u0120Simpsons": 34376, "\u0120bland": 34377, "\u0120Guantanamo": 34378, "Writer": 34379, "liners": 34380, "\u0120Dice": 34381, "JC": 34382, "\u0120parity": 34383, "\u0120sided": 34384, "\u0120237": 34385, "\u0120Pyrrha": 34386, "atters": 34387, "dk": 34388, "Fine": 34389, "compan": 34390, "\u0120formulated": 34391, "\u0120Idol": 34392, "ilers": 34393, "hemoth": 34394, "\u0120Fav": 34395, "\u0120intrusion": 34396, "\u0120carrots": 34397, "\u0120Layer": 34398, "\u0120Hacker": 34399, "\u0120----------------": 34400, "\u0120moderation": 34401, "\u00e9\u0123": 34402, "ococ": 34403, "\u0120characterize": 34404, "\u0120Teresa": 34405, "\u0120socioeconomic": 34406, "\u0120perk": 34407, "\u0120Participation": 34408, "training": 34409, "\u0120Paulo": 34410, "phys": 34411, "\u0120trustworthy": 34412, "\u0120embodied": 34413, "\u0120Merch": 34414, "currency": 34415, "\u0120Priority": 34416, "\u0120teasing": 34417, "\u0120absorbing": 34418, "\u0120unfinished": 34419, "\u0120Comparison": 34420, "\u0120disple": 34421, "writers": 34422, "\u0120professions": 34423, "\u0120Penguin": 34424, "\u0120angrily": 34425, "\u0120LINK": 34426, "688": 34427, "\u0120Correspond": 34428, "\u0120prevailed": 34429, "\u0120cartel": 34430, "lp": 34431, "asms": 34432, "\u0120Redemption": 34433, "\u0120Islamists": 34434, "effects": 34435, "dose": 34436, "\u0120Latter": 34437, "\u0120Halifax": 34438, "\u0120vas": 34439, "\u0120Topics": 34440, "\u0120Named": 34441, "advertising": 34442, "zza": 34443, "ICES": 34444, "\u0120retarded": 34445, "achable": 34446, "\u0120Puppet": 34447, "\u0120ItemLevel": 34448, "\u0120retract": 34449, "\u0120identifiable": 34450, "Aaron": 34451, "\u0120Buster": 34452, "sol": 34453, "helle": 34454, "assemb": 34455, "Hope": 34456, "ranged": 34457, "Ba": 34458, "\u0120Purch": 34459, "\u00e9\u0122": 34460, "\u0120Siri": 34461, "\u0120arrivals": 34462, "\u01201912": 34463, "\u0120shortened": 34464, "\u0120312": 34465, "\u0120discrepancy": 34466, "\u0120Temperature": 34467, "\u0120Walton": 34468, "\u0120kinderg": 34469, "polit": 34470, "\u0120remix": 34471, "\u0120connectors": 34472, "\u00e3\u0125\u013a\u00e3\u0125\u00a9": 34473, "\u0120Kazakhstan": 34474, "dominated": 34475, "\u0120sugars": 34476, "imble": 34477, "\u0120Panic": 34478, "\u0120Demand": 34479, "\u0120Colony": 34480, "onen": 34481, "\u0120MER": 34482, "775": 34483, "uria": 34484, "azaar": 34485, "\u0120Degree": 34486, "Pri": 34487, "\u0120sunshine": 34488, "\u0120251": 34489, "\u0120psychedelic": 34490, "\u0120digitally": 34491, "\u0120Braun": 34492, "\u0120shimmer": 34493, "\u0120shave": 34494, "\u0120Telesc": 34495, "\u0120Astral": 34496, "\u0120Venezuelan": 34497, "\u0120OG": 34498, "\u0120crawling": 34499, "Integ": 34500, "\u0120Feather": 34501, "\u0120unfolding": 34502, "\u0120appropriation": 34503, "\u0120\u00e8\u00a3\u0131\u00e8": 34504, "\u0120Mobility": 34505, "\u0120Ney": 34506, "-.": 34507, "bilt": 34508, "LIN": 34509, "\u0120Tube": 34510, "\u0120Conversely": 34511, "\u0120keyboards": 34512, "\u0120Cao": 34513, "\u0120overth": 34514, "\u0120laure": 34515, ">>\\": 34516, "\u0120Viper": 34517, "acha": 34518, "Offset": 34519, "\u0120Raleigh": 34520, "\u0120Jae": 34521, "Jordan": 34522, "jp": 34523, "\u0120totalitarian": 34524, "Connector": 34525, "\u0120observes": 34526, "\u0120Spartan": 34527, "\u0120Immediately": 34528, "\u0120Scal": 34529, "Cool": 34530, "\u0120taps": 34531, "\u0120roar": 34532, "Past": 34533, "\u0120chars": 34534, "\u0120Bender": 34535, "\u0120Sheldon": 34536, "\u0120painter": 34537, "\u0120beacon": 34538, "\u0120Creatures": 34539, "\u0120downturn": 34540, "\u0120hinder": 34541, "\u0120Andromeda": 34542, "\u00c3\u013d": 34543, "ccoli": 34544, "\u0120Fitness": 34545, "etrical": 34546, "\u0120utilizes": 34547, "\u0120senate": 34548, "\u0120ensemble": 34549, "\u0120cheers": 34550, "TW": 34551, "\u0120affluent": 34552, "kil": 34553, "rylic": 34554, "ordering": 34555, "Computer": 34556, "\u0120gruesome": 34557, "ostics": 34558, "\u0120Ubisoft": 34559, "\u0120Kelley": 34560, "\u0120wrench": 34561, "\u0120bourgeoisie": 34562, "IBLE": 34563, "\u0120Preston": 34564, "worn": 34565, "arist": 34566, "reating": 34567, "\u0120stained": 34568, "arine": 34569, "\u0120slime": 34570, "ENN": 34571, "\u0120chests": 34572, "\u0120groundwater": 34573, "annot": 34574, "\u0120Tray": 34575, "\u0120Locke": 34576, "\u0120CTR": 34577, "\u0120dudes": 34578, "\u0120External": 34579, "\u0120Decoder": 34580, "\u0120paramed": 34581, "\u0120Medline": 34582, "809": 34583, "\u0120Dinner": 34584, "rupal": 34585, "gz": 34586, "\u0120Gum": 34587, "\u0120Demo": 34588, "jee": 34589, "\u0120dh": 34590, "berman": 34591, "archs": 34592, "\u0120enqu": 34593, "\u0120Epstein": 34594, "\u0120devastation": 34595, "\u0120friendships": 34596, "\u0120Ard": 34597, "\u0120231": 34598, "\u0120Rubin": 34599, "\u0120Distance": 34600, "\u0120spurred": 34601, "\u0120dossier": 34602, "\u0120overlooking": 34603, "\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\": 34604, "Forest": 34605, "\u0120Comes": 34606, "\\\",": 34607, "\u0120Iranians": 34608, "\u0120fixtures": 34609, "Laughs": 34610, "\u0120curry": 34611, "\u0120Kingston": 34612, "\u0120squash": 34613, "\u0120catalogue": 34614, "\u0120abnormalities": 34615, "\u0120digestive": 34616, ".........": 34617, "\u0120subordinate": 34618, "ogly": 34619, "\u0120249": 34620, "Middle": 34621, "\u0120massac": 34622, "\u0120burgers": 34623, "\u0120downstairs": 34624, "\u01201931": 34625, "394": 34626, "\u0120VG": 34627, "\u0120lasers": 34628, "\u0120Sikh": 34629, "\u0120Alexa": 34630, "derived": 34631, "\u0120cyclist": 34632, "\u00e3\u0123\u00ae\u00e9\u0143\u0136": 34633, "oneliness": 34634, "!!!!!!!!": 34635, "\u0120buffs": 34636, "legate": 34637, "\u0120raping": 34638, "\u0120recommending": 34639, "rored": 34640, "\u0120multicultural": 34641, "unique": 34642, "\u0120businessmen": 34643, "\u0120uneasy": 34644, "\u0120MAP": 34645, "\u0120dispersed": 34646, "cipline": 34647, "Jess": 34648, "\u0120Kerala": 34649, "\u00e5\u00a7": 34650, "\u0120abstraction": 34651, "Surv": 34652, "Uh": 34653, "\u0120printers": 34654, "ija": 34655, "owder": 34656, "\u0120analogous": 34657, "\u0120ASP": 34658, "afer": 34659, "\u0120unfolded": 34660, "\u0120leveling": 34661, "\u0120breached": 34662, "\u0120Hearing": 34663, "\u0120nat": 34664, "\u0120translating": 34665, "critical": 34666, "\u0120antagonist": 34667, "\u0120Yesterday": 34668, "\u0120fuzzy": 34669, "wash": 34670, "mere": 34671, "\u0120bewild": 34672, "\u0120Mae": 34673, "Virgin": 34674, "phrase": 34675, "\u0120signaled": 34676, "\u0120HIGH": 34677, "\u0120protester": 34678, "\u0120garner": 34679, "unknown": 34680, "\u0120kay": 34681, "\u0120abducted": 34682, "\u0120stalking": 34683, "amn": 34684, "\u0120deserving": 34685, "\u0120Riv": 34686, "\u0120Jorge": 34687, "\u0120scratching": 34688, "\u0120Saving": 34689, "iping": 34690, "\u0120tease": 34691, "\u0120missionary": 34692, "\u0120Morrow": 34693, "TIME": 34694, "Present": 34695, "\u0120chemotherapy": 34696, "terness": 34697, "\u0120Homes": 34698, "\u0120Purdue": 34699, "\u0120staunch": 34700, "\u0120Whitney": 34701, "\u0120THERE": 34702, "\u00ce\u00bc": 34703, "iatus": 34704, "\u0120Ernest": 34705, "\u0120Deploy": 34706, "\u0120coveted": 34707, "FML": 34708, "\u0120Dialogue": 34709, "\u0120exited": 34710, "fruit": 34711, "\u0120nerd": 34712, "\":\"\",\"": 34713, "\u0120vivo": 34714, "ruly": 34715, "460": 34716, "\u0120Amen": 34717, "rehensible": 34718, "\u0120\u00e2\u013a": 34719, "DIR": 34720, "\u0120adherence": 34721, "\u0120chew": 34722, "\u0120Coke": 34723, "\u0120Sergei": 34724, "digital": 34725, "\u0120Neck": 34726, "gently": 34727, "enthal": 34728, "/)": 34729, "\u0120weary": 34730, "\u0120guise": 34731, "\u0120Concord": 34732, "\u0120Onion": 34733, "atcher": 34734, "\u0120binge": 34735, "\u0120Directive": 34736, "\u0120manned": 34737, "ansk": 34738, "\u0120illusions": 34739, "\u0120billionaires": 34740, "383": 34741, "olyn": 34742, "odynamic": 34743, "\u0120Wheat": 34744, "\u0120Alic": 34745, "\u0120coloured": 34746, "\u0120NAFTA": 34747, "abo": 34748, "\u0120macros": 34749, "independent": 34750, "sweet": 34751, "\u0120spac": 34752, "\u0120Kabul": 34753, "\u0120\u00c4": 34754, "eme": 34755, "\u0120dictated": 34756, "\u0120shouts": 34757, "={": 34758, "\u0120ripping": 34759, "\u0120Shay": 34760, "\u0120Cricket": 34761, "directed": 34762, "\u0120analysed": 34763, "\u0120WARRANT": 34764, "agons": 34765, "\u0120Blazers": 34766, "\u0120cheered": 34767, "\u0120arithmetic": 34768, "\u0120Tanz": 34769, "373": 34770, "\u0120Flags": 34771, "\u0120295": 34772, "\u0120witches": 34773, "\u0120Included": 34774, "\u0120Gained": 34775, "\u0120Blades": 34776, "Gam": 34777, "\u0120Samantha": 34778, "\u0120Atlantis": 34779, "\u0120Pratt": 34780, "\u0120spoiled": 34781, "\u0120IB": 34782, "\u0120Ramirez": 34783, "Probably": 34784, "rero": 34785, "\u0120Ng": 34786, "\u0120Warlock": 34787, "tp": 34788, "\u0120overhe": 34789, "\u0120administrations": 34790, "\u0120tint": 34791, "\u0120regiment": 34792, "\u0120pistols": 34793, "\u0120blankets": 34794, "\u0120epist": 34795, "\u0120bowls": 34796, "\u0120hydraulic": 34797, "\u0120dean": 34798, "\u0120jung": 34799, "\u0120ascend": 34800, "705": 34801, "\u0120Santiago": 34802, "\u00c3\u00ae": 34803, "\u0120unavoid": 34804, "\u0120Shaman": 34805, "reb": 34806, "\u0120stemming": 34807, "998": 34808, "\u0120MG": 34809, "sticks": 34810, "esthesia": 34811, "ERO": 34812, "\u0120morbid": 34813, "\u0120Grill": 34814, "\u0120Poe": 34815, "anyl": 34816, "\u0120deleting": 34817, "\u0120Surveillance": 34818, "\u0120directives": 34819, "\u0120iterations": 34820, "\u0120Rox": 34821, "\u0120Milky": 34822, "Father": 34823, "\u0120patented": 34824, "447": 34825, "\u0120precursor": 34826, "\u0120maiden": 34827, "\u0120Phen": 34828, "\u0120Vegan": 34829, "\u0120Patent": 34830, "Kelly": 34831, "Redditor": 34832, "\u0120nods": 34833, "\u0120ventilation": 34834, "\u0120Schwarz": 34835, "\u0120wizards": 34836, "\u0120ominous": 34837, "\u0120Heads": 34838, "\u0120BG": 34839, "\u0120lumber": 34840, "\u0120Spiel": 34841, "\u0120isEnabled": 34842, "\u0120ancestral": 34843, "\u0120Ships": 34844, "\u0120wrestler": 34845, "phi": 34846, "\u0120yuan": 34847, "\u0120Rebellion": 34848, "\u0120iceberg": 34849, "\u0120magically": 34850, "\u0120diversion": 34851, "arro": 34852, "ythm": 34853, "\u0120Riders": 34854, "\u0120Robbie": 34855, "\u0120Kara": 34856, "\u0120Maintenance": 34857, "\u0120Herb": 34858, "\u0120harms": 34859, "packed": 34860, "\u0120Feinstein": 34861, "\u0120marrying": 34862, "\u0120blending": 34863, "\u0120Rates": 34864, "\u01201880": 34865, "\u0120wrink": 34866, "\u0120Unch": 34867, "\u0120Torch": 34868, "described": 34869, "\u0120humanoid": 34870, "ilitating": 34871, "\u0120Conv": 34872, "\u0120Feld": 34873, "IGHTS": 34874, "\u0120whistleblower": 34875, "ortmund": 34876, "etsy": 34877, "arrett": 34878, "\u0120Mono": 34879, "\u0120Ike": 34880, "\u0120CNBC": 34881, "\u0120WAY": 34882, "\u0120MDMA": 34883, "\u0120Individuals": 34884, "\u0120supplemental": 34885, "\u0120powerhouse": 34886, "\u0120Stru": 34887, "Focus": 34888, "aphael": 34889, "\u0120Colleg": 34890, "atti": 34891, "ZA": 34892, "\u0120perenn": 34893, "\u0120Signature": 34894, "\u0120Rodney": 34895, "\u0120cubes": 34896, "iddled": 34897, "\u0120Dante": 34898, "\u0120INV": 34899, "ilingual": 34900, "\u0120Cth": 34901, "\u0120sofa": 34902, "\u0120intimidate": 34903, "\u0120Roe": 34904, "\u0120Diplom": 34905, "\u0120Countries": 34906, "ayson": 34907, "\u0120extradition": 34908, "\u0120disabling": 34909, "\u0120Cardiff": 34910, "\u0120memorandum": 34911, "\u0120Trace": 34912, "\u0120???": 34913, "sector": 34914, "\u0120Rouhani": 34915, "\u0120Yates": 34916, "\u0120Freeze": 34917, "\u0120bladder": 34918, "Motor": 34919, "\u0120Promise": 34920, "antasy": 34921, "\u0120foreseeable": 34922, "\u0120Cologne": 34923, "container": 34924, "\u0120Trees": 34925, "\u0120Gors": 34926, "\u0120Sinclair": 34927, "\u0120barring": 34928, "keye": 34929, "\u0120slashed": 34930, "\u0120Statistical": 34931, "\u00e9\u0129": 34932, "\u0120\u00e2\u0138\u00ba": 34933, "Allows": 34934, "\u0120humility": 34935, "\u0120drilled": 34936, "\u0120Furn": 34937, "443": 34938, "\u0120sewage": 34939, "\u0120homepage": 34940, "\u0120courtyard": 34941, "\u0120vile": 34942, "\u0120subsidiaries": 34943, "ajo": 34944, "directory": 34945, "\u0120ammon": 34946, "Vers": 34947, "charges": 34948, "\u0120}}": 34949, "\u0120Chains": 34950, "\u0120246": 34951, "nob": 34952, "\u0120percept": 34953, "\u0120grit": 34954, "\u0120fishermen": 34955, "\u0120Iraqis": 34956, "\u0120DISTR": 34957, "\u0120FULL": 34958, "\u0120Evaluation": 34959, "graph": 34960, "atial": 34961, "\u0120cooperating": 34962, "\u0120melan": 34963, "\u0120enlightened": 34964, "\u0120ali": 34965, "tailed": 34966, "\u0120salute": 34967, "\u0120weakest": 34968, "\u0120Bulldogs": 34969, "UA": 34970, "\u0120Alloy": 34971, "\u0120semen": 34972, "ocene": 34973, "\u0120Williamson": 34974, "spr": 34975, ",\u00e2\u0122\u0136": 34976, "\u0120GF": 34977, "ittens": 34978, "Beat": 34979, "\u0120Junk": 34980, "iphate": 34981, "\u0120Farmers": 34982, "\u0120Bitcoins": 34983, "igers": 34984, "dh": 34985, "\u0120Loyal": 34986, "payer": 34987, "\u0120entertained": 34988, "\u0120penned": 34989, "\u0120coupon": 34990, "Queue": 34991, "\u0120weakening": 34992, "carry": 34993, "\u0120underestimate": 34994, "\u0120shootout": 34995, "\u0120charismatic": 34996, "\u0120Procedure": 34997, "\u0120prudent": 34998, "inances": 34999, "\u0120riches": 35000, "\u0120cortical": 35001, "\u0120strides": 35002, "\u0120drib": 35003, "\u0120Oilers": 35004, "540": 35005, "\u0120Perform": 35006, "\u0120Bangkok": 35007, "\u0120euth": 35008, "SER": 35009, "\u0120simplistic": 35010, "tops": 35011, "campaign": 35012, "Quality": 35013, "\u0120impoverished": 35014, "\u0120Eisenhower": 35015, "\u0120augment": 35016, "\u0120Harden": 35017, "\u0120intervened": 35018, "\u0120listens": 35019, "\u0120Kok": 35020, "\u0120sage": 35021, "\u0120rubbish": 35022, "\u0120Ded": 35023, "\u0120mull": 35024, "pelling": 35025, "\u0120videot": 35026, "Production": 35027, "DJ": 35028, "miah": 35029, "\u0120adaptations": 35030, "\u0120medically": 35031, "\u0120boarded": 35032, "\u0120arrogance": 35033, "\u0120scrapped": 35034, "\u0120oppress": 35035, "FORMATION": 35036, "\u0120junction": 35037, "415": 35038, "EEEE": 35039, "Skill": 35040, "\u0120subdu": 35041, "\u0120Suggest": 35042, "\u0120Pett": 35043, "\u0120lett": 35044, "\u0120Manip": 35045, "\u0120Caf": 35046, "\u0120Cooperation": 35047, "Ther": 35048, "\u0120regained": 35049, "\u00b6\u00e6": 35050, "reflect": 35051, "\u0120thugs": 35052, "\u0120Shelby": 35053, "\u0120dictates": 35054, "\u0120Weiner": 35055, "\u0120Hale": 35056, "\u0120battleground": 35057, "schild": 35058, "\u0120condol": 35059, "hunt": 35060, "ositories": 35061, "\u0120accuses": 35062, "Filename": 35063, "\u0120shri": 35064, "\u0120motivate": 35065, "\u0120reflections": 35066, "Null": 35067, "\u0120Lobby": 35068, "\u00a5\u00b5": 35069, "\u0120SATA": 35070, "\u0120Backup": 35071, "\u00d1\u0125": 35072, "nin": 35073, "\u0120Correction": 35074, "\u0120juicy": 35075, "utra": 35076, "\u0120Pric": 35077, "\u0120restraining": 35078, "\u0120Airbnb": 35079, "\u0120Arrest": 35080, "\u0120appropriations": 35081, "\u0120slopes": 35082, "\u0120manslaughter": 35083, "\u0120workings": 35084, "\u0120Huss": 35085, "\u0120Frey": 35086, "Leave": 35087, "\u0120Harmony": 35088, "\u0120Feder": 35089, "\u0120430": 35090, "\u0120trench": 35091, "\u0120gladly": 35092, "\u0120bullpen": 35093, "\u0120Gau": 35094, "bones": 35095, "\u0120groove": 35096, "\u0120pretext": 35097, "\u00e3\u0127\u012d": 35098, "\u0120transmitter": 35099, "\u0120Component": 35100, "\u0120underage": 35101, "\u0120Empires": 35102, "Tile": 35103, "\u0120oy": 35104, "\u0120Marvin": 35105, "\u0120CAS": 35106, "\u0120bloss": 35107, "\u0120replicated": 35108, "\u0120Mariners": 35109, "Marcus": 35110, "\u0120Blocks": 35111, "\u0120liberated": 35112, "\u0120butterfly": 35113, "Feel": 35114, "\u0120fermentation": 35115, "\u0120youtube": 35116, "\u0120offend": 35117, "\u0120Term": 35118, "resist": 35119, "\u0120cessation": 35120, "\u0120insurgency": 35121, "\u0120bir": 35122, "\u0120Raise": 35123, "595": 35124, "\u0120hypotheses": 35125, "502": 35126, "\u0120plaque": 35127, "ocrat": 35128, "\u0120jackets": 35129, "\u0120HuffPost": 35130, "among": 35131, "\u0120confer": 35132, "487": 35133, "\u0120Lilly": 35134, "\u0120adapting": 35135, "\u0120Fay": 35136, "\u0120shoved": 35137, "vec": 35138, "\u0120refine": 35139, "\u0120gon": 35140, "\u0120gunmen": 35141, "zai": 35142, "\u0120Shuttle": 35143, "\u0120Izan": 35144, "\u01201913": 35145, "\u0120plethora": 35146, "\u00c2\u00b7\u00c2\u00b7": 35147, "\u0120510": 35148, "\u0120puberty": 35149, "\u0120241": 35150, "\u0120Wealth": 35151, "\u0120Alma": 35152, "\u0120MEM": 35153, "\u0120Adults": 35154, "Cas": 35155, "prison": 35156, "Race": 35157, "\u0120waterproof": 35158, "\u0120athleticism": 35159, "\u0120capitalize": 35160, "\u0120Juice": 35161, "\u0120illuminated": 35162, "\u0120Pascal": 35163, "\u0120irritation": 35164, "\u0120Witnesses": 35165, "adle": 35166, "\u0120Astro": 35167, "\u0120fax": 35168, "\u0120Elvis": 35169, "Primary": 35170, "\u0120Lich": 35171, "\u0120Elves": 35172, "\u0120residing": 35173, "\u0120stumble": 35174, "319": 35175, "\u0120PKK": 35176, "\u0120adversaries": 35177, "DOS": 35178, "\u0120Ritual": 35179, "\u0120smear": 35180, "\u0120arson": 35181, "idental": 35182, "\u0120scant": 35183, "\u0120monarchy": 35184, "\u0120halftime": 35185, "\u0120residue": 35186, "\u0120indign": 35187, "\u0120Shaun": 35188, "\u0120Elm": 35189, "auri": 35190, "Aff": 35191, "WATCH": 35192, "\u0120Lyon": 35193, "helps": 35194, "361": 35195, "\u0120lobbyist": 35196, "\u0120diminishing": 35197, "\u0120outbreaks": 35198, "\u0120goats": 35199, "favorite": 35200, "\u0120Nah": 35201, "sonian": 35202, "\u0120Booster": 35203, "\u0120sandbox": 35204, "\u0120Fare": 35205, "\u0120Malta": 35206, "\u0120attRot": 35207, "\u0120MOR": 35208, "lde": 35209, "\u0120navigating": 35210, "Touch": 35211, "\u0120untrue": 35212, "\u0120Disaster": 35213, "\u0120ludicrous": 35214, "Password": 35215, "\u0120JFK": 35216, "blogspot": 35217, "416": 35218, "\u0120UNDER": 35219, "ernal": 35220, "\u0120delaying": 35221, "TOP": 35222, "\u0120implants": 35223, "\u0120AVG": 35224, "\u0120Huge": 35225, "attr": 35226, "\u0120journalistic": 35227, "\u0120Peyton": 35228, "\u0120IA": 35229, "Rap": 35230, "goal": 35231, "\u0120Programme": 35232, "\u0120smashing": 35233, "wives": 35234, "println": 35235, "\u0120Plague": 35236, "inus": 35237, "EEP": 35238, "\u0120cruiser": 35239, "\u0120Parish": 35240, "uminium": 35241, "\u0120occupants": 35242, "\u0120Jihad": 35243, "mop": 35244, "\u0120pint": 35245, "\u0120hect": 35246, "\u0120Mecca": 35247, "director": 35248, "\u0120Funding": 35249, "\u0120Mixed": 35250, "\u0120stag": 35251, "Tier": 35252, "\u0120gust": 35253, "\u0120brightly": 35254, "orsi": 35255, "\u0120uphill": 35256, "RD": 35257, "\u0120lesions": 35258, "\u0120Bundy": 35259, "livious": 35260, "\u0120biologist": 35261, "\u0120Faculty": 35262, "\u0120Authorization": 35263, "\u0120244": 35264, "Allow": 35265, "\u00ef\u00b8": 35266, "\u0120Giul": 35267, "\u0120pertinent": 35268, "otaur": 35269, "esse": 35270, "\u0120Roof": 35271, "\u0120unmanned": 35272, "351": 35273, "\u0120Shak": 35274, "\u0120Orient": 35275, "\u0120endanger": 35276, "Dir": 35277, "\u0120replen": 35278, "edient": 35279, "\u0120tailor": 35280, "\u0120gadgets": 35281, "\u0120audible": 35282, "\u00e2\u013a\u0128": 35283, "Nice": 35284, "\u0120bombard": 35285, "\u0120Rape": 35286, "\u0120defiance": 35287, "\u0120TWO": 35288, "\u0120Filipino": 35289, "\u0120unaffected": 35290, "ervatives": 35291, "\u0120soared": 35292, "\u0120Bolton": 35293, "\u0120compromising": 35294, "\u0120Brewers": 35295, "RAL": 35296, "\u0120AHL": 35297, "icycle": 35298, "\u0120vampires": 35299, "\u0120dipped": 35300, "oyer": 35301, "\u0120XIII": 35302, "\u0120sideways": 35303, "\u0120Waste": 35304, "\u0120Diss": 35305, "\u0120\u00e2\u0136\u013e\u00e2\u0136\u0122\u00e2\u0136\u0122": 35306, "$.": 35307, "\u0120habitats": 35308, "\u0120Beef": 35309, "truth": 35310, "trained": 35311, "split": 35312, "Rus": 35313, "Andy": 35314, "\u0120Bram": 35315, "REP": 35316, "pid": 35317, "\u00e8\u00a3\u0127": 35318, "\u0120Mutant": 35319, "Anim": 35320, "\u0120Marina": 35321, "\u0120futile": 35322, "highest": 35323, "frequency": 35324, "\u0120epilepsy": 35325, "\u0120coping": 35326, "\u0120concise": 35327, "\u0120tracing": 35328, "\u0120SUN": 35329, "panel": 35330, "\u0120Sophie": 35331, "\u0120Crowley": 35332, "\u0120Adolf": 35333, "\u0120Shooter": 35334, "\u0120shaky": 35335, "\u0120IG": 35336, "\u0120Lies": 35337, "\u0120Barber": 35338, "pkg": 35339, "\u0120uptake": 35340, "\u0120predatory": 35341, "ULTS": 35342, "/**": 35343, "\u0120intoxicated": 35344, "\u0120Westbrook": 35345, "odder": 35346, "hement": 35347, "\u0120baseman": 35348, "APD": 35349, "storage": 35350, "\u0120Fifty": 35351, "editor": 35352, "GEN": 35353, "UTION": 35354, "irting": 35355, "\u0120sewing": 35356, "rift": 35357, "\u0120agony": 35358, "\u0120Sands": 35359, "\u0120254": 35360, "Cash": 35361, "\u0120lodge": 35362, "\u0120punt": 35363, "Natural": 35364, "\u0120Ideas": 35365, "\u0120erroneous": 35366, "\u0120Sensor": 35367, "\u0120Hannity": 35368, "\u01201921": 35369, "\u0120mould": 35370, "\u0120Gon": 35371, "kaya": 35372, "\u0120anonymously": 35373, "\u0120KEY": 35374, "\u0120simulator": 35375, "Winter": 35376, "\u0120streamed": 35377, "507": 35378, "?\",": 35379, "\u0120teased": 35380, "\u0120coefficient": 35381, "\u0120wartime": 35382, "\u0120THR": 35383, "''.": 35384, "\u0120Banking": 35385, "mpire": 35386, "\u0120fandom": 35387, "\u0120lia": 35388, "Ga": 35389, "\u0120downhill": 35390, "\u0120interpreting": 35391, "Individual": 35392, "Norm": 35393, "\u0120jealousy": 35394, "bitcoin": 35395, "\u0120pleasures": 35396, "\u0120Toys": 35397, "\u0120Chevrolet": 35398, "\u0120Advisor": 35399, "IZE": 35400, "\u0120receptions": 35401, "706": 35402, "Cro": 35403, "\u0120262": 35404, "\u0120citrus": 35405, "iru": 35406, "Reviewer": 35407, "jected": 35408, "UES": 35409, "anz": 35410, "1981": 35411, "\u0120Worker": 35412, "\u0120complied": 35413, "orescent": 35414, "continental": 35415, "Ton": 35416, "\u0120Prism": 35417, "\u0120Sheep": 35418, "\u0120288": 35419, "nox": 35420, "\u0120Vog": 35421, "Ord": 35422, "\u0120realms": 35423, "tek": 35424, "\u0120irrigation": 35425, "\u0120bicycles": 35426, "\u0120electronically": 35427, "poly": 35428, "tall": 35429, "());": 35430, "\u0120aesthetics": 35431, "\u0120Integrated": 35432, "Explore": 35433, "\u0120dunk": 35434, "476": 35435, "pain": 35436, "\u0120Jacques": 35437, "\u0120Dmit": 35438, "Frames": 35439, "\u0120reunited": 35440, "\u0120humid": 35441, "Dro": 35442, "Political": 35443, "\u0120youthful": 35444, "\u0120entails": 35445, "\u0120mosquito": 35446, "363": 35447, "species": 35448, "\u0120coordinating": 35449, "\u0120Mayhem": 35450, "\u0120Magnus": 35451, "Mount": 35452, "Improved": 35453, "\u0120STATE": 35454, "ATTLE": 35455, "\u0120flowed": 35456, "\u0120tackled": 35457, "\u0120fashioned": 35458, "\u0120reorgan": 35459, "ivari": 35460, "finger": 35461, "\u0120reluctantly": 35462, "etting": 35463, "\u0120Vand": 35464, "young": 35465, "\u0120Garland": 35466, "\u0120presumption": 35467, "\u0120amenities": 35468, "\u0120Pleasant": 35469, "onential": 35470, "\u0120Oxy": 35471, "\u0120morals": 35472, "\u0120Yah": 35473, "Ready": 35474, "Simon": 35475, "Enh": 35476, "Demon": 35477, "\u0120clich": 35478, "Monitor": 35479, "\u0120DU": 35480, "\u0120welcomes": 35481, "\u0120standout": 35482, "\u0120dreadful": 35483, "\u0120bananas": 35484, "\u0120balloons": 35485, "hooting": 35486, "basic": 35487, "\u0120suffix": 35488, "\u0120duly": 35489, "cano": 35490, "Chain": 35491, "atos": 35492, "\u0120geopolitical": 35493, "\u0120(&": 35494, "\u0120Gemini": 35495, "\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124\u00c3\u0125\u00c3\u0124": 35496, "\u0120acquitted": 35497, "Luck": 35498, "protect": 35499, "1024": 35500, "\u0120scarcity": 35501, "\u0120mindfulness": 35502, "ecided": 35503, "DN": 35504, "prime": 35505, "\u0120Presidents": 35506, "\u0120VIDEO": 35507, "\u0120(\u00e2\u012a\u0134": 35508, "addock": 35509, "NOR": 35510, "\u0120Pru": 35511, "pun": 35512, "\u0120LOL": 35513, "))))": 35514, "\u0120Liqu": 35515, "\u0120SAS": 35516, "\u0120styling": 35517, "\u0120punishments": 35518, "\u0120numb": 35519, "\u0120ascertain": 35520, "\u0120Rockies": 35521, "flu": 35522, "Thumbnail": 35523, "\u0120perpetrated": 35524, "\u0120Semi": 35525, "\u0120disarm": 35526, "\u0120Older": 35527, "\u0120Exception": 35528, "\u0120exponentially": 35529, "\u0120Communities": 35530, "\u0120abolish": 35531, "\u0120Partner": 35532, "ptoms": 35533, "\u0120777": 35534, "\u0120Foley": 35535, "\u0120Cases": 35536, "\u0120grease": 35537, "\u0120Rebirth": 35538, "Ground": 35539, "\u0120;)": 35540, "\u0120Doctrine": 35541, "ikini": 35542, "Ye": 35543, "\u0120Blossom": 35544, "\u0120persists": 35545, "bill": 35546, "\u0120infusion": 35547, "\u0120buddies": 35548, "911": 35549, "\u0120Patient": 35550, "\u0120demos": 35551, "\u0120acquaintance": 35552, "\u0120Paw": 35553, "atari": 35554, "\u0120xml": 35555, "\u0120fascination": 35556, "\u0120Serve": 35557, "\u00cf\u0124": 35558, "branded": 35559, "\u0120az": 35560, "Returns": 35561, "\u0120overshadow": 35562, "\u0120roam": 35563, "\u0120speedy": 35564, "numbered": 35565, "helial": 35566, "\u0120disciple": 35567, "\u0120assurances": 35568, "given": 35569, "pecting": 35570, "\u0120Natalie": 35571, "\u00e7\u0136\u00b0": 35572, "\u0120mosquitoes": 35573, "rotein": 35574, "\u0120numeric": 35575, "\u0120independents": 35576, "\u0120transitional": 35577, "\u0120reactionary": 35578, "\u0120Mechdragon": 35579, "doctor": 35580, "\u0120shortest": 35581, "\u0120sequential": 35582, "\u0120Bac": 35583, "\u0120Accounts": 35584, "\u00e3\u0123\u012e": 35585, "achy": 35586, "ractive": 35587, "\u0120Regiment": 35588, "\u0120breathtaking": 35589, "fficiency": 35590, "\u0120Bates": 35591, "\u0120311": 35592, "\u0120wardrobe": 35593, "fts": 35594, "\u0120Berk": 35595, "Simply": 35596, "\u0120Riverside": 35597, "ivering": 35598, "idential": 35599, "lucent": 35600, "\u0120enriched": 35601, "\u0120Conver": 35602, "\u0120Giving": 35603, "\u00e3\u0125\u013b": 35604, "\u0120legalize": 35605, "\u0120FTC": 35606, "\u0120freaking": 35607, "Mix": 35608, "\u0120terrestrial": 35609, "esian": 35610, "cients": 35611, "Wing": 35612, "LOAD": 35613, "\u0120ledge": 35614, "\u0120Violent": 35615, "\u0120Metall": 35616, "\u0120308": 35617, "\u0120southeastern": 35618, "hetto": 35619, "Meat": 35620, "\u0120slowdown": 35621, "\u0120retreated": 35622, "Jeremy": 35623, "endas": 35624, "*****": 35625, "eric": 35626, "\u0120reins": 35627, "oppable": 35628, "\u0120Humanity": 35629, "earances": 35630, "rigan": 35631, "Camera": 35632, "\u0120waivers": 35633, "soc": 35634, "\u0120alteration": 35635, "transform": 35636, "\u0120Cemetery": 35637, "506": 35638, "\u0120indefinite": 35639, "\u0120stimulating": 35640, "yg": 35641, "603": 35642, "\u0120Sop": 35643, "\u0120descriptive": 35644, "Phase": 35645, "\u0120Edmund": 35646, "\u0120pneumonia": 35647, "ventus": 35648, "Amb": 35649, "\u0120laboratories": 35650, "\u0120Exclusive": 35651, "ugar": 35652, "Were": 35653, "\u0120malfunction": 35654, "\u0120homosexuals": 35655, "\u0120-------": 35656, "uni": 35657, "\u0120turbines": 35658, "\u0120Equity": 35659, "Du": 35660, "\u0120minded": 35661, "\u0120RH": 35662, "\u0120Blackhawks": 35663, "\u0120feats": 35664, "\u01201700": 35665, "repl": 35666, "362": 35667, "laden": 35668, "\u0120indispensable": 35669, "lyss": 35670, "tti": 35671, "\u0120reel": 35672, "\u0120diverted": 35673, "\u0120likeness": 35674, "\u0120subscriptions": 35675, "\u0120fingert": 35676, "\u0120filthy": 35677, "destruct": 35678, "draft": 35679, "\u0120Bernardino": 35680, "launch": 35681, "\u0120perplex": 35682, "\u0120SUM": 35683, "carb": 35684, "\u0120sweater": 35685, "\u0120Venture": 35686, "\u0120Jag": 35687, "\u0120Celeb": 35688, "\u0120Voters": 35689, "\u0120steadfast": 35690, "\u0120athletics": 35691, "\u0120Hanson": 35692, "\u0120Drac": 35693, "Tracker": 35694, "\u0120commend": 35695, "\u0120Presidency": 35696, "\u0120DID": 35697, "informed": 35698, "\u0120webpage": 35699, "Pretty": 35700, "\u0120forcefully": 35701, "\u00e3\u0125\u0125\u00e3\u0124\u00af": 35702, "\u0120relocation": 35703, "\u0120satire": 35704, "\u00e2\u012b": 35705, "\u0120Sunderland": 35706, "\u00e6\u0126": 35707, "Voice": 35708, "????????": 35709, "\u0120informant": 35710, "\u0120bowel": 35711, "\u0120Uniform": 35712, "\u0120...\"": 35713, "\u0120purge": 35714, "\u0120picnic": 35715, "\u0120Umb": 35716, "\u0120UPDATE": 35717, "\u0120Sapphire": 35718, "\u0120Stall": 35719, "learn": 35720, "\u0120objectively": 35721, "\u0120obliter": 35722, "\u0120loophole": 35723, "\u0120journeys": 35724, "\u0120omission": 35725, "Pros": 35726, "\u0120Sidney": 35727, "ploma": 35728, "\u0120sprayed": 35729, "\u0120guru": 35730, "\u0120traitor": 35731, "\u0120timet": 35732, "\u0120snapping": 35733, "\u0120Sevent": 35734, "urnal": 35735, "\u0120Ukip": 35736, "\u0120bowed": 35737, "poral": 35738, "liberal": 35739, "Ros": 35740, "Questions": 35741, "iOS": 35742, "\u0120summarize": 35743, "STAT": 35744, "\u01201850": 35745, "apest": 35746, "\u0120lender": 35747, "\u0120Variable": 35748, "bringing": 35749, "\u0120LORD": 35750, ",)": 35751, "\u0120collapses": 35752, "xiety": 35753, "\u0120Ned": 35754, "YD": 35755, "\u0120Scha": 35756, "\u0120antibody": 35757, "\u0120disband": 35758, "yre": 35759, "illusion": 35760, "\u0120rover": 35761, "shed": 35762, "\u0120Hirosh": 35763, "cci": 35764, "\u0120calam": 35765, "\u0120Morton": 35766, "Pinterest": 35767, "\u01201928": 35768, "\u0120Euras": 35769, "ordes": 35770, "\u0120fences": 35771, "\u0120Inventory": 35772, "\u0120Valencia": 35773, "\u0120Ud": 35774, "\u0120Tiff": 35775, "\u0120sque": 35776, "\u0120quotation": 35777, "\u0120troublesome": 35778, "erker": 35779, "QUEST": 35780, "\u0120Kingdoms": 35781, "south": 35782, "\u0120levy": 35783, "Prince": 35784, "\u0120Sting": 35785, "\u0120nicknamed": 35786, "\u0120appe": 35787, "\u0120photographic": 35788, "\u0120corpus": 35789, "reference": 35790, "\u0120Trog": 35791, "Unt": 35792, ")=(": 35793, "\u0120Latvia": 35794, "\u0120activating": 35795, "\u0120licensee": 35796, "\u0120disparities": 35797, "\u0120Newsletter": 35798, "\u00e3\u0125\u0125\u00e3\u0125\u012a": 35799, "\u0120freeing": 35800, "\u0120Jeep": 35801, "\u0120Perception": 35802, "insk": 35803, "\u0120silicone": 35804, "\u0120Hayden": 35805, "Lean": 35806, "\u0120Suzuki": 35807, "ibrarian": 35808, "668": 35809, "\u0120spor": 35810, "\u0120correlations": 35811, "aghetti": 35812, "\u0120tuber": 35813, "\u0120IPCC": 35814, "ilus": 35815, "\u0120Vu": 35816, "\u0120wealthiest": 35817, "\u0120Carbuncle": 35818, "anza": 35819, "\u0120fooled": 35820, "\u0120Zur": 35821, "\u0120daddy": 35822, "rano": 35823, "ilian": 35824, "\u0120knockout": 35825, "fman": 35826, "required": 35827, "\u0120Wikileaks": 35828, "\u0120Duffy": 35829, "ONT": 35830, "\u0120insol": 35831, "\u0120Objects": 35832, "\u0120bou": 35833, "\u0120Nordic": 35834, "\u0120Insert": 35835, "scan": 35836, "\u0120dancers": 35837, "\u0120idiots": 35838, "majority": 35839, "\u0120Neville": 35840, "\u0120FreeBSD": 35841, "\u0120tart": 35842, "panic": 35843, "690": 35844, "\u0120cocoa": 35845, "\u0120sampled": 35846, "\u0120lookup": 35847, "Indust": 35848, "\u0120injections": 35849, "genre": 35850, "\u0120au": 35851, "\u0120roadway": 35852, "\u0120genitals": 35853, "Kind": 35854, "\u0120Examiner": 35855, "\u0120Yaz": 35856, "Fresh": 35857, "\u0120paralysis": 35858, "\u0120Aluminum": 35859, "\u0120reap": 35860, "ok\u00c3\u00a9": 35861, "\u0120sloppy": 35862, "\u0120Tunnel": 35863, "posium": 35864, "nery": 35865, "enic": 35866, "\u0120herbal": 35867, "\u0120Outer": 35868, "\u0120Builder": 35869, "\u0120incur": 35870, "\u0120ideologies": 35871, "\u0120backups": 35872, "consuming": 35873, "\u0120Detect": 35874, "deck": 35875, "\u0120KNOW": 35876, "\u0120Gret": 35877, "\u0120MIC": 35878, "\u0120toughness": 35879, "\u0120Exhibit": 35880, "\u0120hive": 35881, "Les": 35882, "\u0120SCHOOL": 35883, "\u0120Atari": 35884, "alde": 35885, "\u0120Null": 35886, "andestine": 35887, "mouse": 35888, "\u0120brigade": 35889, "489": 35890, "\u0120revol": 35891, "\u0120Lawson": 35892, "\u0120Wah": 35893, "opoly": 35894, "ebted": 35895, "\u0120Saunders": 35896, "\u0120313": 35897, "\u0120Winc": 35898, "\u0120taboo": 35899, "\u0120Helmet": 35900, "\u0120wedge": 35901, "chip": 35902, "\u0120Tina": 35903, "bg": 35904, "\u0120infuri": 35905, "rn": 35906, "\u0120anomalies": 35907, "\u0120Sync": 35908, "\u0120Exam": 35909, "\u0120Commit": 35910, "\u0120Diary": 35911, "\u0120ALSO": 35912, "\u0120Debor": 35913, "omedical": 35914, "\u0120comprehension": 35915, "655": 35916, "\u0120empowering": 35917, "\u0120ire": 35918, "\u0120juices": 35919, "\u0120ETH": 35920, "\u0120Boxing": 35921, "=\"/": 35922, "\u0120facilitated": 35923, "poke": 35924, "\u0120Parsons": 35925, "\u0120Moder": 35926, "travel": 35927, "\u0120civilizations": 35928, "\u0120libertarians": 35929, "\u0120rune": 35930, "\u0120Clarks": 35931, "athed": 35932, "\u0120campaigners": 35933, "\u0120Dispatch": 35934, "\u0120Fahrenheit": 35935, "\u0120Capcom": 35936, "----------": 35937, "\u0120lace": 35938, "\u0120draining": 35939, "\u0120liner": 35940, "\u0120Artificial": 35941, "\u00c3\u00a9n": 35942, "task": 35943, "]).": 35944, "\u0120GMO": 35945, "\u0120Operator": 35946, "ordinary": 35947, "\u0120Influence": 35948, "\u0120Ups": 35949, "\u0120potency": 35950, "ussen": 35951, "ospons": 35952, "\u0120Swim": 35953, "\u0120Deadline": 35954, "Unity": 35955, "\u0120culinary": 35956, "\u0120enlightenment": 35957, "\u0120wearer": 35958, "\u0120mined": 35959, "\u0120ply": 35960, "\u0120incest": 35961, "\u0120DVDs": 35962, "Walk": 35963, "BTC": 35964, "Trade": 35965, "\u0120deval": 35966, "iband": 35967, "\u0120Oversight": 35968, "Palestinian": 35969, "\u0120dart": 35970, "\u0120mul": 35971, "LR": 35972, "\u0120removable": 35973, "\u0120Realms": 35974, "\u00ec\u013f": 35975, "\u0120miscar": 35976, "\u0120Vulkan": 35977, "685": 35978, "\u00c3\u00a8re": 35979, "\u0120Sap": 35980, "\u0120merging": 35981, "\u0120Carly": 35982, "chester": 35983, "\u0120brisk": 35984, "\u0120luxurious": 35985, "\u0120Generator": 35986, "\u0120bitterness": 35987, "\u0120edible": 35988, "\u0120243": 35989, "TG": 35990, "\u0120rectangle": 35991, "WithNo": 35992, "below": 35993, "Jenn": 35994, "\u0120darkest": 35995, "\u0120hitch": 35996, "\u0120dosage": 35997, "\u0120scaven": 35998, "\u0120Keller": 35999, "\u0120Illustrated": 36000, "Certainly": 36001, "\u0120Mavericks": 36002, "Marginal": 36003, "\u0120diarrhea": 36004, "\u0120enormously": 36005, "\u0120999": 36006, "shr": 36007, "quart": 36008, "\u0120adamant": 36009, "\u0120Mew": 36010, "\u0120renovation": 36011, "\u0120cervical": 36012, "\u0120Percentage": 36013, "eners": 36014, "\u0120Kimber": 36015, "\u0120floats": 36016, "\u0120dex": 36017, "\u0120Witcher": 36018, "\u0120Swansea": 36019, "dm": 36020, "\u0120salty": 36021, "yellow": 36022, "\u0120cape": 36023, "\u0120Drain": 36024, "\u0120Paula": 36025, "\u0120Toledo": 36026, "lesi": 36027, "Magazine": 36028, "\u0120Wick": 36029, "\u0120Mn": 36030, "\u0120Ack": 36031, "\u0120Riding": 36032, "ASON": 36033, "\u0120homophobic": 36034, "ARP": 36035, "\u0120wandered": 36036, "CPU": 36037, "oodoo": 36038, "\u0120Pipe": 36039, "\u0120tightening": 36040, "\u0120Butt": 36041, "318": 36042, "\u0120deserted": 36043, "Session": 36044, "\u0120facilitating": 36045, "Jump": 36046, "\u0120emergencies": 36047, "OWER": 36048, "\u0120exhaustive": 36049, "\u0120AFTER": 36050, "\u0120heartbeat": 36051, "\u0120Label": 36052, "acky": 36053, "\u0120Certified": 36054, "iltration": 36055, "Ze": 36056, "\u0120Utt": 36057, "\u01201300": 36058, "\u0120presume": 36059, "\u0120Disp": 36060, "\u0120surged": 36061, "\u0120dolls": 36062, "Columb": 36063, "\u0120chimpan": 36064, "\u0120Razor": 36065, "\u0120ticks": 36066, "\u0120councillor": 36067, "\u0120pilgrimage": 36068, "\u0120Rebels": 36069, "\u0120QC": 36070, "\u0120Auction": 36071, "xia": 36072, "ikk": 36073, "bred": 36074, "\u0120insertion": 36075, "\u0120coarse": 36076, "dB": 36077, "SEE": 36078, "\u0120Zap": 36079, "\u0120Foo": 36080, "\u0120contempor": 36081, "\u0120Quarterly": 36082, "otions": 36083, "\u0120Alchemist": 36084, "\u0120Trey": 36085, "\u0120Duo": 36086, "Sweet": 36087, "804": 36088, "\u0120Giov": 36089, "\u0120funn": 36090, "Nin": 36091, "hoff": 36092, "\u0120ramifications": 36093, "\u01201922": 36094, "\u0120Experts": 36095, "azes": 36096, "\u0120garments": 36097, "arial": 36098, "\u0120Nab": 36099, "\u0120257": 36100, "\u0120Ved": 36101, "\u0120humorous": 36102, "\u0120Pompe": 36103, "\u0120nylon": 36104, "\u0120lurking": 36105, "\u0120Sergey": 36106, "\u0120Mattis": 36107, "\u0120misogyny": 36108, "\u0120Components": 36109, "\u0120Watching": 36110, "\u0120Folk": 36111, "ractical": 36112, "Bush": 36113, "\u0120taped": 36114, "\u0120grouping": 36115, "\u0120beads": 36116, "\u01202048": 36117, "\u0120condu": 36118, "querque": 36119, "Reading": 36120, "\u0120grievances": 36121, "Ultra": 36122, "\u0120endpoint": 36123, "Hig": 36124, "\u0120Static": 36125, "\u0120Scarborough": 36126, "Lua": 36127, "\u0120Messi": 36128, "aqu": 36129, "\u0120PsyNet": 36130, "\u0120Rudd": 36131, "\u0120avenue": 36132, "vp": 36133, "Jer": 36134, "\u0120shady": 36135, "\u0120Resist": 36136, "\u0120Artemis": 36137, "\u0120careless": 36138, "\u0120brokers": 36139, "\u0120temperament": 36140, "\u0120520": 36141, "Tags": 36142, "\u0120Turning": 36143, "\u0120uttered": 36144, "\u0120pedd": 36145, "\u0120improvised": 36146, "\u0120:(": 36147, "\u0120tabl": 36148, "\u0120plains": 36149, "1600": 36150, "pressure": 36151, "\u0120Essence": 36152, "margin": 36153, "friends": 36154, "\u0120Restoration": 36155, "\u0120pollut": 36156, "\u0120Poker": 36157, "\u0120Augustine": 36158, "\u0120CIS": 36159, "\u0120SEAL": 36160, "orama": 36161, "\u0120thwart": 36162, "seek": 36163, "\u0120pagan": 36164, "\u00c2\u00ba": 36165, "cpu": 36166, "\u0120garn": 36167, "\u0120assortment": 36168, "\u0120ILCS": 36169, "tower": 36170, "Recommended": 36171, "\u0120unborn": 36172, "\u0120RandomRedditor": 36173, "\u0120RandomRedditorWithNo": 36174, "\u0120paralyzed": 36175, "\u0120eruption": 36176, "\u0120intersect": 36177, "\u0120Stoke": 36178, "\u0120Sco": 36179, "Bind": 36180, "\u00e5\u00be": 36181, "\u0120PNG": 36182, "\u0120Negative": 36183, "\u0120NOAA": 36184, "Leon": 36185, "\u0120alloy": 36186, "\u0120Lama": 36187, "\u0120Diversity": 36188, "575": 36189, "\u0120underestimated": 36190, "\u0120Scor": 36191, "\u0120mural": 36192, "\u0120busted": 36193, "soon": 36194, "lif": 36195, "\u0120nonex": 36196, "\u0120allergy": 36197, "\u0120Underworld": 36198, "\u0120Rays": 36199, "\u0120Blasio": 36200, "\u0120hrs": 36201, "\u0120Dir": 36202, "\u0120327": 36203, "byter": 36204, "\u0120replacements": 36205, "\u0120activates": 36206, "rived": 36207, "MH": 36208, "\u0120pans": 36209, "\u0120HI": 36210, "\u0120longitudinal": 36211, "\u0120nuisance": 36212, "aler": 36213, "\u0120swell": 36214, "\u0120Signed": 36215, "sci": 36216, "\u0120Isles": 36217, "\u0120AGA": 36218, "\u0120defiant": 36219, "\u0120sonic": 36220, "ocon": 36221, "KC": 36222, "\u0120Aim": 36223, "tie": 36224, "ahah": 36225, "\u0120mL": 36226, "DX": 36227, "\u0120bisc": 36228, "\u0120Billboard": 36229, "\u0120SYSTEM": 36230, "NEY": 36231, "gaard": 36232, "\u0120distressed": 36233, "formerly": 36234, "Alan": 36235, "\u0120chefs": 36236, "\u0120optics": 36237, "\u0120Comet": 36238, "\u0120AMC": 36239, "\u0120redesigned": 36240, "irmation": 36241, "\u0120sightings": 36242, "382": 36243, "311": 36244, "\u0120WB": 36245, "\u0120contraction": 36246, "\u0120TOTAL": 36247, "Dual": 36248, "\u0120startled": 36249, "\u0120understandably": 36250, "\u0120sunglasses": 36251, "ETHOD": 36252, "\u0120docker": 36253, "\u0120surfing": 36254, "\u0120HEL": 36255, "\u0120Slack": 36256, "tones": 36257, "\u0120shalt": 36258, "Visual": 36259, "498": 36260, "Department": 36261, "cussion": 36262, "\u0120unrestricted": 36263, "\u0120tad": 36264, "\u0120rename": 36265, "employed": 36266, "\u0120educating": 36267, "\u0120grinned": 36268, "bedroom": 36269, "\u0120Activities": 36270, "\u0120Velvet": 36271, "\u0120SWAT": 36272, "\u0120shuffle": 36273, "igor": 36274, "\u0120saturation": 36275, "Finding": 36276, "cream": 36277, "icter": 36278, "\u0120vodka": 36279, "tracking": 36280, "tec": 36281, "\u0120foreground": 36282, "iesta": 36283, "\u0120vehement": 36284, "\u0120ECB": 36285, "\u0120Tie": 36286, "Ey": 36287, "\u0120turtles": 36288, "\u0120Railroad": 36289, "\u0120Katz": 36290, "\u0120Frames": 36291, "\u0120menace": 36292, "\u0120Fellowship": 36293, "\u0120Essential": 36294, "uggish": 36295, "\u0120drip": 36296, "chwitz": 36297, "\u0120Kyoto": 36298, "sb": 36299, "\u0120Nina": 36300, "Parameter": 36301, "\u0120alarms": 36302, "\u0120Claud": 36303, "\u0120pioneering": 36304, "\u0120chiefly": 36305, "\u0120Scream": 36306, "Collection": 36307, "\u0120thankfully": 36308, "\u0120Ronaldo": 36309, "\u00e5\u0143\u0132": 36310, "strip": 36311, "\u0120Disneyland": 36312, "commercial": 36313, "Seeing": 36314, "Soul": 36315, "\u0120evacuate": 36316, "\u0120civ": 36317, "\u0120Ashe": 36318, "\u0120divides": 36319, "\u0120Dagger": 36320, "rehensive": 36321, "\u0120berries": 36322, "\u0120DF": 36323, "\u0120sushi": 36324, "\u0120plurality": 36325, "WI": 36326, "\u0120disadvantaged": 36327, "\u0120battalion": 36328, "obiles": 36329, "451": 36330, "\u0120cling": 36331, "\u0120undeniable": 36332, "\u0120Lounge": 36333, "\u0120haunt": 36334, "phe": 36335, "\u0120quantify": 36336, "\u0120differed": 36337, "\u0120[*]": 36338, "\u0120Viz": 36339, "cum": 36340, "slave": 36341, "\u0120videog": 36342, "\u0120quar": 36343, "\u0120bundles": 36344, "\u0120Alonso": 36345, "tackle": 36346, "\u0120neuronal": 36347, "\u0120landslide": 36348, "confirmed": 36349, "\u0120Depth": 36350, "\u0120renewables": 36351, "Bear": 36352, "\u0120Macedonia": 36353, "\u0120jerseys": 36354, "\u0120bunk": 36355, "\u0120Spawn": 36356, "\u0120Controls": 36357, "\u0120Buchanan": 36358, "\u0120robotics": 36359, "\u0120emphasizing": 36360, "\u0120Tutorial": 36361, "hyp": 36362, "iston": 36363, "\u0120monumental": 36364, "\u00e6\u00b0": 36365, "\u0120Carry": 36366, "\u0120tbsp": 36367, "enance": 36368, "Hill": 36369, "arthed": 36370, "\u0120rotten": 36371, "Dean": 36372, "\u0120twisting": 36373, "\u0120goodwill": 36374, "\u0120immersion": 36375, "Living": 36376, "\u0120brushes": 36377, "\u0120CGI": 36378, "\u0120Atk": 36379, "traditional": 36380, "\u0120phantom": 36381, "\u0120Stamina": 36382, "\u0120expansions": 36383, "\u0120Marin": 36384, "\u0120embarked": 36385, "\u0120Eg": 36386, "intestinal": 36387, "\u0120PEOPLE": 36388, "\u0120Booth": 36389, "\u0120Appalach": 36390, "\u0120relegated": 36391, "VT": 36392, "MIT": 36393, "\u0120muster": 36394, "\u0120withdrawing": 36395, "\u0120microscope": 36396, "\u0120Gathering": 36397, "\u0120Crescent": 36398, "\u0120Argentine": 36399, "\u0120Decre": 36400, "\u0120Dominic": 36401, "\u0120buds": 36402, "antage": 36403, "\u0120Ion": 36404, "\u0120widened": 36405, "ONSORED": 36406, "\u0120Gloves": 36407, "iannopoulos": 36408, "razen": 36409, "feel": 36410, "\u0120repayment": 36411, "\u0120hindsight": 36412, "\u0120REALLY": 36413, "\u0120Pistol": 36414, "\u0120Brah": 36415, "\u0120watts": 36416, "\u0120survives": 36417, "\u0120flurry": 36418, "issy": 36419, "Alert": 36420, "\u0120Uruguay": 36421, "Phoenix": 36422, "Slow": 36423, "\u0120Grave": 36424, "\u0120Fir": 36425, "\u0120manageable": 36426, "\u0120tariff": 36427, "\u0120UDP": 36428, "\u0120Pistons": 36429, "\u0120Nigerian": 36430, "\u0120strikeouts": 36431, "\u0120cosmetics": 36432, "whelming": 36433, "fab": 36434, "cape": 36435, "proxy": 36436, "\u0120rethink": 36437, "\u0120overcoming": 36438, "simple": 36439, "\u0120woo": 36440, "\u0120distracting": 36441, "\u0120Stanton": 36442, "\u0120Tulsa": 36443, "\u0120Dock": 36444, "659": 36445, "\u0120discord": 36446, "\u0120Emacs": 36447, "\u0120Ves": 36448, "\u0120ROB": 36449, "\u0120reassuring": 36450, "\u0120consortium": 36451, "Muslims": 36452, "321": 36453, "\u0120prompts": 36454, "sei": 36455, "\u0120Hitch": 36456, "imposed": 36457, "\u0120Fool": 36458, "\u0120indiscrim": 36459, "wrong": 36460, "buquerque": 36461, "Davis": 36462, "!]": 36463, "\u0120timeless": 36464, "\u0120NEED": 36465, "\u0120pesticide": 36466, "\u0120rallying": 36467, "\u0120Calder": 36468, "\u0120\u00e5\u00a4": 36469, "\u0120xp": 36470, "\u0120Unle": 36471, "\u0120Export": 36472, "luaj": 36473, "Buff": 36474, ")[": 36937, "\u0120sqor": 36938, "Saudi": 36939, "\u0120istg": 36940, "\u0120indulge": 36941, "proc": 36942, "\u0120disgusted": 36943, "\u0120compounded": 36944, "\u0120nem": 36945, "\u0120schooling": 36946, "\u0120Cure": 36947, "processing": 36948, "Sol": 36949, "\u0120proverb": 36950, "itized": 36951, "\u0120Alvarez": 36952, "\u0120scarf": 36953, "\u0120rectangular": 36954, "reve": 36955, "\u0120hormonal": 36956, "\u0120Stress": 36957, "itizen": 36958, "\u0120425": 36959, "girls": 36960, "\u0120Noir": 36961, "\u0120Rapp": 36962, "\u0120marches": 36963, "church": 36964, "\u0120Uses": 36965, "\u0120405": 36966, "\u0120Berm": 36967, "\u0120ordinances": 36968, "\u0120Judgment": 36969, "Charges": 36970, "\u0120Zin": 36971, "\u0120dusty": 36972, "\u0120strawberries": 36973, "\u0120perce": 36974, "\u0120Thur": 36975, "\u0120Deborah": 36976, "netflix": 36977, "\u0120Lambert": 36978, "\u0120amused": 36979, "\u0120Guang": 36980, "YOU": 36981, "RGB": 36982, "\u0120CCTV": 36983, "\u0120fiat": 36984, "rang": 36985, "\u0120federation": 36986, "\u0120Mant": 36987, "\u0120Bust": 36988, "\u0120Mare": 36989, "respective": 36990, "\u0120Migration": 36991, "\u0120BIT": 36992, "590": 36993, "\u0120patriotism": 36994, "\u0120outlining": 36995, "region": 36996, "\u0120Jos\u00c3\u00a9": 36997, "\u0120blasting": 36998, "\u0120Ezra": 36999, "Bs": 37000, "\u0120undermines": 37001, "\u0120Smooth": 37002, "\u0120clashed": 37003, "radio": 37004, "\u0120transitioning": 37005, "\u0120Buccaneers": 37006, "\u0120Owl": 37007, "\u0120plugs": 37008, "\u0120hiatus": 37009, "\u0120Pinball": 37010, "\u0120mig": 37011, "\u0120Nutr": 37012, "\u0120Wolfe": 37013, "\u0120integers": 37014, "\u0120orbits": 37015, "\u0120Edwin": 37016, "\u0120DirectX": 37017, "bite": 37018, "\u0120blazing": 37019, "vr": 37020, "Edge": 37021, "\u0120PID": 37022, "exit": 37023, "\u0120Comed": 37024, "\u0120Pathfinder": 37025, "\u0120Guid": 37026, "\u0120Signs": 37027, "\u0120Zer": 37028, "\u0120Agenda": 37029, "\u0120reimbursement": 37030, "Mesh": 37031, "iPhone": 37032, "\u0120Marcos": 37033, "\u0120Sites": 37034, "hate": 37035, "enburg": 37036, "\u0120sockets": 37037, "pend": 37038, "Batman": 37039, "vir": 37040, "\u0120SHOW": 37041, "\u0120provisional": 37042, "conn": 37043, "\u0120Deaths": 37044, "ATIVE": 37045, "Profile": 37046, "sym": 37047, "JA": 37048, "\u0120ninja": 37049, "installed": 37050, "idates": 37051, "ebra": 37052, "\u0120Omaha": 37053, "\u0120seizing": 37054, "\u0120Beasts": 37055, "\u0120salts": 37056, "Mission": 37057, "Generally": 37058, "\u0120Trilogy": 37059, "heon": 37060, "legates": 37061, "\u0120dime": 37062, "\u0120faire": 37063, "parable": 37064, "Graph": 37065, "\u0120totaling": 37066, "\u0120diagrams": 37067, "\u0120Yanuk": 37068, "plet": 37069, "\u0120Meh": 37070, "\u0120mythical": 37071, "\u0120Stephens": 37072, "autical": 37073, "ochemistry": 37074, "\u0120kilograms": 37075, "\u0120elbows": 37076, "ancock": 37077, "\u0120BCE": 37078, "\u0120Prague": 37079, "\u0120improv": 37080, "\u0120Devin": 37081, "\u0120\"\\": 37082, "paralle": 37083, "\u0120supremacists": 37084, "\u0120Billion": 37085, "\u0120regimen": 37086, "innacle": 37087, "\u0120requisite": 37088, "angan": 37089, "\u0120Burlington": 37090, "ainment": 37091, "\u0120Objective": 37092, "omsky": 37093, "GV": 37094, "\u0120unilateral": 37095, "\u0120tc": 37096, "\u0120hires": 37097, "mental": 37098, "\u0120involuntary": 37099, "\u0120transpl": 37100, "\u0120ASCII": 37101, "\u00c2\u00a8": 37102, "Events": 37103, "\u0120doubted": 37104, "\u0120Kaplan": 37105, "\u0120Courage": 37106, "igon": 37107, "\u0120Managing": 37108, "\u0120Tart": 37109, "\u0120falsehood": 37110, "\u0120Violet": 37111, "\u0120airs": 37112, "\u0120fertilizer": 37113, "Britain": 37114, "\u0120aquatic": 37115, "ouf": 37116, "Words": 37117, "\u0120Hartford": 37118, "\u0120evenings": 37119, "\u0120Vengeance": 37120, "quite": 37121, "Gall": 37122, "\u0120Pret": 37123, "\u0120pdf": 37124, "\u0120LM": 37125, "\u0120Sochi": 37126, "\u0120Intercept": 37127, "920": 37128, "\u0120profitability": 37129, "\u0120Idle": 37130, "\u0120MacDonald": 37131, "\u0120Establishment": 37132, "umsy": 37133, "\u0120gatherings": 37134, "\u0120Naj": 37135, "Charlie": 37136, "\u0120ascent": 37137, "\u0120Protector": 37138, "\u0120algebra": 37139, "\u0120bios": 37140, "forums": 37141, "ELS": 37142, "Introduced": 37143, "\u0120335": 37144, "\u0120astronomy": 37145, "Contribut": 37146, "\u0120Polic": 37147, "Platform": 37148, "\u0120containment": 37149, "wrap": 37150, "\u0120coronary": 37151, "\u0120Jelly": 37152, "manager": 37153, "\u0120heartbreaking": 37154, "cair": 37155, "\u0120Chero": 37156, "cgi": 37157, "Medical": 37158, "\u0120Accountability": 37159, "!!\"": 37160, "ophile": 37161, "\u0120psychotic": 37162, "\u0120Restrict": 37163, "\u0120equitable": 37164, "issues": 37165, "\u01201905": 37166, "\u0120Nek": 37167, "cised": 37168, "\u0120Tracking": 37169, "\u0120ozone": 37170, "\u0120cooker": 37171, "rosis": 37172, "\u0120reopen": 37173, "\u0120infinity": 37174, "\u0120Pharmaceutical": 37175, "ensional": 37176, "Attempt": 37177, "\u0120Rory": 37178, "Marco": 37179, "\u0120awaits": 37180, "HOW": 37181, "treated": 37182, "\u0120bolst": 37183, "\u0120revered": 37184, "\u0120pods": 37185, "oppers": 37186, "0010": 37187, "\u0120amplitude": 37188, "rican": 37189, "SPONSORED": 37190, "\u0120trousers": 37191, "\u0120halves": 37192, "\u0120Kaine": 37193, "\u0120Cutler": 37194, "\u0120AUTH": 37195, "\u0120splendid": 37196, "\u0120preventive": 37197, "\u0120Dudley": 37198, "ifacts": 37199, "uminati": 37200, "\u0120Yin": 37201, "\u0120admon": 37202, "\u0120Vag": 37203, "\u0120inverted": 37204, "\u0120hastily": 37205, "\u0120Hague": 37206, "Lyn": 37207, "\u0120ledger": 37208, "\u0120astronomical": 37209, "getting": 37210, "\u0120circa": 37211, "\u0120Cic": 37212, "\u0120Tennis": 37213, "Limited": 37214, "\u0120dru": 37215, "\u0120BYU": 37216, "\u0120travellers": 37217, "\u0120pane": 37218, "\u0120Intro": 37219, "\u0120patiently": 37220, "\u0120aiding": 37221, "\u0120loos": 37222, "\u0120Tough": 37223, "\u0120293": 37224, "\u0120consumes": 37225, "SourceFile": 37226, "\u0120\"\"\"": 37227, "\u0120bonding": 37228, "\u0120tilted": 37229, "\u0120menstrual": 37230, "\u0120Celestial": 37231, "ULAR": 37232, "Plugin": 37233, "\u0120risking": 37234, "Naz": 37235, "\u0120Riyadh": 37236, "\u0120accredited": 37237, "\u0120skirm": 37238, "\u00e9\u013d": 37239, "\u0120examiner": 37240, "\u0120messing": 37241, "\u0120nearing": 37242, "\u0120Chern": 37243, "\u0120Beckham": 37244, "\u0120swapped": 37245, "\u0120goose": 37246, "Kay": 37247, "\u0120lofty": 37248, "\u0120Wallet": 37249, "\u0120['": 37250, "\u0120apocalypse": 37251, "\u0120bamboo": 37252, "\u0120SPACE": 37253, "\u0120Elena": 37254, "\u0120306": 37255, "acons": 37256, "\u0120tightened": 37257, "\u0120adolescence": 37258, "\u0120rainy": 37259, "\u0120vandalism": 37260, "\u0120Newtown": 37261, "\u0120conject": 37262, "cakes": 37263, "\u0120cheated": 37264, "\u0120moderators": 37265, "params": 37266, "EFF": 37267, "\u0120deceit": 37268, "\u0120STL": 37269, "\u0120Tanzania": 37270, "\u0120RI": 37271, "\u01201923": 37272, "\u0120Exile": 37273, "thel": 37274, "\u0120theolog": 37275, "\u0120quirky": 37276, "\u0120Irvine": 37277, "\u0120needy": 37278, "oris": 37279, "Um": 37280, "Ka": 37281, "\u0120mailbox": 37282, "322": 37283, "\u0120bos": 37284, "\u0120Petra": 37285, "KING": 37286, "\u0120enlarged": 37287, "Often": 37288, "\u0120badass": 37289, "\u0120343": 37290, "\u0120Places": 37291, "\u0120CAD": 37292, "\u0120pristine": 37293, "\u0120intervening": 37294, "direction": 37295, "\u0120laz": 37296, "\u0120DSM": 37297, "\u0120projecting": 37298, "\u0120Funk": 37299, "agog": 37300, "payment": 37301, "nov": 37302, "\u0120chatter": 37303, "ARB": 37304, "\u0120examinations": 37305, "\u0120Household": 37306, "\u0120Gus": 37307, "Ford": 37308, "414": 37309, "Boss": 37310, "\u0120mystic": 37311, "\u0120leaps": 37312, "\u0120Bav": 37313, "ulz": 37314, "budget": 37315, "Football": 37316, "\u0120subsidized": 37317, "\u0120firsthand": 37318, "\u0120coincide": 37319, "ocular": 37320, "Conn": 37321, "\u0120Collabor": 37322, "\u0120fools": 37323, "amura": 37324, "ahar": 37325, "rists": 37326, "\u0120swollen": 37327, "\u0120expended": 37328, "\u0120Pau": 37329, "sup": 37330, "\u0120spar": 37331, "\u0120keynote": 37332, "suff": 37333, "\u0120unequal": 37334, "\u0120progressing": 37335, "strings": 37336, "\u0120Gamergate": 37337, "Disney": 37338, "\u0120Eleven": 37339, "omnia": 37340, "\u0120scripted": 37341, "\u0120earners": 37342, "brother": 37343, "\u0120Enabled": 37344, "\u00e6\u00b3": 37345, "\u0120larvae": 37346, "\u0120LOC": 37347, "mess": 37348, "Wilson": 37349, "\u0120Template": 37350, "successfully": 37351, "\u0120paramount": 37352, "\u0120camouflage": 37353, "\u0120binds": 37354, "\u0120Quiet": 37355, "\u0120Shutterstock": 37356, "rush": 37357, "\u0120mascot": 37358, "fortune": 37359, "\u0120Colt": 37360, "\u0120Beyon": 37361, "habi": 37362, "\u0120hairc": 37363, "\u0120267": 37364, "\u0120Deus": 37365, "\u0120twitch": 37366, "\u0120concentrating": 37367, "\u0120nipples": 37368, "cible": 37369, "\u0120gir": 37370, "NZ": 37371, "Math": 37372, "nih": 37373, "Required": 37374, "\u0120ponder": 37375, "\u0120SAN": 37376, "\u0120weddings": 37377, "\u0120loneliness": 37378, "NES": 37379, "\u0120Mahjong": 37380, "695": 37381, "addle": 37382, "\u0120Garner": 37383, "\u0120COUR": 37384, "Bridge": 37385, "\u0120spree": 37386, "\u0120Caldwell": 37387, "\u0120bribery": 37388, "\u0120\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd\u00ef\u00bf\u00bd": 37389, "plugins": 37390, "\u0120racket": 37391, "\u0120champagne": 37392, "versible": 37393, "Vote": 37394, "\u0120modifiers": 37395, "Mayor": 37396, "680": 37397, "\u0120assemblies": 37398, "\u0120Sultan": 37399, "\u0120Ning": 37400, "\u0120Ladies": 37401, "\u0120sulfur": 37402, "\u0120orbs": 37403, "\u0120-----": 37404, "_______": 37405, "\u0120Journalism": 37406, "\u0120esports": 37407, "\u0120lush": 37408, "\u0120hue": 37409, "\u0120spectral": 37410, "Honest": 37411, "\u00e3\u0125\u0131": 37412, "\u0120bushes": 37413, "\u0120reinforcement": 37414, "\u0120reopened": 37415, "\u0120Wheels": 37416, "\u0120Morg": 37417, "rieving": 37418, "\u0120auxiliary": 37419, "\u0120jQuery": 37420, "\u0120BAT": 37421, "tesque": 37422, "\u0120vertex": 37423, "pure": 37424, "frey": 37425, "\u00e3\u0124\u00ba": 37426, "dos": 37427, "\u0120typh": 37428, "\u0120cull": 37429, "\u0120eq": 37430, "\u0120decon": 37431, "\u0120tossing": 37432, "\u0120disparate": 37433, "\u0120Brigham": 37434, "printf": 37435, "ledged": 37436, "\u0120sund": 37437, "\u0120cozy": 37438, "\u0120hepatitis": 37439, "performing": 37440, "\u0120aval": 37441, "\u0120GG": 37442, "future": 37443, "\u0120petertodd": 37444, "\u0120Kosovo": 37445, "\u0120magnets": 37446, "Already": 37447, "\u0120Edison": 37448, "\u0120Ceres": 37449, "\u0120RAID": 37450, "\u0120brilliance": 37451, "576": 37452, "\u0120derives": 37453, "\u0120hypertension": 37454, "\u0120\u00ce\u0136": 37455, "\u0120lambda": 37456, "\u0120flair": 37457, "\u0120missionaries": 37458, "\u0120rapes": 37459, "\u0120Starter": 37460, "\u0120Months": 37461, "\u0120defy": 37462, "\u0120seismic": 37463, "\u0120Raphael": 37464, "\u0120eurozone": 37465, "656": 37466, "zsche": 37467, "\u0120scratched": 37468, "\u0120bows": 37469, "\u0120Lennon": 37470, "\u0120Gaia": 37471, "\u0120dripping": 37472, "facts": 37473, "Ale": 37474, "\u0120frogs": 37475, "\u0120Breast": 37476, "ogeneity": 37477, "\u0120Prosecutor": 37478, "\u0120amplified": 37479, "\u0120Hodg": 37480, "\u0120Fn": 37481, "Thousands": 37482, "\u0120NIH": 37483, "\u0120Monitoring": 37484, "FTWARE": 37485, "\u0120Priebus": 37486, "\u0120Growing": 37487, "hunter": 37488, "\u0120diagnose": 37489, "\u0120Mald": 37490, "\u0120LR": 37491, "\u0120crowned": 37492, "\u0120bursting": 37493, "\u0120dissolution": 37494, "javascript": 37495, "\u0120usefulness": 37496, "\u0120Execution": 37497, ":(": 37498, "\u0120Ivory": 37499, "aah": 37500, "\u0120persecuted": 37501, "violence": 37502, "istas": 37503, "\u0120Crate": 37504, "\u0120impulses": 37505, "\u0120Spani": 37506, "edes": 37507, "Handle": 37508, "\u0120Zerg": 37509, "thinkable": 37510, "Lastly": 37511, "\u0120spontaneously": 37512, "\u0120inconvenient": 37513, "\u0120dismissing": 37514, "\u0120plotted": 37515, "\u0120eighty": 37516, "\u0120737": 37517, "rish": 37518, "\u0120Thornton": 37519, "atham": 37520, "\u0120sitcom": 37521, "Ven": 37522, "Recipe": 37523, "tel": 37524, "lund": 37525, "\u0120clears": 37526, "\u0120Sasuke": 37527, "\u0120258": 37528, "\u0120opting": 37529, "\u0120enraged": 37530, "esthetic": 37531, "\u0120Ae": 37532, "uchs": 37533, "Prep": 37534, "Flow": 37535, "\u0120runoff": 37536, "\u0120Eating": 37537, "\u0120Giles": 37538, "\u0120Acting": 37539, "resources": 37540, "ibaba": 37541, "\u0120rpm": 37542, "\u0120skewed": 37543, "\u0120Blanc": 37544, "\u0120Sakuya": 37545, "\u0120hotter": 37546, "\u01201924": 37547, "opian": 37548, "cko": 37549, "\u0120crumbling": 37550, "\u0120captains": 37551, "\u0120Appropriations": 37552, "leaders": 37553, "dropping": 37554, "anuts": 37555, "\u0120reversing": 37556, "\u0120Pose": 37557, "\u0120Sek": 37558, "Scot": 37559, "\u0120Idea": 37560, "cise": 37561, "\u0120Slovenia": 37562, "\u0120317": 37563, "Doctor": 37564, "\u0120crocod": 37565, "aldi": 37566, "Sea": 37567, "\u0120Farrell": 37568, "\u0120mercenaries": 37569, "\u0120RNC": 37570, "\u0120Guess": 37571, "\u0120pacing": 37572, "Machine": 37573, "StreamerBot": 37574, "\u0120Charity": 37575, "\u0120298": 37576, "\u0120cannons": 37577, "\u0120Toby": 37578, "TPPStreamerBot": 37579, "\u0120Passion": 37580, "cfg": 37581, "Thom": 37582, "\u0120badges": 37583, "\u0120Bernstein": 37584, ".\u00e2\u0122\u0135": 37585, "\u0120POP": 37586, "\u0120Conj": 37587, "\u0120initialization": 37588, "\u0120biodiversity": 37589, "Dub": 37590, "\u0120feudal": 37591, "\u0120disclaimer": 37592, "\u0120crow": 37593, "\u0120ignition": 37594, "arf": 37595, "SHA": 37596, "\u0120kHz": 37597, "hazard": 37598, "\u0120Artists": 37599, "oeuv": 37600, "679": 37601, "\u0120Rudy": 37602, "Nine": 37603, "\u0120Ramadan": 37604, "\u00e5\u00bd": 37605, "itto": 37606, "\u0120adrenaline": 37607, "Cert": 37608, "\u0120smelled": 37609, "\u0120impunity": 37610, "\u0120agendas": 37611, "\u0120Reborn": 37612, "\u0120Concent": 37613, "\u0120Seems": 37614, "\u0120omega": 37615, "\u0120Dustin": 37616, "\u0120backer": 37617, "\u0120Sauce": 37618, "\u0120Boyle": 37619, "WIN": 37620, "\u0120spins": 37621, "\u0120pauses": 37622, "upt": 37623, "\u0120shredded": 37624, "\u0120strapped": 37625, "\u0120Corruption": 37626, "\u0120scratches": 37627, "\u0120ni": 37628, "\u0120attire": 37629, "\u0120SAF": 37630, "FactoryReloaded": 37631, "\u0120IPS": 37632, "\u0120(%": 37633, "\u0120seminar": 37634, "focus": 37635, "civil": 37636, "\u01201860": 37637, "intosh": 37638, "\u0120continual": 37639, "\u0120abbrevi": 37640, "\u0120Sok": 37641, "ocobo": 37642, "XM": 37643, "\u0120frantic": 37644, "\u0120unavoidable": 37645, "\u0120artery": 37646, "\u0120annotations": 37647, "bath": 37648, "Climate": 37649, "\u0120dors": 37650, "\u0120Slide": 37651, "coord": 37652, "\u0120Reload": 37653, "\u0120LDL": 37654, "\u0120Lovecraft": 37655, "\u0120unimagin": 37656, "\u0120resembled": 37657, "\u0120barracks": 37658, "np": 37659, "\u0120surrogate": 37660, "\u0120categorized": 37661, "\u00e3\u0124\u00a9": 37662, "\u0120vaccinated": 37663, "\u0120drainage": 37664, "\u0120indist": 37665, "\u0120WhatsApp": 37666, "\u01201870": 37667, "olerance": 37668, "invoke": 37669, "amorph": 37670, "\u0120reconnect": 37671, "\u0120emanc": 37672, "\u0120blindness": 37673, "\u01201280": 37674, "internet": 37675, "collar": 37676, "\u0120altru": 37677, "\u0120abyss": 37678, "\u0120TRI": 37679, "657": 37680, "\u0120infused": 37681, "HEAD": 37682, "\u0120forestry": 37683, "\u0120Woody": 37684, "\u0120Ci": 37685, "wi": 37686, "sam": 37687, "784": 37688, "holiday": 37689, "\u0120mogul": 37690, "\u0120Fees": 37691, "\u0120DEN": 37692, "Internal": 37693, "urbed": 37694, "fusc": 37695, "atom": 37696, "\u0120Illusion": 37697, "\u0120polled": 37698, "\u0120flap": 37699, "\u0120coax": 37700, "LGBT": 37701, "Analy": 37702, "\u0120Sections": 37703, "\u0120Californ": 37704, "emn": 37705, "\u0120hither": 37706, "\u0120NIGHT": 37707, "\u0120nailed": 37708, "\u0120Pipeline": 37709, "391": 37710, "oof": 37711, "\u0120Primal": 37712, "verend": 37713, "\u0120slashing": 37714, "\u0120retri": 37715, "aviour": 37716, "\u0120departing": 37717, "gil": 37718, "ISC": 37719, "\u0120midway": 37720, "\u0120ultrasound": 37721, "\u0120behaving": 37722, "\u0120Tara": 37723, "classes": 37724, "Virtual": 37725, "\u0120Colonial": 37726, "\u0120stripping": 37727, "\u0120orchestrated": 37728, "\u0120Graves": 37729, "452": 37730, "\u0120Ironically": 37731, "\u0120Writers": 37732, "\u0120lends": 37733, "\u0120Manz": 37734, "\u0120raven": 37735, "\u0120oxidative": 37736, "\u0120266": 37737, "ELF": 37738, "actually": 37739, "ascar": 37740, "Draft": 37741, "\u0120favourable": 37742, "\u0120humiliating": 37743, "\u0120fidelity": 37744, "\u0120Hof": 37745, "\u0120Xuan": 37746, "496": 37747, "\u0120layered": 37748, "atis": 37749, "790": 37750, "\u0120paycheck": 37751, "iton": 37752, "Kar": 37753, "\u0120VMware": 37754, "\u0120Farmer": 37755, "\u0120servic": 37756, "glomer": 37757, "\u0120slump": 37758, "\u0120Fabric": 37759, "\u0120DOC": 37760, "esting": 37761, "\u0120reassure": 37762, "\u0120phyl": 37763, "volt": 37764, "itory": 37765, "Rules": 37766, "\u0120oxidation": 37767, "\u0120prized": 37768, "\u0120mistress": 37769, "\u0120Django": 37770, "WARN": 37771, "\u00e5\u0133": 37772, "\u0120encode": 37773, "\u0120Feedback": 37774, "\u0120stupidity": 37775, "Ian": 37776, "\u0120Yugoslavia": 37777, "\u00d7\u00a8": 37778, "acl": 37779, "UTE": 37780, "1977": 37781, "\u0120qualifies": 37782, "\u0120pulses": 37783, "pretty": 37784, "\u0120froze": 37785, "\u0120ss": 37786, "Iterator": 37787, "\u0120urgently": 37788, "\u0120mailed": 37789, "\u0120Cham": 37790, "\u0120sustaining": 37791, "\u0120basil": 37792, "\u0120puppies": 37793, "ilant": 37794, "\u0120PLEASE": 37795, "lap": 37796, "aceous": 37797, "Fear": 37798, "\u0120Mastery": 37799, "automatic": 37800, "\u0120TAG": 37801, "\u0120antim": 37802, "agles": 37803, "473": 37804, "frames": 37805, "\u0120whispers": 37806, "\u0120Whoever": 37807, "\u0120bravery": 37808, "\u0120UKIP": 37809, "ractions": 37810, "\"\"\"": 37811, "\u0120tame": 37812, "\u0120parted": 37813, "everything": 37814, "CONT": 37815, "\u0120indebted": 37816, "\u0120addr": 37817, "rek": 37818, "IRED": 37819, "\u0120eminent": 37820, "clinton": 37821, "\u0120ousted": 37822, "\u0120reviewer": 37823, "\u0120meltdown": 37824, "\u0120rearr": 37825, "\u0120Yao": 37826, "thereal": 37827, "abyte": 37828, "\u0120stumbling": 37829, "\u0120batches": 37830, "\u0120259": 37831, "\u0120contraceptive": 37832, "\u0120prostitute": 37833, "ensis": 37834, "Decl": 37835, "\u0120Strikes": 37836, "Military": 37837, "\u0120Oath": 37838, "vacc": 37839, "ppings": 37840, "052": 37841, "\u0120partName": 37842, "amping": 37843, "Reports": 37844, "KI": 37845, "CHR": 37846, "\u0120subtly": 37847, "swers": 37848, "Blake": 37849, "usual": 37850, "\u0120contestants": 37851, "\u0120cartridges": 37852, "\u0120GREAT": 37853, "\u0120blush": 37854, "\u0120\u00e2\u0122\u00ba": 37855, "472": 37856, "\u0120reasoned": 37857, "\u00e3\u0125\u00a4": 37858, "paralleled": 37859, "\u0120dyn": 37860, "agate": 37861, "\u0120nightly": 37862, "\u00e5\u0128": 37863, "556": 37864, "\u0120semantic": 37865, "\u0120Advoc": 37866, "\u0120!!": 37867, "\u0120disagrees": 37868, "\u0120BW": 37869, "Veh": 37870, "\u0120harming": 37871, "\u0120embraces": 37872, "\u0120strives": 37873, "\u0120inland": 37874, "\u0120Kard": 37875, "\u0120heats": 37876, "\u0120Ginny": 37877, "utan": 37878, "ernaut": 37879, "ylene": 37880, "\u0120Elev": 37881, "JD": 37882, "\u0120hars": 37883, "\u0120Starr": 37884, "\u0120skysc": 37885, "\u0120collaborators": 37886, "Usually": 37887, "\u0120revolutions": 37888, "\u0120STATS": 37889, "\u0120dismantle": 37890, "\u0120confidently": 37891, "\u0120kinetic": 37892, "Ali": 37893, "\u0120percentile": 37894, "\u0120extracting": 37895, "illian": 37896, "estead": 37897, "\u0120physicists": 37898, "\u0120Marshal": 37899, "\u0120fellowship": 37900, "\u0120dashed": 37901, "\u0120UR": 37902, "\u0120Sioux": 37903, "\u0120Compact": 37904, "amide": 37905, "Python": 37906, "\u0120Leigh": 37907, "\u0120Pharmac": 37908, "istrates": 37909, "herical": 37910, "\u0120fue": 37911, "\u0120Emin": 37912, "\u0120({": 37913, "\u0120Neighborhood": 37914, "\u0120disrupting": 37915, "\u0120Dup": 37916, "\u0120gland": 37917, "\u0120Sev": 37918, "\u0120Marian": 37919, "argon": 37920, "\u0120Dund": 37921, "\u0120": 46904, "\u0120Philips": 46905, "\u0120Kafka": 46906, "\u0120upheaval": 46907, "\u0120sentimental": 46908, "\u0120sax": 46909, "\u0120Akira": 46910, "serial": 46911, "Matrix": 46912, "\u0120electing": 46913, "\u0120commenter": 46914, "\u0120Nebula": 46915, "plets": 46916, "\u0120Nadu": 46917, "\u0120Adren": 46918, "\u0120enshr": 46919, "\u0120RAND": 46920, "financial": 46921, "\u0120Clyde": 46922, "utherford": 46923, "\u0120signage": 46924, "\u0120deline": 46925, "\u0120phosphate": 46926, "roversial": 46927, "fascist": 46928, "\u0120Vall": 46929, "\u0120Bethlehem": 46930, "\u0120fors": 46931, "\u0120english": 46932, "Solid": 46933, "Nature": 46934, "\u0120va": 46935, "\u0120Guests": 46936, "\u0120tantal": 46937, "\u0120autoimmune": 46938, ";;;;;;;;;;;;": 46939, "\u0120Totally": 46940, "\u0120Ov": 46941, "\u0120defences": 46942, "\u0120Coconut": 46943, "\u0120tranquil": 46944, "\u0120ploy": 46945, "\u0120flavours": 46946, "\u0120Flask": 46947, "\u00e3\u0124\u00a8\u00e3\u0125\u00ab": 46948, "\u0120Weston": 46949, "\u0120Volvo": 46950, "870": 46951, "\u0120microphones": 46952, "verbal": 46953, "RPG": 46954, "\u0120iii": 46955, ";}": 46956, "028": 46957, "\u0120headlined": 46958, "\u0120primed": 46959, "\u0120hoard": 46960, "\u0120Shad": 46961, "\u0120ENTER": 46962, "\u0120triangular": 46963, "\u0120capit": 46964, "lik": 46965, "\u0120Ancients": 46966, "\u0120lash": 46967, "\u0120convol": 46968, "\u0120colonel": 46969, "enemy": 46970, "Gra": 46971, "\u0120pubs": 46972, "utters": 46973, "\u0120assigns": 46974, "\u0120Penet": 46975, "\u0120Monstrous": 46976, "\u0120Bowen": 46977, "ilver": 46978, "Haunted": 46979, "\u0120Ding": 46980, "started": 46981, "plin": 46982, "\u0120contaminants": 46983, "\u0120DOE": 46984, "ffen": 46985, "\u0120Technician": 46986, "Ry": 46987, "\u0120robbers": 46988, "\u0120hotline": 46989, "\u0120Guardiola": 46990, "\u0120Kaufman": 46991, "rower": 46992, "\u0120Dresden": 46993, "\u0120Alpine": 46994, "Elf": 46995, "\u0120fmt": 46996, "\u0120Sard": 46997, "urses": 46998, "gpu": 46999, "Unix": 47000, "\u0120unequivocally": 47001, "\u0120Citizenship": 47002, "quad": 47003, "mire": 47004, "\u0120Sweeney": 47005, "Battery": 47006, "615": 47007, "\u0120pancakes": 47008, "\u0120oats": 47009, "Maps": 47010, "\u0120Contrast": 47011, "mbudsman": 47012, "\u0120EPS": 47013, "\u0120subcommittee": 47014, "\u0120sourcing": 47015, "\u0120sizing": 47016, "\u0120Buffer": 47017, "\u0120Mandatory": 47018, "\u0120moderates": 47019, "\u0120Patterns": 47020, "\u0120Chocobo": 47021, "\u0120Zan": 47022, "\u0120STATES": 47023, "\u0120Judging": 47024, "\u0120Inher": 47025, "*:": 47026, "\u0120bil": 47027, "\u0120Yen": 47028, "\u0120exhilar": 47029, "ollower": 47030, "zers": 47031, "\u0120snug": 47032, "maximum": 47033, "\u0120despicable": 47034, "\u0120PACK": 47035, "\u0120Annex": 47036, "\u0120sarcastic": 47037, "\u0120latex": 47038, "\u0120tamp": 47039, "\u0120Sao": 47040, "bah": 47041, "\u0120Reverend": 47042, "\u0120Chinatown": 47043, "\u0120AUT": 47044, "documented": 47045, "\u0120GABA": 47046, "\u0120Canaan": 47047, "\u0120\u00d9\u0127": 47048, "\u0120governs": 47049, "prev": 47050, "Esc": 47051, "\u0120Estimates": 47052, "OSP": 47053, "\u0120endeavour": 47054, "\u0120Closing": 47055, "ometime": 47056, "everyone": 47057, "\u0120worsen": 47058, "\u0120scanners": 47059, "\u0120deviations": 47060, "\u0120Robotics": 47061, "\u0120Compton": 47062, "\u0120sorcerer": 47063, "\u0120endogenous": 47064, "\u0120emulation": 47065, "\u0120Piercing": 47066, "\u0120Aph": 47067, "\u0120Socket": 47068, "\u0120bould": 47069, "\u0120OU": 47070, "\u0120Borderlands": 47071, "\u01201863": 47072, "Gordon": 47073, "\u0120WTO": 47074, "\u0120restricts": 47075, "\u0120mosaic": 47076, "\u0120melodies": 47077, "\u00e7\u0126": 47078, "Tar": 47079, "\u0120disson": 47080, "\u0120Provides": 47081, "\u0120......": 47082, "bek": 47083, "FIX": 47084, "\u0120broom": 47085, "anship": 47086, "Doctors": 47087, "\u0120nerds": 47088, "\u0120Regions": 47089, "naissance": 47090, "\u0120mete": 47091, "\u0120crept": 47092, "plings": 47093, "\u0120girlfriends": 47094, "knit": 47095, "igent": 47096, "owe": 47097, "\u0120ushered": 47098, "\u0120Baz": 47099, "Mobil": 47100, "434": 47101, "\u0120Presents": 47102, "origin": 47103, "\u0120insomnia": 47104, "\u0120Aux": 47105, "439": 47106, "\u0120Chili": 47107, "irsch": 47108, "GAME": 47109, "\u0120gestation": 47110, "algia": 47111, "romising": 47112, "$,": 47113, "crow": 47114, "\u0120Inspection": 47115, "atomic": 47116, "Relations": 47117, "JOHN": 47118, "roman": 47119, "\u0120Clockwork": 47120, "\u0120Bakr": 47121, "mone": 47122, "MET": 47123, "\u0120thirsty": 47124, "\u0120bc": 47125, "\u0120faculties": 47126, "Rum": 47127, "\u0120nuance": 47128, "\u0120Darius": 47129, "pleting": 47130, "fters": 47131, "etchup": 47132, "Registration": 47133, "\u0120KE": 47134, "Rah": 47135, "\u0120preferential": 47136, "\u0120Lash": 47137, "\u0120HH": 47138, "Valid": 47139, "\u0120NAV": 47140, "\u0120starve": 47141, "\u0120Gong": 47142, "zynski": 47143, "\u0120Actress": 47144, "\u0120wik": 47145, "\u0120unaccompanied": 47146, "lvl": 47147, "Bride": 47148, "ADS": 47149, "\u0120Commando": 47150, "\u0120Vaughn": 47151, "Wallet": 47152, "\u0120hopping": 47153, "\u0120Vie": 47154, "\u0120caveats": 47155, "\u0120alas": 47156, "ifled": 47157, "abuse": 47158, "661": 47159, "\u0120ibn": 47160, "\u0120gul": 47161, "\u0120robbing": 47162, "til": 47163, "ILA": 47164, "\u0120mitigating": 47165, "\u0120aptly": 47166, "\u0120tyrant": 47167, "\u0120midday": 47168, "\u0120Gilmore": 47169, "\u0120Decker": 47170, "\u0120\u00c2\u00a7\u00c2\u00a7": 47171, "partial": 47172, "Exactly": 47173, "\u0120phenotype": 47174, "\u0120[+]": 47175, "\u0120Plex": 47176, "\u0120Ips": 47177, "versions": 47178, "\u0120ebook": 47179, "\u0120chic": 47180, "gross": 47181, "\":\"\"},{\"": 47182, "\u0120Surprisingly": 47183, "Morgan": 47184, "\u0120residues": 47185, "\u0120Confederation": 47186, "infeld": 47187, "\u0120lyr": 47188, "moderate": 47189, "\u0120perpendicular": 47190, "VK": 47191, "\u0120synchronized": 47192, "\u0120refreshed": 47193, "\u0120adore": 47194, "\u0120Torment": 47195, "olina": 47196, "\u01202600": 47197, "ItemTracker": 47198, "\u0120pies": 47199, "\u0120FAT": 47200, "\u0120RHP": 47201, "048": 47202, "\u0120RESP": 47203, "\u0120BJ": 47204, "allows": 47205, "Pand": 47206, "\u0120unwelcome": 47207, "\u0120Voc": 47208, "\u0120Bastard": 47209, "\u0120OW": 47210, "\u0120LAR": 47211, "\u0120Healer": 47212, "Environmental": 47213, "\u0120Kenyan": 47214, "\u0120Trance": 47215, "\u0120Pats": 47216, "\u0120aliases": 47217, "\u0120Garfield": 47218, "\u0120campaigner": 47219, "\u0120advancements": 47220, "\u0120Okinawa": 47221, "\u0120Coh": 47222, "owsky": 47223, "\u0120starved": 47224, "\u0120sizeable": 47225, "\u0120:-)": 47226, "\u0120mRNA": 47227, "\u0120suspensions": 47228, "istar": 47229, "Scotland": 47230, "Prin": 47231, "------------------------------------------------": 47232, "\u0120502": 47233, "\u0120teaspoons": 47234, "\u01201050": 47235, "\u0120coercive": 47236, "\u0120Masonic": 47237, "edded": 47238, "\u0120Passenger": 47239, "\u0120latt": 47240, "\u0120braces": 47241, "\u0120Steal": 47242, "\u0120NYT": 47243, "\u0120Kats": 47244, "\u0120Celest": 47245, "aez": 47246, "Tu": 47247, "\u0120Coulter": 47248, "\u00f0\u0141\u013a": 47249, "Flickr": 47250, "\u0120Wilmington": 47251, "iths": 47252, "++;": 47253, "\u0120vending": 47254, "\u0120negro": 47255, "\u0120Phi": 47256, "\u0120Yellowstone": 47257, "Callback": 47258, "\u0120shampoo": 47259, "\u0120Shades": 47260, "wat": 47261, "\u0120superhuman": 47262, "\u0120ridiculed": 47263, "\u0120holiest": 47264, "ombo": 47265, "\u0120interns": 47266, "\u0120hone": 47267, "\u0120Paragu": 47268, "URI": 47269, "\u0120dangling": 47270, "\u00e3\u0124\u00bb": 47271, "sov": 47272, "ictional": 47273, "availability": 47274, "\u0120revocation": 47275, "\u0120dow": 47276, "inic": 47277, "\u0120THEIR": 47278, "\u0120iso": 47279, "\u0120outings": 47280, "\u0120Lethal": 47281, "\u0120)))": 47282, "\u0120inaccur": 47283, "\u0120outlandish": 47284, "\u0120anus": 47285, "letico": 47286, "idon": 47287, "lol": 47288, "\u0120unregulated": 47289, "\u0120succumbed": 47290, "\u0120cuff": 47291, "\u0120Wasteland": 47292, "letal": 47293, "\u0120substr": 47294, "\u0120coffers": 47295, "\u0120automakers": 47296, "ovi": 47297, "\u0120Xue": 47298, "\u0120Daytona": 47299, "\u0120jarring": 47300, "\u0120fumes": 47301, "\u0120disbanded": 47302, "zik": 47303, "itton": 47304, "\u0120strikingly": 47305, "\u0120spores": 47306, "Adapter": 47307, ".):": 47308, "\u0120Lyndon": 47309, "ivalry": 47310, "\u0120orally": 47311, "\u0120tumultuous": 47312, "\u0120displeasure": 47313, "\u0120cones": 47314, "orrect": 47315, "\u0120appease": 47316, "\u0120derby": 47317, "\u0120Tripoli": 47318, "\u0120Aless": 47319, "\u0120poked": 47320, "\u0120Guilty": 47321, "vP": 47322, "Enough": 47323, "\u0120originals": 47324, "699": 47325, "\u0120rabbi": 47326, "\u0120proverbial": 47327, "\u0120postpone": 47328, "elope": 47329, "\u0120Misty": 47330, "\u0120staffed": 47331, "\u0120Unemployment": 47332, "reditary": 47333, "\u0120diligent": 47334, "recomm": 47335, "measures": 47336, "asin": 47337, "825": 47338, "\u0120ponds": 47339, "\u0120mmol": 47340, "\u0120SAR": 47341, "\u0120CARE": 47342, "\u0120371": 47343, "\u0120clenched": 47344, "\u0120Corsair": 47345, "\u0120caricature": 47346, "zn": 47347, "attach": 47348, "\u0120Schro": 47349, "speak": 47350, "painted": 47351, "\u0120Suc": 47352, "\u0120ENT": 47353, "\u0120cellul": 47354, "\u0120Paid": 47355, "diagn": 47356, "WHERE": 47357, "\u0120texted": 47358, "Barn": 47359, "\u0120retracted": 47360, "\u0120Referred": 47361, "Sav": 47362, "\u0120upkeep": 47363, "\u0120workplaces": 47364, "\u0120Tokens": 47365, "\u0120amplify": 47366, "clinical": 47367, "\u0120multic": 47368, "mberg": 47369, "\u0120convoluted": 47370, "Region": 47371, "565": 47372, "\u0120Topic": 47373, "\u0120snail": 47374, "\u0120saline": 47375, "\u0120insurrection": 47376, "\u0120Petr": 47377, "forts": 47378, "BAT": 47379, "\u0120Navajo": 47380, "\u0120rudimentary": 47381, "\u0120Laksh": 47382, "ONDON": 47383, "Measure": 47384, "\u0120transformer": 47385, "\u0120Goddard": 47386, "\u0120coincides": 47387, "irin": 47388, "Rex": 47389, "\u0120Bok": 47390, "quit": 47391, "\u0120shotguns": 47392, "\u0120proletarian": 47393, "\u0120scorp": 47394, "\u0120Ada": 47395, "514": 47396, "\u0120slander": 47397, "recorded": 47398, "\u0120embell": 47399, "risome": 47400, "\u0120apologizing": 47401, "\u0120Mulcair": 47402, "\u0120Gibraltar": 47403, "Cla": 47404, "\u0120allot": 47405, "\u0120Attention": 47406, "\u0120433": 47407, "leave": 47408, "\u0120whine": 47409, "\u0120Issa": 47410, "\u0120Faust": 47411, "\u0120Barron": 47412, "heny": 47413, "\u0120victimized": 47414, "Jews": 47415, "\u0120nurturing": 47416, "ettel": 47417, "Winged": 47418, "\u0120Subtle": 47419, "\u0120flavorful": 47420, "\u0120Reps": 47421, "enged": 47422, "callback": 47423, "\u0120directional": 47424, "\u0120clasp": 47425, "\u0120Directions": 47426, "planet": 47427, "iculture": 47428, "Helper": 47429, "icion": 47430, "acia": 47431, "\u0120\u00e7\u00a5\u0140": 47432, "\u0120surges": 47433, "\u0120canoe": 47434, "\u0120Premiership": 47435, "been": 47436, "\u0120defied": 47437, "\u0120Trooper": 47438, "\u0120tripod": 47439, "\u0120gasp": 47440, "\u0120Euph": 47441, "\u0120Ads": 47442, "vernight": 47443, "highly": 47444, "Role": 47445, "\u0120entangled": 47446, "\u0120Zeit": 47447, "618": 47448, "\u0120Rusty": 47449, "\u0120havens": 47450, "\u0120Vaughan": 47451, "HAEL": 47452, "\u0120SERVICE": 47453, "/,": 47454, "\u0120stricken": 47455, "\u0120delusions": 47456, "\u0120bis": 47457, "\u0120Haf": 47458, "\u0120gratification": 47459, "\u0120enticing": 47460, "UNCH": 47461, "Adams": 47462, "\u0120OLED": 47463, "\u0120Beetle": 47464, "\u01201899": 47465, "\u0120SOFTWARE": 47466, "ategor": 47467, "VL": 47468, "\u0120Totem": 47469, "\u0120Gators": 47470, "ATURES": 47471, "\u0120impedance": 47472, "Registered": 47473, "\u0120Cary": 47474, "\u0120Aerial": 47475, "onne": 47476, "enium": 47477, "\u0120dred": 47478, "\u0120Beg": 47479, "\u0120concurrently": 47480, "\u0120superpower": 47481, "\u0120Xan": 47482, "jew": 47483, "imester": 47484, "\u0120Dickinson": 47485, "\u00e2\u0136\u0123": 47486, "Fla": 47487, "\u0120pree": 47488, "\u0120Rollins": 47489, "\u00a9\u00b6\u00e6": 47490, "\u0120denomination": 47491, "\u0120Lana": 47492, "516": 47493, "\u0120inciting": 47494, "scribed": 47495, "juries": 47496, "\u0120Wonders": 47497, "approximately": 47498, "\u0120suspending": 47499, "\u0120mountainous": 47500, "\u0120Laugh": 47501, "oidal": 47502, "Ns": 47503, "Detect": 47504, ")=": 47505, "\u0120Luthor": 47506, "\u0120Schwarzenegger": 47507, "\u0120Muller": 47508, "\u0120Devi": 47509, "ecycle": 47510, "Jar": 47511, "613": 47512, "\u0120Longh": 47513, "Bah": 47514, "\u0120SPORTS": 47515, "nw": 47516, "\u0120refinement": 47517, "\u0120waterways": 47518, "\u0120diner": 47519, "Blade": 47520, "683": 47521, "Fac": 47522, "\u0120initials": 47523, "\u0120rog": 47524, "\u0120paranormal": 47525, "BUT": 47526, "\u0120[(": 47527, "\u0120Swanson": 47528, "\u0120Mesh": 47529, "\u00e2\u0138\u00ac": 47530, "Improve": 47531, "\u0120Radiation": 47532, "\u0120Esther": 47533, "\u0120Esk": 47534, "\u0120Aly": 47535, "iky": 47536, "\u0120irrad": 47537, "\u0120Buckingham": 47538, "\u0120refill": 47539, "\u0120._": 47540, "Repe": 47541, "CONCLUS": 47542, "\u0120differentiated": 47543, "\u0120chirop": 47544, "\u0120Atkins": 47545, "Pattern": 47546, "\u0120excise": 47547, "\u0120cabal": 47548, "NSA": 47549, "\u0120STA": 47550, "\u0120SIL": 47551, "\u0120Paraly": 47552, "\u0120rye": 47553, "\u0120Howell": 47554, "\u0120Countdown": 47555, "nesses": 47556, "alysed": 47557, "\u0120resize": 47558, "\u00e3\u0124\u00bd": 47559, "\u0120budgetary": 47560, "\u0120Stras": 47561, "wang": 47562, "\u0120apiece": 47563, "\u0120precincts": 47564, "\u0120peach": 47565, "\u0120skyline": 47566, "\u0120353": 47567, "popular": 47568, "Appearances": 47569, "\u0120Mechanics": 47570, "\u0120DevOnline": 47571, "Sullivan": 47572, "Zen": 47573, "\u0120pu": 47574, "opolis": 47575, "544": 47576, "\u0120deform": 47577, "\u0120counteract": 47578, "\u0120Lange": 47579, "\u0120417": 47580, "Console": 47581, "774": 47582, "\u0120nodding": 47583, "\u0120populism": 47584, "\u0120hep": 47585, "\u0120counselling": 47586, "compliance": 47587, "UFF": 47588, "\u0120undeniably": 47589, "\u0120railing": 47590, "\u0120Horowitz": 47591, "\u0120Simone": 47592, "\u0120Bungie": 47593, "\u0120ak": 47594, "\u0120Talks": 47595, "xff": 47596, "flake": 47597, "Crash": 47598, "\u0120sweaty": 47599, "\u0120banquet": 47600, "\u0120OFFIC": 47601, "\u0120inventive": 47602, "\u0120astronomer": 47603, "\u0120Stamford": 47604, "\u0120Scare": 47605, "\u0120GREEN": 47606, "olicited": 47607, "\u0120rusher": 47608, "\u0120centrist": 47609, "ighting": 47610, "\u0120subclass": 47611, "\u0120disav": 47612, "\u0120defund": 47613, "\u0120Nanto": 47614, "ociate": 47615, "mast": 47616, "\u0120pacif": 47617, "\u0120mend": 47618, "eers": 47619, "immigration": 47620, "ESSION": 47621, "\u0120numbering": 47622, "\u0120laughable": 47623, "\u0120Ended": 47624, "viation": 47625, "emark": 47626, "Pitt": 47627, "\u0120meticulous": 47628, "\u0120LF": 47629, "\u0120congratulated": 47630, "\u0120Birch": 47631, "\u0120swayed": 47632, "\u0120semifinals": 47633, "\u0120humankind": 47634, "matter": 47635, "\u0120Equip": 47636, "opausal": 47637, "Said": 47638, "\u0120Layout": 47639, "\u0120voicing": 47640, "\u0120thug": 47641, "\u0120pornographic": 47642, "IPS": 47643, "\u0120moaning": 47644, "\u0120grievance": 47645, "\u0120confessions": 47646, "escal": 47647, "TEXTURE": 47648, "Authent": 47649, "osaurus": 47650, "Purchase": 47651, "\u0120relegation": 47652, "alter": 47653, "\u0120\u00c2\u0142\u00c2\u0142": 47654, "\u0120riddled": 47655, "\u0120ogre": 47656, "\u0120Lowell": 47657, "Occup": 47658, "Eat": 47659, "\u0120Hyder": 47660, "\u0120Adviser": 47661, "Commerce": 47662, "Hunt": 47663, "\u0120Orth": 47664, "\u0120Competitive": 47665, "\u0120CLA": 47666, "CDC": 47667, "\u0120salads": 47668, "Fle": 47669, "\u0120industrialized": 47670, "`,": 47671, "\u0120OWN": 47672, "\u0120beck": 47673, "\u0120Particularly": 47674, "oubt": 47675, "\u0120mM": 47676, "\u0120Hussain": 47677, "\u0120Chennai": 47678, "\u0120920": 47679, "\u0120appointing": 47680, "\u0120Cullen": 47681, ",,,,,,,,": 47682, "\u0120pores": 47683, "verified": 47684, "\u0120biochemical": 47685, "emate": 47686, "\u0120cowardly": 47687, "\u0120Helsinki": 47688, "\u0120Ethiopian": 47689, "SOURCE": 47690, "ERC": 47691, "estro": 47692, "\u0120biotech": 47693, "\u0120Sour": 47694, "\u0120brewer": 47695, "Bloomberg": 47696, "\u0120intensify": 47697, "Glass": 47698, "anco": 47699, "\u0120FDR": 47700, "greSQL": 47701, "\u0120Fires": 47702, "\u00a9\u00b6\u00e6\u00a5\u00b5": 47703, "eco": 47704, "1001": 47705, "\u0120Homeless": 47706, "\u0120instantaneous": 47707, "\u0120Haste": 47708, "igel": 47709, "Diamond": 47710, "\u0120paving": 47711, "\u0120landfill": 47712, "\u0120dads": 47713, "houn": 47714, ":]": 47715, "\u0120incendiary": 47716, "\u0120Livingston": 47717, "\u0120Hilbert": 47718, "\u0120Checks": 47719, "styles": 47720, "inators": 47721, "\u0120Clive": 47722, "phrine": 47723, "\u0120chimpanzees": 47724, "\u0120pall": 47725, "\u0120JM": 47726, "\u0120Aadhaar": 47727, "\u00f0\u013f": 47728, "\u0120achievable": 47729, "disabled": 47730, "PET": 47731, "OOOOOOOO": 47732, "Mot": 47733, "\u0120intangible": 47734, "\u0120ballet": 47735, "\u0120Webs": 47736, "\u0120Estimated": 47737, "Effects": 47738, "\u0120bailed": 47739, "Joshua": 47740, "\u0120turbulence": 47741, "\u0120occupant": 47742, "\u0120Daylight": 47743, "\u0120361": 47744, "meet": 47745, "\u0120statically": 47746, "\u0120onlook": 47747, "\u0120ki": 47748, "illegal": 47749, "\u0120velvet": 47750, "\u0120dehydration": 47751, "\u0120acquies": 47752, "\u0120Rez": 47753, "akura": 47754, "\u0120Upton": 47755, "atro": 47756, "\u0120incomprehensible": 47757, "\u0120backdoor": 47758, "\u0120Rhino": 47759, "727": 47760, "\u0120maths": 47761, ")+": 47762, "\u0120heresy": 47763, "\u0120df": 47764, "\u0120Roche": 47765, "\u0120Lydia": 47766, "\u0120pancreat": 47767, "reply": 47768, "arrell": 47769, "\u0120solicitation": 47770, "\u0120circadian": 47771, "BIP": 47772, "\u0120foray": 47773, "\u0120cryptic": 47774, "izu": 47775, "imeo": 47776, "\u0120Tomato": 47777, "\u0120Homs": 47778, "examination": 47779, "\u0120quarry": 47780, "\u0120Valiant": 47781, "\u0120Jericho": 47782, "\u0120INCLUD": 47783, "\u01201840": 47784, "519": 47785, "\u0120resists": 47786, "\u0120snapshots": 47787, "\u0120Spur": 47788, "\u0120Antiqu": 47789, "Login": 47790, "\u0120bestselling": 47791, "\u0120antic": 47792, "\u0120Sutherland": 47793, "\u00e3\u0124\u00a2\u00e3\u0125\u00ab": 47794, "\u0120~/": 47795, "\u0120Parm": 47796, "\u00e8\u0125": 47797, "Pages": 47798, "intensity": 47799, "\u0120immobil": 47800, "\u01201865": 47801, "zzo": 47802, "\u0120nifty": 47803, "\u0120fentanyl": 47804, "\u0120Preservation": 47805, "ophen": 47806, "\u0120darts": 47807, "\u0120Dinosaur": 47808, "pointers": 47809, "\u0120Rite": 47810, "suggest": 47811, "awareness": 47812, "\u0120Sheridan": 47813, "\u0120stances": 47814, "\u0120sorcery": 47815, "\u0120perjury": 47816, "\u0120Nikola": 47817, "iever": 47818, "\u0120fiance": 47819, "\u0120Jordanian": 47820, "\u0120Balloon": 47821, "\u0120nab": 47822, "\u0120kb": 47823, "\u0120humanities": 47824, "\u0120Tanaka": 47825, "hillary": 47826, "\u0120consultancy": 47827, "\u0120Zub": 47828, "\u0120remission": 47829, "\u0120confid": 47830, "CHQ": 47831, "\u0120Fug": 47832, "\u0120improvis": 47833, "Yep": 47834, "/_": 47835, "\u0120unwillingness": 47836, "\u0120portfolios": 47837, "055": 47838, "\u0120Instructor": 47839, "aiman": 47840, "\u0120claimants": 47841, "Mbps": 47842, "\u0120Bye": 47843, "received": 47844, "Tweet": 47845, "\u0120indemn": 47846, "riz": 47847, "amara": 47848, "Nat": 47849, "\u0120evaluates": 47850, "\u0120Lur": 47851, "epad": 47852, "FOX": 47853, "\u0120Thro": 47854, "\u0120rusty": 47855, "\u0120bedrock": 47856, "\u0120Oprah": 47857, "JB": 47858, "\u0120manipulative": 47859, "\u0120willful": 47860, "\u0120relapse": 47861, "\u0120extant": 47862, "Theme": 47863, "Sensor": 47864, "\u0120Stability": 47865, "govern": 47866, "\u0120poppy": 47867, "\u0120knack": 47868, "\u0120insulated": 47869, "\u0120Tile": 47870, "\u0120Extrem": 47871, "\u0120untold": 47872, "\u0120converge": 47873, "\u0120refuel": 47874, "igroup": 47875, "\u0120distortions": 47876, "\u0120ravaged": 47877, "\u0120mechanically": 47878, "\u0120Reilly": 47879, "\u0120Nose": 47880, "\u0120Incarnation": 47881, "\u0120Becky": 47882, "abbling": 47883, "\u0120taco": 47884, "\u0120rake": 47885, "\u0120melancholy": 47886, "\u0120illustrious": 47887, "\u0120Dartmouth": 47888, "Guide": 47889, "\u0120Razer": 47890, "\u0120Benz": 47891, "Ultimate": 47892, "\u0120Surprise": 47893, "\u0120pageant": 47894, "offer": 47895, "Whoever": 47896, "\u0120wiser": 47897, "\u0120chemist": 47898, "\u0120HELL": 47899, "\u0120Bulk": 47900, "\u0120plutonium": 47901, "\u0120COVER": 47902, "\u00d6\u00bc": 47903, "failed": 47904, "\u0120tirelessly": 47905, "\u0120infertility": 47906, "\u0120Trident": 47907, "\u0120Showtime": 47908, "\u0120Civ": 47909, "Vice": 47910, "requires": 47911, "ittance": 47912, "\u0120uncontrolled": 47913, "interesting": 47914, "561": 47915, "\u0120innovate": 47916, "ategic": 47917, "Lie": 47918, "\u0120Selling": 47919, "Ul": 47920, "\u0120savior": 47921, "\u0120Tosh": 47922, "\u0120swast": 47923, "PASS": 47924, "\u0120rink": 47925, "\u0120cardio": 47926, "\u0120Iro": 47927, "udi": 47928, "\u0120vantage": 47929, "\u0120vans": 47930, "\u0120Ni\u00c3\u00b1o": 47931, "+=": 47932, "\u0120propagate": 47933, "": 49029, "\u0120leukemia": 49030, "\u0120eluc": 49031, "\u0120announcer": 49032, "\u0120Lithuan": 49033, "\u0120Armageddon": 49034, "\u00e5\u0129": 49035, "Lenin": 49036, "\u0120Ruk": 49037, "\u0120pepp": 49038, "\u0120Romantic": 49039, "\u0120PIT": 49040, "\u0120Interstellar": 49041, "\u0120Atkinson": 49042, "Raid": 49043, "Js": 49044, "Goal": 49045, "Course": 49046, "\u0120vanishing": 49047, "esley": 49048, "\u0120Rounds": 49049, "Elsa": 49050, "593": 49051, "\u0120redundancy": 49052, "\u0120STAND": 49053, "\u0120prophetic": 49054, "\u0120habitable": 49055, "ryu": 49056, "\u0120faintly": 49057, "MODE": 49058, "\u0120flanked": 49059, "IRC": 49060, "Awesome": 49061, "\u0120spurious": 49062, "\u0120Zah": 49063, "\u0120MSG": 49064, "\u0120shading": 49065, "\u0120motivational": 49066, "\u0120Santana": 49067, "\u0120SPR": 49068, "\u0120excruciating": 49069, "omial": 49070, "\u0120Miko": 49071, "\u0120Leopard": 49072, "Abyss": 49073, "\u0120[|": 49074, "dirty": 49075, "\u0120baths": 49076, "\u0120demoral": 49077, "andre": 49078, "PB": 49079, "\u0120unification": 49080, "\u0120sacrament": 49081, "\u0120[&": 49082, "\u0120priceless": 49083, "\u0120gelatin": 49084, "\u0120emanating": 49085, "\u0120Allaah": 49086, "986": 49087, "\u0120outburst": 49088, "\u0120eras": 49089, "\u0120XVI": 49090, "\u0120SPI": 49091, "Ott": 49092, "\u0120Lazarus": 49093, "PLIED": 49094, "Flying": 49095, "blogs": 49096, "Wisconsin": 49097, "Raven": 49098, "\u0120rebate": 49099, "\u0120creeps": 49100, "\u0120Span": 49101, "\u0120Painter": 49102, "\u0120Kira": 49103, "\u0120Amos": 49104, "\u0120Corvette": 49105, "Consumer": 49106, "\u0120Recover": 49107, "cki": 49108, "\u0120pesky": 49109, "\u0120Invention": 49110, "Companies": 49111, "\u0120challengers": 49112, "ademic": 49113, "\u0120Ukrainians": 49114, "\u0120Neurolog": 49115, "\u0120Forsaken": 49116, "\u0120entrants": 49117, "\u0120embattled": 49118, "\u0120defunct": 49119, "\u0120Glacier": 49120, "\u0120poisons": 49121, "\u0120Horses": 49122, "makes": 49123, "\u0120Dirt": 49124, "\u0120423": 49125, "hhh": 49126, "\u0120Transformation": 49127, "QUIRE": 49128, "..................": 49129, "\u0120traveller": 49130, "\u0120Sexy": 49131, "\u0120Kern": 49132, "ipolar": 49133, "\u0120ransomware": 49134, "oooooooooooooooo": 49135, "Ec": 49136, "ruby": 49137, "Professional": 49138, "\u0120Outbreak": 49139, "argument": 49140, "Grey": 49141, "\u0120Fifa": 49142, "\u0120CHO": 49143, "\u0120FORM": 49144, "\u0120Amtrak": 49145, "-[": 49146, "\u0120cradle": 49147, "\u0120antioxidants": 49148, "\u00e3\u0123\u00ae\u00e5\u00ae": 49149, "736": 49150, "\u0120NASL": 49151, "\u0120Contributions": 49152, "Indiana": 49153, "\u0120STEP": 49154, "CSS": 49155, "\u0120salient": 49156, "\u0120allocations": 49157, "yrights": 49158, "\u0120mashed": 49159, "\u0120Cutter": 49160, "Sexual": 49161, "\u0120pounded": 49162, "\u0120fanbase": 49163, "\u0120casc": 49164, "\u0120Transparency": 49165, "\u0120analytic": 49166, "\u0120Summoner": 49167, "\u00d7\u0140": 49168, "\u0120ADC": 49169, "detail": 49170, "\u0120vanquished": 49171, "\u0120crabs": 49172, "arie": 49173, "Destroy": 49174, "\u0120Sack": 49175, "\u0120transistor": 49176, "Alabama": 49177, "\u0120Koen": 49178, "\u0120Fisheries": 49179, "cone": 49180, "\u0120annexed": 49181, "\u0120MGM": 49182, "esa": 49183, "\u0120faked": 49184, "\u0120Congratulations": 49185, "\u0120hindered": 49186, "\u0120correctional": 49187, "\u0120ITV": 49188, "leeve": 49189, "\u0120inappropriately": 49190, "licks": 49191, "\u0120trespass": 49192, "\u0120paws": 49193, "\u0120negotiator": 49194, "\u0120Christensen": 49195, "limits": 49196, "\u0120Dianne": 49197, "\u0120elegance": 49198, "\u0120Contracts": 49199, "anke": 49200, "Obj": 49201, "\u0120vigilance": 49202, "\u0120castles": 49203, "\u0120NAD": 49204, "\u0120Holo": 49205, "\u0120emphatically": 49206, "\u0120Titus": 49207, "\u0120Serving": 49208, "\u0120Richie": 49209, "\u0120Pigs": 49210, "568": 49211, "\u0120animosity": 49212, "\u0120Attributes": 49213, "\u0120Uriel": 49214, "MQ": 49215, "myra": 49216, "\u0120Applicant": 49217, "\u0120psychiatrists": 49218, "\u0120Vij": 49219, "\u0120Abby": 49220, "agree": 49221, "Push": 49222, "\u0120kWh": 49223, "hiba": 49224, "\u0120incite": 49225, "\u0120Weasley": 49226, "\u0120Taxi": 49227, "ministic": 49228, "hyper": 49229, "\u0120Farn": 49230, "\u0120601": 49231, "\u0120Nationwide": 49232, "Fake": 49233, "952": 49234, "\u0120maize": 49235, "\u0120interacted": 49236, "\u0120transitioned": 49237, "\u0120parasitic": 49238, "\u0120harmonic": 49239, "\u0120decaying": 49240, "\u0120baseless": 49241, "nsics": 49242, "\u0120transpired": 49243, "\u0120abundantly": 49244, "\u0120Forensic": 49245, "\u0120treadmill": 49246, "\u0120Jav": 49247, "aband": 49248, "\u0120sshd": 49249, "\u0120frontman": 49250, "\u0120Jakarta": 49251, "oller": 49252, "drops": 49253, "\u0120SERVICES": 49254, "romptu": 49255, "ophical": 49256, "hospital": 49257, "bledon": 49258, "645": 49259, "\u0120midrange": 49260, "\u0120EVENT": 49261, "culated": 49262, "rawled": 49263, "\u0120perched": 49264, "\u0120overboard": 49265, "\u0120Peel": 49266, "\u0120Pwr": 49267, "\u0120Carth": 49268, "\u0120COMPLE": 49269, "coe": 49270, "shall": 49271, "\u0120deterrence": 49272, "METHOD": 49273, "\u0120Absent": 49274, "MEN": 49275, "\u0120sill": 49276, "\u0120LEVEL": 49277, "York": 49278, "\u0120sinners": 49279, "\u0120OPEC": 49280, "\u0120Nur": 49281, "\u0120Designs": 49282, "selection": 49283, "\u0120unworthy": 49284, "CHA": 49285, "\u0120strengthens": 49286, "883": 49287, "edly": 49288, "\u0120slicing": 49289, "\u0120malnutrition": 49290, "\u0120filmmaking": 49291, "\u0120Polk": 49292, "urated": 49293, "\u0120421": 49294, "breakers": 49295, "!'\"": 49296, "\u0120wetlands": 49297, "\u0120Discrimination": 49298, "\u0120allowable": 49299, "\u0120steered": 49300, "\u0120Sicily": 49301, "SAM": 49302, "\u0120mustache": 49303, "\u0120mids": 49304, "\u0120clipped": 49305, "\u0120circulate": 49306, "\u0120brittle": 49307, "\u0120Buildings": 49308, "raised": 49309, "\u0120Roundup": 49310, "\u0120wealthier": 49311, "\u0120overwrite": 49312, "\u0120overpowered": 49313, "\u0120Gerrard": 49314, "sites": 49315, "PDATED": 49316, "\u0120acutely": 49317, "\u0120Gamble": 49318, "\u0120pim": 49319, "\u0120Kus": 49320, "Typically": 49321, "Deploy": 49322, "\u0120Moroccan": 49323, "potion": 49324, "combe": 49325, "\u0120vigilante": 49326, "\u0120363": 49327, "Stew": 49328, "\u0120Bagg": 49329, "\u0120resided": 49330, "\u0120Spo": 49331, "\u0120remnant": 49332, "\u0120emptiness": 49333, "brainer": 49334, "\u0120outpatient": 49335, "priority": 49336, "\u0120leptin": 49337, "\u0120Payton": 49338, "\u0120Gleaming": 49339, "\u0120Shed": 49340, "\u0120Polo": 49341, "\u0120Mormonism": 49342, "restricted": 49343, "arlane": 49344, "wx": 49345, "\u0120creatine": 49346, "\u0120Anon": 49347, "\u0120STUD": 49348, "\u0120JUL": 49349, "\u0120Tee": 49350, "528": 49351, "089": 49352, "\u0120hatched": 49353, "Dispatch": 49354, "\u0120Composite": 49355, "\u0120451": 49356, "puff": 49357, "\u0120XCOM": 49358, "\u0120Orn": 49359, "\u0120THANK": 49360, "ENDED": 49361, "\u0120Asheville": 49362, "\u0120\u00c3\u013e": 49363, "\u0120mango": 49364, "\u0120Slightly": 49365, "worldly": 49366, "\u0120Wander": 49367, "\u0120Expand": 49368, "\u0120Chr": 49369, "Mist": 49370, "\u0120orthodoxy": 49371, "\u0120UNESCO": 49372, "regate": 49373, "Elsewhere": 49374, "kie": 49375, "irled": 49376, "\u0120topple": 49377, "\u0120adoptive": 49378, "\u0120Legs": 49379, "dress": 49380, "\u0120Sagan": 49381, "bare": 49382, "\u0120Glou": 49383, "Crunch": 49384, "\u0120helpers": 49385, "\u0120chronically": 49386, "\u0120Huma": 49387, "10000": 49388, "\u0120accommodating": 49389, "\u00e4\u00ba\u0136": 49390, "\u0120wrinkles": 49391, "\u0120dodged": 49392, "fourth": 49393, "\u0120precon": 49394, "\u0120compressor": 49395, "\u0120Kare": 49396, "\u0120evict": 49397, "\u0120Warwick": 49398, "imar": 49399, "\u0120modernization": 49400, "\u0120bandwagon": 49401, "\u0120refuted": 49402, "\u0120netted": 49403, "\u0120Naples": 49404, "\u0120Genie": 49405, "perors": 49406, "\u0120fielded": 49407, "\u0120dere": 49408, "\u0120Parables": 49409, "lees": 49410, "\u0120trout": 49411, "aspers": 49412, "\u0120nihil": 49413, "\u0120happiest": 49414, "\u0120floppy": 49415, "\u0120Loft": 49416, "\u0120Heard": 49417, "\u0120unison": 49418, "\u0120lug": 49419, "\u0120Redmond": 49420, "classic": 49421, "Supporters": 49422, "SHIP": 49423, "GMT": 49424, "\u0120fuelled": 49425, "\u00e7\u0132": 49426, "\u0120dd": 49427, "\u0120Eminem": 49428, "\u01201897": 49429, "NYSE": 49430, "\u0120secretaries": 49431, "\u0120FIA": 49432, "\u0120Canaveral": 49433, "Favorite": 49434, "\u0120pomp": 49435, "\u0120detainee": 49436, "ership": 49437, "aimon": 49438, "iour": 49439, "\u0120Apex": 49440, "\u0120plantations": 49441, "amia": 49442, "acion": 49443, "Rust": 49444, "\u0120towed": 49445, "\u0120Truly": 49446, "577": 49447, "\u0120sheltered": 49448, "rider": 49449, "Wo": 49450, "\u0120lair": 49451, "\u0120Intelligent": 49452, "improve": 49453, "matically": 49454, "\u0120etiquette": 49455, "adra": 49456, "allo": 49457, "\u0120Juno": 49458, "anything": 49459, "\u0120Struggle": 49460, "\u0120Predict": 49461, "\u0120Grimes": 49462, "\u0120AMERICA": 49463, "ctx": 49464, "\u0120Situation": 49465, "WOOD": 49466, "\u0120soluble": 49467, "meier": 49468, "\u0120intolerable": 49469, "angering": 49470, "\u0120uninterrupted": 49471, "\u0120tooltip": 49472, "\u0120interrogated": 49473, "\u0120gunned": 49474, "\u0120Sneak": 49475, "\u00e6\u0143\u00a6": 49476, "\u0120tether": 49477, "\u0120crumble": 49478, "Lens": 49479, "\u0120clustered": 49480, "\u0120Syl": 49481, "\u0120Hasan": 49482, "\u0120dystopian": 49483, "wana": 49484, "\u0120joystick": 49485, "\u0120Thib": 49486, "ammu": 49487, "Tomorrow": 49488, "546": 49489, "\u0120overcame": 49490, "\u0120minimized": 49491, "ceptor": 49492, "Runner": 49493, "ENGTH": 49494, "\u0120Brenda": 49495, "\u0120Achievements": 49496, "\u0120torches": 49497, "\u0120rapport": 49498, "\u0120Investigator": 49499, "\u0120Handling": 49500, "relation": 49501, "grey": 49502, "815": 49503, "\u0120kcal": 49504, "\u0120Commands": 49505, "dq": 49506, "\u0120curls": 49507, "\u0120bearer": 49508, "\u0120cynicism": 49509, "itri": 49510, "\u0120Useful": 49511, "Bee": 49512, "DCS": 49513, "\u0120abras": 49514, "Pract": 49515, "BILITIES": 49516, "712": 49517, "\u0120debugger": 49518, "\u0120debtor": 49519, "\u0120Lia": 49520, "\u0120Kers": 49521, "\u0120exacerbate": 49522, "\u0120Stacy": 49523, "\u0120Bland": 49524, "\u0120Scenes": 49525, "\u0120branching": 49526, "\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a\u00e2\u0138\u012a": 49527, "apeake": 49528, "\u0120salsa": 49529, "\u0120mishand": 49530, "\u0120Konami": 49531, "\u0120Nib": 49532, "\u0120anecdote": 49533, "\u0120agreeable": 49534, "\u00cf\u012b": 49535, "\u0120Nathaniel": 49536, "\u0120Heisman": 49537, "\u0120Beware": 49538, "\u01201886": 49539, "spective": 49540, "691": 49541, "522": 49542, "\u0120inhibits": 49543, "\u0120hashing": 49544, "\u01201889": 49545, "\u00e5\u00b0\u0128": 49546, "vich": 49547, "Pure": 49548, "\u0120solidly": 49549, "\u0120aspirin": 49550, "imaru": 49551, "\u0120streetcar": 49552, "\u0120UCS": 49553, "\u0120Judd": 49554, "\u0120flashbacks": 49555, "pins": 49556, "\u01201440": 49557, "\u0120UNHCR": 49558, "\u0120Symptoms": 49559, "TIT": 49560, "538": 49561, "Fra": 49562, "%);": 49563, "\u0120ooz": 49564, "\u0120curfew": 49565, "\u0120calmed": 49566, "\u0120participates": 49567, "TeX": 49568, "\u0120nonsensical": 49569, "\u0120fullback": 49570, "\u0120DeL": 49571, "monkey": 49572, "hari": 49573, "\u0120metabolites": 49574, "\u0120looted": 49575, "\u0120ALWAYS": 49576, "\u0120BCC": 49577, "Lt": 49578, "ochet": 49579, "Bone": 49580, "\u0120vetoed": 49581, "\u0120gcc": 49582, "\u0120CLICK": 49583, "\u01201888": 49584, "saf": 49585, "\u0120stiffness": 49586, "\u0120lowly": 49587, "\u0120Geh": 49588, "verson": 49589, "orset": 49590, "\u0120unforeseen": 49591, "\u0120anesthesia": 49592, "\u0120Optical": 49593, "\u0120reconstructed": 49594, "\u0120Tup": 49595, "shows": 49596, "NEWS": 49597, "\u0120Newspaper": 49598, "\u0120ASA": 49599, "tera": 49600, "Numbers": 49601, "\u0120inexplicable": 49602, "\u00d7\u0133": 49603, "\u0120hardness": 49604, "untarily": 49605, "\u0120Acer": 49606, "gradient": 49607, "ARDIS": 49608, "\u0120woodland": 49609, "\u0120metaphors": 49610, "\u0120Wembley": 49611, "\u0120Pavel": 49612, "philis": 49613, "\u0120rewriting": 49614, "\u0120perceptual": 49615, "\u01201070": 49616, "worms": 49617, "\u0120Downs": 49618, "\u0120unsurprisingly": 49619, "\u0120tagging": 49620, "flame": 49621, "\u0120litres": 49622, "\u0120bounces": 49623, "\u0120Babe": 49624, "shut": 49625, "\u0120overdoses": 49626, "\u0120Sheila": 49627, "\u0120Chau": 49628, "\u0120Bless": 49629, "Capture": 49630, "\u0120Significant": 49631, "\u0120Scion": 49632, "\u0120389": 49633, "\u0120McH": 49634, "\u0120Titanium": 49635, "\u0120Meal": 49636, "ameda": 49637, "agents": 49638, "aggressive": 49639, "Billy": 49640, "763": 49641, "\u0120Saying": 49642, "DERR": 49643, "itone": 49644, "Collins": 49645, "Bound": 49646, "\u0120bolted": 49647, "\u0120DMCA": 49648, "953": 49649, "\u0120uniqueness": 49650, "\u0120epigen": 49651, "unci": 49652, "antam": 49653, "\u0120reckoning": 49654, "chairs": 49655, "OGR": 49656, "\u0120Senegal": 49657, "\u01201862": 49658, "relevant": 49659, "\u0120\u00c2\u00af": 49660, "\u0120pharmacies": 49661, "\u0120Geral": 49662, "vier": 49663, "Yan": 49664, "ORPG": 49665, "\u0120rabid": 49666, "bending": 49667, "\u0120UNITED": 49668, "\u0120465": 49669, "Assembly": 49670, "\u0120weep": 49671, "\u0120behest": 49672, "\u0120Mothers": 49673, "\u0120Jace": 49674, "hid": 49675, "\u0120whirlwind": 49676, "\u0120UNIVERS": 49677, "\u0120utopian": 49678, "\u0120kidnap": 49679, "Philipp": 49680, "Kin": 49681, "893": 49682, "\u0120livestream": 49683, "\u0120MISS": 49684, "\u0120subversive": 49685, "\u0120Techniques": 49686, "\u0120JUSTICE": 49687, "\u0120BASE": 49688, "\u0120387": 49689, "\u0120assailants": 49690, "\u0120Hardcore": 49691, "\u0120sprinkled": 49692, "\u0120Pse": 49693, "\u00e9\u013c": 49694, "printed": 49695, "\u0120Hau": 49696, "ORGE": 49697, "\u0120TOUR": 49698, "\u0120laced": 49699, "\u0120itch": 49700, "Giving": 49701, "\u0120ported": 49702, "781": 49703, "////////////////////////////////": 49704, "breeding": 49705, "\u0120logger": 49706, "\u0120HOL": 49707, "innie": 49708, "Firstly": 49709, "\u0120embryonic": 49710, "\u0120delegated": 49711, "pai": 49712, "OIL": 49713, "\u0120centrally": 49714, "\u0120Rx": 49715, "\u0120Scouting": 49716, "Dutch": 49717, "\u0120hereditary": 49718, "\u0120Cruiser": 49719, "sat": 49720, "529": 49721, "\u0120Marriott": 49722, "othermal": 49723, "\u0120prohibitions": 49724, "Earn": 49725, "\u0120Stab": 49726, "\u0120Colleges": 49727, "\u0120Belief": 49728, "stretched": 49729, "\u0120LH": 49730, "\u0120EntityItem": 49731, "CIA": 49732, "\u0120unrem": 49733, "\u0120laureate": 49734, "\u0120denominations": 49735, "summary": 49736, "hler": 49737, "Spect": 49738, "\u0120Klaus": 49739, "\u0120Beans": 49740, "\u0120insur": 49741, "\u0120PAX": 49742, "\u0120fielder": 49743, "\u0120Vet": 49744, "\u0120Sparrow": 49745, "zie": 49746, "\u0120SQ": 49747, "\u0120Mondays": 49748, "\u0120Offline": 49749, "\u0120Lerner": 49750, "\u0120Extensions": 49751, "Ireland": 49752, "\u0120patronage": 49753, "\u0120contrasted": 49754, "\u0120Mania": 49755, "hirt": 49756, "Moscow": 49757, "\u0120condemns": 49758, "\u0120Ange": 49759, "\u0120composing": 49760, "\u0120Pepe": 49761, "\u0120Paddock": 49762, "\u0120heterogeneity": 49763, "\u0120ideologically": 49764, "\u0120fishes": 49765, "\u0120cursing": 49766, "\u0120Rutherford": 49767, "\u0120Floating": 49768, "\u0120Amelia": 49769, "Tea": 49770, "Synopsis": 49771, "\u0120stunts": 49772, "\u0120bead": 49773, "\u0120stocking": 49774, "\u0120MILL": 49775, "obook": 49776, "massive": 49777, "\\<": 49778, "\u0120hump": 49779, "\u0120Preferences": 49780, "EngineDebug": 49781, "geist": 49782, "\u0120Nieto": 49783, "omever": 49784, "ishy": 49785, "evaluate": 49786, "colonial": 49787, "Alternative": 49788, "\u0120GoPro": 49789, "\u0120Vortex": 49790, "\u0120NETWORK": 49791, "ansky": 49792, "Secure": 49793, "\u0120Thrust": 49794, "Snake": 49795, "\u0120parcels": 49796, "\u0120samurai": 49797, "\u0120actresses": 49798, "Nap": 49799, "MF": 49800, "iferation": 49801, "Beer": 49802, "523": 49803, "\u0120Ily": 49804, "ointment": 49805, "Ping": 49806, "\u0120striped": 49807, "\u0120Mellon": 49808, "ossession": 49809, "\u0120neutron": 49810, "endium": 49811, "\u0120aph": 49812, "\u0120Flavoring": 49813, "\u0120383": 49814, "\u0120responsiveness": 49815, "\u0120Jindal": 49816, "\u0120Hitchcock": 49817, "Denver": 49818, "\u0120DRAGON": 49819, "smanship": 49820, "\u0120Dupl": 49821, "\u0120sly": 49822, "\u0120webcam": 49823, "\u0120Twain": 49824, "\u0120Darling": 49825, "iliate": 49826, "consumer": 49827, "DIT": 49828, "\u0120namesake": 49829, "\u0120unorthodox": 49830, "\u0120funer": 49831, "\u0120PLoS": 49832, "\u0120CONTROL": 49833, "ozyg": 49834, "oglobin": 49835, "FACE": 49836, "ERG": 49837, "\u0120Dia": 49838, "\u0120Fiesta": 49839, "cele": 49840, "034": 49841, "\u0120enclave": 49842, "\u00e2\u0138\u00ac\u00e2\u0138\u00ac": 49843, "onement": 49844, "alist": 49845, "Mand": 49846, "\u0120homegrown": 49847, "\u0120Fancy": 49848, "\u0120conceptions": 49849, "\u0120Contains": 49850, "ureen": 49851, "\u0120reiterate": 49852, "\u0120meager": 49853, "\u0120installments": 49854, "Spawn": 49855, "627": 49856, "\u0120photoc": 49857, "\u0120Cabrera": 49858, "\u0120Rosenthal": 49859, "\u0120Lansing": 49860, "isner": 49861, "\u0120invests": 49862, "\u0120UFOs": 49863, "EXP": 49864, "Hardware": 49865, "\u0120tragically": 49866, "\u0120concedes": 49867, "ieft": 49868, "cham": 49869, "borgh": 49870, "\u0120Schr": 49871, "\u0120Melanie": 49872, "\u0120Hoy": 49873, "\u0120visitation": 49874, "\u0120idiosyncr": 49875, "\u0120fractions": 49876, "\u0120foreskin": 49877, "obos": 49878, "\u0120poaching": 49879, "\u0120VIEW": 49880, "\u0120stimulates": 49881, "\u0120Gork": 49882, "canon": 49883, "MIC": 49884, "\u0120Nemesis": 49885, "\u0120Indra": 49886, "\u0120DMV": 49887, "\u0120529": 49888, "\u0120inspecting": 49889, "\u0120grandma": 49890, "\u0120Whedon": 49891, "\u0120Shant": 49892, "\u0120Purg": 49893, "ikan": 49894, "\u0120Teg": 49895, "\u0120CLR": 49896, "zac": 49897, "Victoria": 49898, "\u0120Verify": 49899, "ionics": 49900, "\u0120partying": 49901, "\u0120Mou": 49902, "colour": 49903, "\u0120testimonies": 49904, "lations": 49905, "\u0120pressuring": 49906, "hiro": 49907, "acers": 49908, "\u0120fid": 49909, "angler": 49910, "\u0120CSI": 49911, "\u0120hereafter": 49912, "\u0120dissidents": 49913, "reporting": 49914, "iphany": 49915, "chev": 49916, "\u0120solitude": 49917, "\u0120lobe": 49918, "\u0120indis": 49919, "\u0120credential": 49920, "recent": 49921, "adult": 49922, "\u0120Nirvana": 49923, "\u0120Franchise": 49924, "Layer": 49925, "Hyp": 49926, "\u0120Berkshire": 49927, "\u0120wills": 49928, "tif": 49929, "\u0120totem": 49930, "\u0120Judah": 49931, "repair": 49932, "Instant": 49933, "548": 49934, "\u0120embassies": 49935, "\u0120bottleneck": 49936, "\u0120bount": 49937, "\u0120typew": 49938, "\u0120Alvin": 49939, "jing": 49940, "imilar": 49941, "Rush": 49942, "\u0120brim": 49943, "\u0120HELP": 49944, "Aim": 49945, "]'": 49946, "\u0120passively": 49947, "\u0120bounded": 49948, "\u0120Rated": 49949, "\u0120criminality": 49950, "\u0120biomark": 49951, "\u0120dispatcher": 49952, "\u0120Towards": 49953, "\u0120+++": 49954, "righteous": 49955, "frog": 49956, "\u0120Panc": 49957, "Carter": 49958, "032": 49959, "\u00e6\u00a9\u0141": 49960, "\u0120ultraviolet": 49961, "\u0120Licensed": 49962, "\u0120Tata": 49963, "\u0120Blessing": 49964, "\u0120GAM": 49965, "\u0120chemically": 49966, "\u0120Seaf": 49967, "\u0120RELE": 49968, "\u0120Mercenary": 49969, "capitalist": 49970, "\u0120formulations": 49971, "\u0120annihilation": 49972, "\u0120Verb": 49973, "\u0120Argon": 49974, "\u0120unloaded": 49975, "\u0120morphed": 49976, "\u0120conquering": 49977, "backer": 49978, "IELD": 49979, "\u0120thefts": 49980, "\u0120frontrunner": 49981, "\u0120Royale": 49982, "\u0120Fundamental": 49983, "elight": 49984, "Chip": 49985, "necessary": 49986, "ayn": 49987, "\u0120Slip": 49988, "\u0120448": 49989, "cerned": 49990, "Pause": 49991, "\u0120shockingly": 49992, "\u0120ABV": 49993, "\u0120composure": 49994, "733": 49995, "\u0120Motorsport": 49996, "ahime": 49997, "Murray": 49998, "Mach": 49999, "\u0120grids": 50000, "\u0120debian": 50001, "\u0120furthermore": 50002, "\u0120dexterity": 50003, "\u0120Collections": 50004, "oslov": 50005, "ilage": 50006, "bj": 50007, "\u0120Monteneg": 50008, "\u0120strutConnector": 50009, "\u0120massacres": 50010, "\u0120briefs": 50011, "fetched": 50012, "uvian": 50013, "olition": 50014, "Failure": 50015, "emonic": 50016, "\u0120flared": 50017, "\u0120claimant": 50018, "\u0120cures": 50019, "\u0120giveaways": 50020, "\u0120Substance": 50021, "alions": 50022, "\u0120cringe": 50023, "\u0120Kul": 50024, "\u0120aristocracy": 50025, "\u0120Ulster": 50026, "olated": 50027, "housing": 50028, "\u0120MIS": 50029, "\u0120glared": 50030, "\u0120Wilhelm": 50031, "needs": 50032, "lambda": 50033, "builders": 50034, "\u0120VIS": 50035, "\u0120radiator": 50036, "\u0120Ghostbusters": 50037, "\u0120436": 50038, "actual": 50039, "\u0120herds": 50040, "\u00c3\u00a7a": 50041, "watching": 50042, "\u0120countering": 50043, "Charge": 50044, "\u0120charred": 50045, "\u0120warheads": 50046, "\u0120iodine": 50047, "\u0120Macy": 50048, "041": 50049, "\u0120departures": 50050, "\u0120Sins": 50051, "\u0120dyed": 50052, "\u0120Concepts": 50053, "gado": 50054, "713": 50055, "\u0120quotations": 50056, "\u0120gist": 50057, "\u0120Christy": 50058, "\u0120antigen": 50059, "\u0120Hemp": 50060, "\u0120Drawn": 50061, "\u0120Barg": 50062, "ezvous": 50063, "\u0120paternity": 50064, "\u0120ardu": 50065, "\u0120Anchorage": 50066, "\u0120Rik": 50067, "\u0120overloaded": 50068, "\u0120Username": 50069, "\u0120Tammy": 50070, "\u0120Nau": 50071, "\u0120Cellular": 50072, "\u0120waning": 50073, "\u0120rodent": 50074, "\u0120Worcester": 50075, "ilts": 50076, "\u0120Tad": 50077, "\u0120dwellings": 50078, "\u0120bullish": 50079, "431": 50080, "\u0120retaliate": 50081, "\u0120migraine": 50082, "\u0120Chevron": 50083, "CHECK": 50084, "\u0120donkey": 50085, "crim": 50086, "SPA": 50087, "\u0120Analog": 50088, "\u0120marquee": 50089, "\u0120Haas": 50090, "Bir": 50091, "\u0120GDDR": 50092, "\u0120Downloads": 50093, "\u0120willpower": 50094, "\u0120Forth": 50095, "\u0120Recorded": 50096, "\u0120impossibility": 50097, "\u0120Logged": 50098, "\u0120Franks": 50099, "\u0120Ratt": 50100, "initions": 50101, "\u0120cleaners": 50102, "\u0120sorely": 50103, "\u0120flickering": 50104, "\u0120Examination": 50105, "catching": 50106, "alloween": 50107, "Msg": 50108, "\u0120dunno": 50109, "Fa": 50110, "\u0120dysph": 50111, "crazy": 50112, ".''.": 50113, "\u0120mainline": 50114, "\u0120cs": 50115, "\u0120ptr": 50116, "\u0120Wally": 50117, "igun": 50118, "951": 50119, "\u0120Bigfoot": 50120, "fights": 50121, "\u0120retrieving": 50122, "Jr": 50123, "\u0120duplication": 50124, "\u0120Explan": 50125, "\u0120relational": 50126, "\u0120quaint": 50127, "\u0120biscuits": 50128, "\u0120ado": 50129, "\u0120shudder": 50130, "\u0120antidote": 50131, "blooded": 50132, "ksh": 50133, "\u0120sauces": 50134, "\u0120reinvest": 50135, "\u0120dispensary": 50136, "\u0120Diver": 50137, "\u01209000": 50138, "student": 50139, "\u0120insepar": 50140, "escap": 50141, "\u0120toddlers": 50142, "\u0120GPIO": 50143, "\u0120Assignment": 50144, "headers": 50145, "\u0120lackluster": 50146, "\u0120aback": 50147, "956": 50148, "\u0120toolbar": 50149, "745": 50150, "\u0120oust": 50151, "\u0120contemplation": 50152, "\u0120PRESIDENT": 50153, "\u0120458": 50154, "======": 50155, "\u0120guaranteeing": 50156, "\u0120Heist": 50157, "\u0120Cannes": 50158, "\u013b\u00bd": 50159, "\u0120collaborator": 50160, "\u0120Amp": 50161, "\u0120gou": 50162, "\u0120SHALL": 50163, "stories": 50164, "783": 50165, "\u0120mobilized": 50166, "\u0120brood": 50167, "\u0120LU": 50168, "\u0120\u00f0\u0141\u0133": 50169, "\u0120refin": 50170, "\u0120Anthropology": 50171, "vind": 50172, "illi": 50173, "\u0120warranties": 50174, "\u0120Babel": 50175, "\u0120swath": 50176, "\u0120caches": 50177, "\u0120antagonists": 50178, "artifacts": 50179, "\u0120hotly": 50180, "\u0120Starts": 50181, "\u0120G\u00c3\u00b6": 50182, "zag": 50183, "!!!!!": 50184, "\u0120scourge": 50185, "\u0120conspiring": 50186, "ruits": 50187, "reverse": 50188, "\u0120Sheen": 50189, "\u0120Jesuit": 50190, "\u0120Giovanni": 50191, "adies": 50192, "\u0120buttocks": 50193, "earcher": 50194, "acan": 50195, "\u0120volleyball": 50196, "\u0120shrouded": 50197, "\u0120scoreboard": 50198, "bats": 50199, "\u0120IPM": 50200, "\u0120asses": 50201, "\u0120deregulation": 50202, "\u0120Telegram": 50203, "\u0120Reboot": 50204, "\u01207000": 50205, "\u0120Canary": 50206, "\u0120kernels": 50207, "\u0120Fran\u00c3\u00a7ois": 50208, "\u0120Duff": 50209, "\u0120Pon": 50210, "\u0120Leica": 50211, "\u0120Garmin": 50212, "\u0120orphans": 50213, "\u0120Claudia": 50214, "\u0120calendars": 50215, "\u0120Leilan": 50216, "ento": 50217, "Rocket": 50218, "\u0120brunch": 50219, "\u0120Hawking": 50220, "ainers": 50221, "\u0120sensibilities": 50222, "\u0120kW": 50223, "\u0120Kand": 50224, "\u0120reclaimed": 50225, "\u0120interestingly": 50226, "\u00d7\u00a9": 50227, "romy": 50228, "JM": 50229, "\u0120Enhancement": 50230, "bush": 50231, "Skip": 50232, "\u0120rappers": 50233, "\u0120gazing": 50234, "pedia": 50235, "athlon": 50236, "Revolution": 50237, "\u0120snipers": 50238, "\u0120reverted": 50239, "\u0120conglomerate": 50240, "Terry": 50241, "794": 50242, "\u0120harsher": 50243, "\u0120desolate": 50244, "\u0120Hitman": 50245, "Commission": 50246, "\u0120(/": 50247, "\u00e2\u0122\u00a6.\"": 50248, "Compar": 50249, "\u0120amplification": 50250, "ominated": 50251, "\u0120regress": 50252, "\u0120Collider": 50253, "\u0120informants": 50254, "\u0120gazed": 50255, "<|endoftext|>": 50256} \ No newline at end of file diff --git a/projects/CLIP/CLIP.png b/projects/CLIP/CLIP.png new file mode 100644 index 0000000000000000000000000000000000000000..a1b5ec9171fd7a51e36e845a02304eb837142ba1 Binary files /dev/null and b/projects/CLIP/CLIP.png differ diff --git a/projects/CLIP/README.md b/projects/CLIP/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a55e5d4326383e953486f94132a8c44d5c44c493 --- /dev/null +++ b/projects/CLIP/README.md @@ -0,0 +1,41 @@ +# CLIP +Contributor{Xingyu.Liao: sherlockliao01@gmail.com} + +> NOTE: We only support inference right now. Stay tuned for training part. + +CLIP (Contrastive Language-Image Pre-Training) is a neural network trained on a variety of (image, text) pairs. It can be instructed in natural language to predict the most relevant text snippet, given an image, without directly optimizing for the task, similarly to the zero-shot capabilities of GPT-2 and 3. We found CLIP matches the performance of the original ResNet50 on ImageNet “zero-shot” without using any of the original 1.28M labeled examples, overcoming several major challenges in computer vision. + +## Approach + +![CLIP](CLIP.png) + + + +## Usage + +```python +import clip +import oneflow as flow +from PIL import Image + +device = "cuda" if flow.cuda.is_available() else "cpu" +model, preprocess = clip.load("ViT-B/32", device=device) + +image = ( + preprocess(Image.open("CLIP.png")) + .unsqueeze(0) + .to_global(sbp=flow.sbp.split(0), placement=flow.placement("cuda", ranks=[0])) +) +text = clip.tokenize(["a diagram", "a dog", "a cat"]).to_global( + sbp=flow.sbp.split(0), placement=flow.placement("cuda", ranks=[0]) +) + +with flow.no_grad(): + image_features = model.encode_image(image) + text_features = model.encode_text(text) + + logits_per_image, logits_per_text = model(image, text) + probs = logits_per_image.softmax(dim=-1).cpu().numpy() + +print("Label probs:", probs) # prints: [[0.9927937 0.00421068 0.00299572]] +``` diff --git a/projects/CLIP/clip/__init__.py b/projects/CLIP/clip/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..df955f9a05718ab37bf6ef88fa540c7140eaa043 --- /dev/null +++ b/projects/CLIP/clip/__init__.py @@ -0,0 +1 @@ +from .clip import load, tokenize diff --git a/projects/CLIP/clip/clip.py b/projects/CLIP/clip/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..89d55046759482478c4818cf467f7e5f116126a5 --- /dev/null +++ b/projects/CLIP/clip/clip.py @@ -0,0 +1,199 @@ +# -------------------------------------------------------- +# Borrow code from: +# https://github.com/openai/CLIP/tree/main/clip/clip.py +# -------------------------------------------------------- + +import hashlib +import os +import urllib +import warnings +from typing import List, Union + +import oneflow as flow +import torch +from flowvision.transforms import CenterCrop, Compose, Normalize, Resize, ToTensor +from PIL import Image +from tqdm import tqdm + +from .model import build_model +from .simple_tokenizer import SimpleTokenizer as _Tokenizer + +try: + from flowvision.transforms import InterpolationMode + + BICUBIC = InterpolationMode.BICUBIC +except ImportError: + BICUBIC = Image.BICUBIC + + +__all__ = ["available_models", "load", "tokenize"] +_tokenizer = _Tokenizer() + +# noqa: +_MODELS = { + "RN50": "https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt", # noqa: E501 + "RN101": "https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt", # noqa: E501 + "RN50x4": "https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt", # noqa: E501 + "RN50x16": "https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt", # noqa: E501 + "RN50x64": "https://openaipublic.azureedge.net/clip/models/be1cfb55d75a9666199fb2206c106743da0f6468c9d327f3e0d0a543a9919d9c/RN50x64.pt", # noqa: E501 + "ViT-B/32": "https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt", # noqa: E501 + "ViT-B/16": "https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt", # noqa: E501 + "ViT-L/14": "https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt", # noqa: E501 + "ViT-L/14@336px": "https://openaipublic.azureedge.net/clip/models/3035c92b350959924f9f00213499208652fc7ea050643e8b385c2dac08641f02/ViT-L-14-336px.pt", # noqa: E501 +} + + +def _download(url: str, root: str): + os.makedirs(root, exist_ok=True) + filename = os.path.basename(url) + + expected_sha256 = url.split("/")[-2] + download_target = os.path.join(root, filename) + + if os.path.exists(download_target) and not os.path.isfile(download_target): + raise RuntimeError(f"{download_target} exists and is not a regular file") + + if os.path.isfile(download_target): + if hashlib.sha256(open(download_target, "rb").read()).hexdigest() == expected_sha256: + return download_target + else: + warnings.warn( + f"{download_target} exists, but the SHA256 checksum does not match; " + "re-downloading the file" + ) + + with urllib.request.urlopen(url) as source, open(download_target, "wb") as output: + with tqdm( + total=int(source.info().get("Content-Length")), + ncols=80, + unit="iB", + unit_scale=True, + unit_divisor=1024, + ) as loop: + while True: + buffer = source.read(8192) + if not buffer: + break + + output.write(buffer) + loop.update(len(buffer)) + + if hashlib.sha256(open(download_target, "rb").read()).hexdigest() != expected_sha256: + raise RuntimeError("Model has been downloaded but the SHA256 checksum does not not match") + + return download_target + + +def _convert_image_to_rgb(image): + return image.convert("RGB") + + +def _transform(n_px): + return Compose( + [ + Resize(n_px, interpolation=BICUBIC), + CenterCrop(n_px), + _convert_image_to_rgb, + ToTensor(), + Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)), + ] + ) + + +def available_models() -> List[str]: + """Returns the names of available CLIP models""" + return list(_MODELS.keys()) + + +def load( + name: str, + device: Union[str, torch.device] = "cuda" if flow.cuda.is_available() else "cpu", + download_root: str = None, +): + """Load a CLIP model + + Parameters + ---------- + name : str + A model name listed by `clip.available_models()`, or the path to a + model checkpoint containing the state_dict + + device : Union[str, torch.device] + The device to put the loaded model + + download_root: str + path to download the model files; by default, it uses "~/.cache/clip" + + Returns + ------- + model : flow.nn.Module + The CLIP model + + preprocess : Callable[[PIL.Image], flow.Tensor] + A flowvision transform that converts a PIL image into a tensor that + the returned model can take as its input + """ + if name in _MODELS: + model_path = _download(_MODELS[name], download_root or os.path.expanduser("~/.cache/clip")) + elif os.path.isfile(name): + model_path = name + else: + raise RuntimeError(f"Model {name} not found; available models = {available_models()}") + + with open(model_path, "rb") as opened_file: + try: + # loading JIT archive + model = torch.jit.load(opened_file, map_location="cpu").eval() + state_dict = None + except RuntimeError: + # loading saved state dict + state_dict = torch.load(opened_file, map_location="cpu") + + model = build_model(state_dict or model.state_dict()).to(device) + if str(device) == "cpu": + model.float() + return model, _transform(model.visual.img_size) + + +def tokenize( + texts: Union[str, List[str]], context_length: int = 77, truncate: bool = False +) -> Union[flow.IntTensor, flow.LongTensor]: + """ + Returns the tokenized representation of given input string(s) + + Parameters + ---------- + texts : Union[str, List[str]] + An input string or a list of input strings to tokenize + + context_length : int + The context length to use; all CLIP models use 77 as the context length + + truncate: bool + Whether to truncate the text in case its encoding is longer than the context length + + Returns + ------- + A two-dimensional tensor containing the resulting tokens, + shape = [number of input strings, context_length]. + """ + if isinstance(texts, str): + texts = [texts] + + sot_token = _tokenizer.encoder["<|startoftext|>"] + eot_token = _tokenizer.encoder["<|endoftext|>"] + all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] for text in texts] + result = flow.zeros(len(all_tokens), context_length, dtype=flow.int) + + for i, tokens in enumerate(all_tokens): + if len(tokens) > context_length: + if truncate: + tokens = tokens[:context_length] + tokens[-1] = eot_token + else: + raise RuntimeError( + f"Input {texts[i]} is too long for context length {context_length}" + ) + result[i, : len(tokens)] = flow.tensor(tokens, dtype=flow.int) + + return result diff --git a/projects/CLIP/clip/model.py b/projects/CLIP/clip/model.py new file mode 100644 index 0000000000000000000000000000000000000000..39a0d34ad051ffbbbd5fa3ef17d67f760a278207 --- /dev/null +++ b/projects/CLIP/clip/model.py @@ -0,0 +1,718 @@ +# -------------------------------------------------------- +# Borrow code from: +# https://github.com/openai/CLIP/tree/main/clip/model.py +# -------------------------------------------------------- + +from collections import OrderedDict +from typing import Dict, Tuple, Union + +import numpy as np +import oneflow as flow +import torch +from oneflow import nn + +from libai.layers import MLP, Embedding, LayerNorm, Linear, MultiheadAttention, TransformerLayer +from libai.layers.activation import build_activation +from libai.layers.attention import AttnMaskType +from libai.models import VisionTransformer as ViT +from libai.utils import distributed as dist +from libai.utils.checkpoint import get_missing_parameters_message, get_unexpected_parameters_message + +from .ops import multi_head_attention_forward + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1): + super().__init__() + + # all conv layers have stride 1. an avgpool is performed + # after the second convolution when stride > 1 + self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.relu1 = nn.ReLU(inplace=True) + + self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.relu2 = nn.ReLU(inplace=True) + + self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity() + + self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.relu3 = nn.ReLU(inplace=True) + + self.downsample = None + self.stride = stride + + if stride > 1 or inplanes != planes * Bottleneck.expansion: + # downsampling layer is prepended with an avgpool, + # and the subsequent convolution has stride 1 + self.downsample = nn.Sequential( + OrderedDict( + [ + ("-1", nn.AvgPool2d(stride)), + ( + "0", + nn.Conv2d(inplanes, planes * self.expansion, 1, stride=1, bias=False), + ), + ("1", nn.BatchNorm2d(planes * self.expansion)), + ] + ) + ) + + def forward(self, x: flow.Tensor): + identity = x + + out = self.relu1(self.bn1(self.conv1(x))) + out = self.relu2(self.bn2(self.conv2(out))) + out = self.avgpool(out) + out = self.bn3(self.conv3(out)) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu3(out) + return out + + +class AttentionPool2d(nn.Module): + def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None): + super().__init__() + self.positional_embedding = nn.Parameter( + flow.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5 + ) + self.k_proj = nn.Linear(embed_dim, embed_dim) + self.q_proj = nn.Linear(embed_dim, embed_dim) + self.v_proj = nn.Linear(embed_dim, embed_dim) + self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) + self.num_heads = num_heads + + def forward(self, x): + x = x.reshape(x.shape[0], x.shape[1], x.shape[2] * x.shape[3]).permute( + 2, 0, 1 + ) # NCHW -> (HW)NC + x = flow.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC + x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC + x, _ = multi_head_attention_forward( + query=x, + key=x, + value=x, + embed_dim_to_check=x.shape[-1], + num_heads=self.num_heads, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + in_proj_weight=None, + in_proj_bias=flow.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]), + bias_k=None, + bias_v=None, + dropout_p=0, + out_proj_weight=self.c_proj.weight, + out_proj_bias=self.c_proj.bias, + use_separate_proj_weight=True, + training=self.training, + need_weights=False, + ) + + return x[0] + + +class ModifiedResNet(nn.Module): + """ + A ResNet class that is similar to flowvision's but contains the following changes: + - There are now 3 "stem" convolutions as opposed to 1, + with an average pool instead of a max pool. + - Performs anti-aliasing strided convolutions, where an avgpool is + prepended to convolutions with stride > 1 + - The final pooling layer is a QKV attention instead of an average pool + """ + + def __init__(self, layers, output_dim, heads, input_resolution=224, width=64): + super().__init__() + self.output_dim = output_dim + self.input_resolution = input_resolution + + # the 3-layer stem + self.conv1 = nn.Conv2d(3, width // 2, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(width // 2) + self.relu1 = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(width // 2, width // 2, kernel_size=3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(width // 2) + self.relu2 = nn.ReLU(inplace=True) + self.conv3 = nn.Conv2d(width // 2, width, kernel_size=3, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(width) + self.relu3 = nn.ReLU(inplace=True) + self.avgpool = nn.AvgPool2d(2) + + # residual layers + self._inplanes = width # this is a *mutable* variable used during construction + self.layer1 = self._make_layer(width, layers[0]) + self.layer2 = self._make_layer(width * 2, layers[1], stride=2) + self.layer3 = self._make_layer(width * 4, layers[2], stride=2) + self.layer4 = self._make_layer(width * 8, layers[3], stride=2) + + embed_dim = width * 32 # the ResNet feature dimension + self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, heads, output_dim) + + def _make_layer(self, planes, blocks, stride=1): + layers = [Bottleneck(self._inplanes, planes, stride)] + + self._inplanes = planes * Bottleneck.expansion + for _ in range(1, blocks): + layers.append(Bottleneck(self._inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + def stem(x): + x = self.relu1(self.bn1(self.conv1(x))) + x = self.relu2(self.bn2(self.conv2(x))) + x = self.relu3(self.bn3(self.conv3(x))) + x = self.avgpool(x) + return x + + x = x.to(dtype=self.conv1.weight.dtype) + x = stem(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.attnpool(x) + + return x + + +class MLPClip(MLP): + def __init__( + self, + hidden_size, + ffn_hidden_size, + output_dropout_prob=0, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + *, + layer_idx=0, + ): + super().__init__( + hidden_size, + ffn_hidden_size, + output_dropout_prob, + init_method, + output_layer_init_method, + bias_gelu_fusion, + bias_dropout_fusion, + layer_idx=layer_idx, + ) + if not bias_gelu_fusion: + self.activation_func = build_activation("quick_gelu") + + +class TransformerLayerClip(TransformerLayer): + def __init__( + self, + hidden_size, + ffn_hidden_size, + num_attention_heads, + is_decoder=False, + attention_dropout_prob=0, + output_dropout_prob=0, + drop_path_prob=0, + layernorm_epsilon=0.00001, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + apply_residual_post_layernorm=False, + attn_mask_type=AttnMaskType.padding, + *, + layer_idx=0, + ): + super().__init__( + hidden_size, + ffn_hidden_size, + num_attention_heads, + is_decoder, + attention_dropout_prob, + output_dropout_prob, + drop_path_prob, + layernorm_epsilon, + init_method, + output_layer_init_method, + bias_gelu_fusion, + bias_dropout_fusion, + scale_mask_softmax_fusion, + apply_query_key_layer_scaling, + apply_residual_post_layernorm, + attn_mask_type, + layer_idx=layer_idx, + ) + self.mlp = MLPClip( + self.hidden_size, + self.ffn_hidden_size, + self.output_dropout_prob, + self.init_method, + output_layer_init_method=self.output_layer_init_method, + bias_gelu_fusion=self.bias_gelu_fusion, + bias_dropout_fusion=self.bias_dropout_fusion, + layer_idx=self.layer_idx, + ) + + +class Transformer(nn.Module): + def __init__(self, width: int, layers: int, heads: int, attn_mask: flow.Tensor = None): + super().__init__() + self.width = width + self.layers = layers + self.attn_mask = attn_mask + self.resblocks = nn.ModuleList( + [TransformerLayerClip(width, 4 * width, heads, layer_idx=i) for i in range(layers)] + ) + + def forward(self, x: flow.Tensor): + for layer in self.resblocks: + x = layer(x, self.attn_mask) + return x + + +class VisionTransformer(ViT): + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=192, + depth=12, + num_heads=3, + mlp_ratio=4, + drop_rate=0, + attn_drop_rate=0, + drop_path_rate=0, + num_classes=1000, + loss_func=None, + ): + super().__init__( + img_size, + patch_size, + in_chans, + embed_dim, + depth, + num_heads, + mlp_ratio, + drop_rate, + attn_drop_rate, + drop_path_rate, + num_classes, + loss_func, + ) + + self.ln_pre = LayerNorm(embed_dim, layer_idx=0) + self.head = Linear(embed_dim, num_classes, bias=False, layer_idx=-1) + + def forward_features(self, x): + # patch embedding + x = self.patch_embed(x) + + cls_token = self.cls_token.expand( + x.shape[0], -1, -1 + ) # stole cls_tokens impl from Phil Wang, thanks + cls_token = cls_token.to_global(sbp=x.sbp, placement=cls_token.placement) + x = flow.cat((cls_token, x), dim=1) + + # position embedding + pos_embed = self.pos_embed.expand(x.shape[0], -1, -1) + pos_embed = pos_embed.to_global(sbp=x.sbp, placement=pos_embed.placement) + x = self.pos_drop(x + pos_embed) + + # layernorm_pre + x = self.ln_pre(x) + + # transformer block + x = self.blocks(x) + return x + + +class CLIP(nn.Module): + def __init__( + self, + embed_dim: int, + # vision + image_resolution: int, + vision_layers: Union[Tuple[int, int, int, int], int], + vision_width: int, + vision_patch_size: int, + # text + context_length: int, + vocab_size: int, + transformer_width: int, + transformer_heads: int, + transformer_layers: int, + ): + super().__init__() + + self.context_length = context_length + + if isinstance(vision_layers, (tuple, list)): + vision_heads = vision_width * 32 // 64 + self.visual = ModifiedResNet( + layers=vision_layers, + output_dim=embed_dim, + heads=vision_heads, + input_resolution=image_resolution, + width=vision_width, + ).to_global(sbp=flow.sbp.broadcast, placement=dist.get_layer_placement(0)) + else: + vision_heads = vision_width // 64 + self.visual = VisionTransformer( + img_size=image_resolution, + patch_size=vision_patch_size, + embed_dim=vision_width, + depth=vision_layers, + num_heads=vision_heads, + num_classes=embed_dim, + ) + + self.transformer = Transformer( + width=transformer_width, + layers=transformer_layers, + heads=transformer_heads, + attn_mask=self.build_attention_mask(), + ) + + self.vocab_size = vocab_size + self.token_embedding = Embedding(vocab_size, transformer_width) + self.positional_embedding = nn.Parameter( + flow.empty( + self.context_length, + transformer_width, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + ) + self.ln_final = LayerNorm((transformer_width,), layer_idx=-1) + + self.text_projection = nn.Parameter( + flow.empty( + transformer_width, + embed_dim, + sbp=flow.sbp.broadcast, + placement=dist.get_layer_placement(0), + ) + ) + self.logit_scale = nn.Parameter( + flow.ones([], sbp=flow.sbp.broadcast, placement=dist.get_layer_placement(0)) + * np.log(1 / 0.07) + ) + + self.initialize_parameters() + + def initialize_parameters(self): + if hasattr(self.visual, "patch_embed"): + nn.init.zeros_(self.visual.patch_embed.proj.bias) + nn.init.normal_(self.token_embedding.weight, std=0.02) + nn.init.normal_(self.positional_embedding, std=0.01) + + if isinstance(self.visual, ModifiedResNet): + if self.visual.attnpool is not None: + std = self.visual.attnpool.c_proj.in_features ** -0.5 + nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std) + + for resnet_block in [ + self.visual.layer1, + self.visual.layer2, + self.visual.layer3, + self.visual.layer4, + ]: + for name, param in resnet_block.named_parameters(): + if name.endswith("bn3.weight"): + nn.init.zeros_(param) + + proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5) + attn_std = self.transformer.width ** -0.5 + fc_std = (2 * self.transformer.width) ** -0.5 + for block in self.transformer.resblocks: + nn.init.normal_(block.self_attention.query_key_value.weight, std=attn_std) + nn.init.normal_(block.self_attention.dense.weight, std=proj_std) + nn.init.normal_(block.mlp.dense_h_to_4h.weight, std=fc_std) + nn.init.normal_(block.mlp.dense_4h_to_h.weight, std=proj_std) + + if self.text_projection is not None: + nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5) + + def build_attention_mask(self): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = flow.ones( + self.context_length, + self.context_length, + sbp=flow.sbp.broadcast, + placement=dist.get_layer_placement(0), + ) + mask = flow.tril(mask) # zero out the lower diagonal + return mask + + @property + def dtype(self): + return self.visual.conv1.weight.dtype + + def encode_image(self, image): + return self.visual(image)["prediction_scores"] + + def encode_text(self, text): + x = self.token_embedding(text) # [batch_size, n_ctx, d_model] + + x = x + self.positional_embedding + # x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + # x = x.permute(1, 0, 2) # LND -> NLD + x = self.ln_final(x) + + # x.shape = [batch_size, n_ctx, transformer.width] + # take features from the eot embedding (eot_token is the highest number in each sequence) + x = ( + x[flow.arange(x.shape[0], sbp=x.sbp, placement=x.placement), text.argmax(dim=-1)] + @ self.text_projection + ) + + return x + + def forward(self, image, text): + image_features = self.encode_image(image) + text_features = self.encode_text(text) + + # normalized features + image_features = image_features / image_features.norm(dim=1, keepdim=True) + text_features = text_features / text_features.norm(dim=1, keepdim=True) + + # cosine similarity as logits + logit_scale = self.logit_scale.exp() + logits_per_image = logit_scale * image_features @ text_features.t() + logits_per_text = logits_per_image.t() + + # shape = [global_batch_size, global_batch_size] + return logits_per_image, logits_per_text + + +def convert_weights(model: nn.Module): + """Convert applicable model parameters to fp16""" + + def _convert_weights_to_fp16(l): + if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Linear)): + l.weight.data = l.weight.data.to(dtype=flow.float16) + if l.bias is not None: + l.bias.data = l.bias.data.to(dtype=flow.float16) + + if isinstance(l, MultiheadAttention): + for attr in ["query_key_value", "dense"]: + layer = getattr(l, attr) + weight = getattr(layer, "weight") + if weight is not None: + weight.data = weight.data.to(dtype=flow.float16) + bias = getattr(layer, "bias") + if bias is not None: + bias.data = bias.data.to(dtype=flow.float16) + + if hasattr(l, "text_projection"): + attr = getattr(l, "text_projection") + if attr is not None: + attr.data = attr.data.to(dtype=flow.float16) + + if hasattr(l, "proj"): + attr = getattr(l, "proj") + if attr is not None: + attr.weight.data = attr.weight.data.to(dtype=flow.float16) + + model.apply(_convert_weights_to_fp16) + + +def load_tensor(tensor_lhs: flow.Tensor, tensor_rhs: torch.Tensor): + tensor_rhs = flow.Tensor( + tensor_rhs.cpu().numpy(), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=flow.env.all_device_placement("cuda"), + ).to_global(sbp=tensor_lhs.sbp, placement=tensor_lhs.placement) + tensor_lhs.data.copy_(tensor_rhs.data) + + +def load_weights(model: nn.Module, state_dict: Dict): + model_state_dict = model.state_dict() + incorrect_shapes = [] + for k in list(state_dict.keys()): + if k in model_state_dict: + shape_model = tuple(model_state_dict[k].shape) + shape_checkpoint = tuple(state_dict[k].shape) + if shape_model != shape_checkpoint: + incorrect_shapes.append((k, shape_checkpoint, shape_model)) + state_dict.pop(k) + + unexpected_keys = [] + for key, value in state_dict.items(): + if key not in model_state_dict: + unexpected_keys.append(key) + # skip this key + continue + model_state_dict.pop(key) + load_tensor(model.state_dict()[key], value) + + missing_keys = list(model_state_dict.keys()) + for k, shape_checkpoint, shape_model in incorrect_shapes: + print( + "Skip loading parameter '{}' to the model due to incompatible " + "shapes: {} in the checkpoint but {} in the " + "model! You might want to double check if this is expected.".format( + k, shape_checkpoint, shape_model + ) + ) + if missing_keys: + print(get_missing_parameters_message(missing_keys)) + if unexpected_keys: + print(get_unexpected_parameters_message(unexpected_keys)) + + +def convert_qkv_weight(qkv_weight, num_heads): + qkv_weight = qkv_weight.view([3, num_heads, 64, num_heads * 64]) + qkv_weight = ( + qkv_weight.permute(1, 0, 2, 3).contiguous().view(3 * num_heads * 64, num_heads * 64) + ) + return qkv_weight + + +def convert_qkv_bias(qkv_bias, num_heads): + qkv_bias = qkv_bias.view(3, num_heads, 64) + qkv_bias = qkv_bias.permute(1, 0, 2).contiguous().view(-1) + return qkv_bias + + +def change_vit_state_dict(state_dict, visual_num_heads, text_num_heads): + new_state_dict = {} + for key, value in state_dict.items(): + # change prefix + if "visual.transformer.resblocks" in key: + key = key.replace("visual.transformer.resblocks", "visual.blocks") + # change "ln_1" to "input_layernorm" + if "ln_1" in key: + key = key.replace("ln_1", "input_layernorm") + # change "ln_2" to "post_attention_layernorm" + if "ln_2" in key: + key = key.replace("ln_2", "post_attention_layernorm") + # change "attn.out_proj" to "attention.dense" + if "attn.out_proj" in key: + key = key.replace("attn.out_proj", "attention.dense") + # change "attn" to "attention.query_key_value" + if "attn.in_proj_weight" in key: + key = key.replace("attn.in_proj_weight", "attention.query_key_value.weight") + if "visual" not in key: + value = convert_qkv_weight(value, text_num_heads) + else: + value = convert_qkv_weight(value, visual_num_heads) + if "attn.in_proj_bias" in key: + key = key.replace("attn.in_proj_bias", "attention.query_key_value.bias") + if "visual" not in key: + value = convert_qkv_bias(value, text_num_heads) + else: + value = convert_qkv_bias(value, visual_num_heads) + # change "mlp.c_fc" to "mlp.dense_h_to_4h" + if "mlp.c_fc" in key: + key = key.replace("mlp.c_fc", "mlp.dense_h_to_4h") + # change "mlp.c_proj" to "mlp.dense_4h_to_h" + if "mlp.c_proj" in key: + key = key.replace("mlp.c_proj", "mlp.dense_4h_to_h") + + # change "class_embedding" to "cls_token" + if "class_embedding" in key: + key = key.replace("class_embedding", "cls_token") + value = value.unsqueeze(0).unsqueeze(0) + # change "pos_embed" to "positional_embedding" + if "visual.positional_embedding" == key: + key = "visual.pos_embed" + value = value.unsqueeze(0) + # change patch_embedding + if key == "visual.conv1.weight": + key = "visual.patch_embed.proj.weight" + # change "ln_post" + if "ln_post" in key: + key = key.replace("ln_post", "norm") + # change "proj" + if "visual.proj" == key: + key = "visual.head.weight" + value = value.transpose(0, 1) + + # added by huangwei + key = key.replace("attention.query_key_value", "self_attention.query_key_value").replace( + "attention.dense", "self_attention.dense" + ) + new_state_dict[key] = value + + return new_state_dict + + +def build_model(state_dict: dict): + vit = "visual.proj" in state_dict + + if vit: + vision_width = state_dict["visual.conv1.weight"].shape[0] + vision_layers = len( + [ + k + for k in state_dict.keys() + if k.startswith("visual.") and k.endswith(".attn.in_proj_weight") + ] + ) + vision_patch_size = state_dict["visual.conv1.weight"].shape[-1] + grid_size = round((state_dict["visual.positional_embedding"].shape[0] - 1) ** 0.5) + image_resolution = vision_patch_size * grid_size + else: + counts: list = [ + len(set(k.split(".")[2] for k in state_dict if k.startswith(f"visual.layer{b}"))) + for b in [1, 2, 3, 4] + ] + vision_layers = tuple(counts) + vision_width = state_dict["visual.layer1.0.conv1.weight"].shape[0] + output_width = round( + (state_dict["visual.attnpool.positional_embedding"].shape[0] - 1) ** 0.5 + ) + vision_patch_size = None + assert output_width ** 2 + 1 == state_dict["visual.attnpool.positional_embedding"].shape[0] + image_resolution = output_width * 32 + + embed_dim = state_dict["text_projection"].shape[1] + context_length = state_dict["positional_embedding"].shape[0] + vocab_size = state_dict["token_embedding.weight"].shape[0] + transformer_width = state_dict["ln_final.weight"].shape[0] + transformer_heads = transformer_width // 64 + transformer_layers = len( + set(k.split(".")[2] for k in state_dict if k.startswith("transformer.resblocks")) + ) + + if vit: + state_dict = change_vit_state_dict(state_dict, vision_width // 64, transformer_heads) + + model = CLIP( + embed_dim, + image_resolution, + vision_layers, + vision_width, + vision_patch_size, + context_length, + vocab_size, + transformer_width, + transformer_heads, + transformer_layers, + ) + + for key in ["input_resolution", "context_length", "vocab_size"]: + if key in state_dict: + del state_dict[key] + + # convert_weights(model) + load_weights(model, state_dict) + return model.eval() diff --git a/projects/CLIP/clip/ops.py b/projects/CLIP/clip/ops.py new file mode 100644 index 0000000000000000000000000000000000000000..f6d2d9871ec0b6a874aacd57134a3be4e5d2b5ce --- /dev/null +++ b/projects/CLIP/clip/ops.py @@ -0,0 +1,274 @@ +# -------------------------------------------------------- +# Reference: +# https://github.com/pyflow/pyflow/blob/1.7/flow/nn/functional.py#L4041 +# -------------------------------------------------------- + +import warnings +from typing import Optional, Tuple + +import oneflow as flow +import oneflow.nn.functional as F +from oneflow import Tensor + + +def multi_head_attention_forward( + query: Tensor, + key: Tensor, + value: Tensor, + embed_dim_to_check: int, + num_heads: int, + in_proj_weight: Tensor, + in_proj_bias: Tensor, + bias_k: Optional[Tensor], + bias_v: Optional[Tensor], + dropout_p: float, + out_proj_weight: Tensor, + out_proj_bias: Tensor, + training: bool = True, + key_padding_mask: Optional[Tensor] = None, + need_weights: bool = True, + attn_mask: Optional[Tensor] = None, + use_separate_proj_weight: bool = False, + q_proj_weight: Optional[Tensor] = None, + k_proj_weight: Optional[Tensor] = None, + v_proj_weight: Optional[Tensor] = None, + static_k: Optional[Tensor] = None, + static_v: Optional[Tensor] = None, +) -> Tuple[Tensor, Optional[Tensor]]: + r""" + Args: + query, key, value: map a query and a set of key-value pairs to an output. + See "Attention Is All You Need" for more details. + embed_dim_to_check: total dimension of the model. + num_heads: parallel attention heads. + in_proj_weight, in_proj_bias: input projection weight and bias. + bias_k, bias_v: bias of the key and value sequences to be added at dim=0. + add_zero_attn: add a new batch of zeros to the key and + value sequences at dim=1. + dropout_p: probability of an element to be zeroed. + out_proj_weight, out_proj_bias: the output projection weight and bias. + training: apply dropout if is ``True``. + key_padding_mask: if provided, specified padding elements in the key will + be ignored by the attention. This is an binary mask. When the value is True, + the corresponding value on the attention layer will be filled with -inf. + need_weights: output attn_output_weights. + attn_mask: 2D or 3D mask that prevents attention to certain positions. + A 2D mask will be broadcasted for all the batches while a 3D mask allows + to specify a different mask for the entries of each batch. + use_separate_proj_weight: the function accept the proj. weights for query, key, + and value in different forms. If false, in_proj_weight will be used, which is + a combination of q_proj_weight, k_proj_weight, v_proj_weight. + q_proj_weight, k_proj_weight, v_proj_weight, in_proj_bias: input projection weight and bias. + static_k, static_v: static key and value used for attention operators. + Shape: + Inputs: + - query: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is + the embedding dimension. + - key: :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is + the embedding dimension. + - value: :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is + the embedding dimension. + - key_padding_mask: :math:`(N, S)` where N is the batch size, S is the source + sequence length. + If a ByteTensor is provided, the non-zero positions will be ignored while + the zero positions will be unchanged. If a BoolTensor is provided, the positions + with the value of ``True`` will be ignored while the position with the value + of ``False`` will be unchanged. + - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, + S is the source sequence length. + 3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target + sequence length, S is the source sequence length. attn_mask ensures that position + i is allowed to attend the unmasked positions. + If a ByteTensor is provided, the non-zero positions are not allowed to attend + while the zero positions will be unchanged. If a BoolTensor is provided, positions + with ``True`` are not allowed to attend while ``False`` values will be unchanged. + If a FloatTensor is provided, it will be added to the attention weight. + - static_k: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length, + N is the batch size, E is the embedding dimension. E/num_heads is the head dimension. + - static_v: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length, + N is the batch size, E is the embedding dimension. E/num_heads is the head dimension. + Outputs: + - attn_output: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, + E is the embedding dimension. + - attn_output_weights: :math:`(N, L, S)` where N is the batch size, + L is the target sequence length, S is the source sequence length. + """ + tgt_len, bsz, embed_dim = query.size() + assert embed_dim == embed_dim_to_check + # allow MHA to have different sizes for the feature dimension + assert key.size(0) == value.size(0) and key.size(1) == value.size(1) + + head_dim = embed_dim // num_heads + assert head_dim * num_heads == embed_dim, "embed_dim must be divisible by num_heads" + scaling = float(head_dim) ** -0.5 + + if not use_separate_proj_weight: + if flow.equal(query, key) and flow.equal(key, value): + # self-attention + q, k, v = F.linear(query, in_proj_weight, in_proj_bias).chunk(3, dim=-1) + + elif flow.equal(key, value): + # encoder-decoder attention + # This is inline in_proj function with in_proj_weight and in_proj_bias + _b = in_proj_bias + _start = 0 + _end = embed_dim + _w = in_proj_weight[_start:_end, :] + if _b is not None: + _b = _b[_start:_end] + q = F.linear(query, _w, _b) + + if key is None: + assert value is None + k = None + v = None + else: + + # This is inline in_proj function with in_proj_weight and in_proj_bias + _b = in_proj_bias + _start = embed_dim + _end = None + _w = in_proj_weight[_start:, :] + if _b is not None: + _b = _b[_start:] + k, v = F.linear(key, _w, _b).chunk(2, dim=-1) + + else: + # This is inline in_proj function with in_proj_weight and in_proj_bias + _b = in_proj_bias + _start = 0 + _end = embed_dim + _w = in_proj_weight[_start:_end, :] + if _b is not None: + _b = _b[_start:_end] + q = F.linear(query, _w, _b) + + # This is inline in_proj function with in_proj_weight and in_proj_bias + _b = in_proj_bias + _start = embed_dim + _end = embed_dim * 2 + _w = in_proj_weight[_start:_end, :] + if _b is not None: + _b = _b[_start:_end] + k = F.linear(key, _w, _b) + + # This is inline in_proj function with in_proj_weight and in_proj_bias + _b = in_proj_bias + _start = embed_dim * 2 + _end = None + _w = in_proj_weight[_start:, :] + if _b is not None: + _b = _b[_start:] + v = F.linear(value, _w, _b) + else: + len1, len2 = q_proj_weight.size() + assert len1 == embed_dim and len2 == query.size(-1) + + len1, len2 = k_proj_weight.size() + assert len1 == embed_dim and len2 == key.size(-1) + + len1, len2 = v_proj_weight.size() + assert len1 == embed_dim and len2 == value.size(-1) + + if in_proj_bias is not None: + q = F.linear(query, q_proj_weight, in_proj_bias[0:embed_dim]) + k = F.linear(key, k_proj_weight, in_proj_bias[embed_dim : (embed_dim * 2)]) + v = F.linear(value, v_proj_weight, in_proj_bias[(embed_dim * 2) :]) + else: + q = F.linear(query, q_proj_weight, in_proj_bias) + k = F.linear(key, k_proj_weight, in_proj_bias) + v = F.linear(value, v_proj_weight, in_proj_bias) + q = q * scaling + + if attn_mask is not None: + assert ( + attn_mask.dtype == flow.float32 + or attn_mask.dtype == flow.float64 + or attn_mask.dtype == flow.float16 + or attn_mask.dtype == flow.uint8 + or attn_mask.dtype == flow.bool + ), "Only float, byte, and bool types are supported for attn_mask, not {}".format( + attn_mask.dtype + ) + if attn_mask.dtype == flow.uint8: + warnings.warn( + "Byte tensor for attn_mask in nn.MultiheadAttention is deprecated. " + "Use bool tensor instead." + ) + attn_mask = attn_mask.to(flow.bool) + + if attn_mask.dim() == 2: + attn_mask = attn_mask.unsqueeze(0) + if list(attn_mask.size()) != [1, query.size(0), key.size(0)]: + raise RuntimeError("The size of the 2D attn_mask is not correct.") + elif attn_mask.dim() == 3: + if list(attn_mask.size()) != [bsz * num_heads, query.size(0), key.size(0)]: + raise RuntimeError("The size of the 3D attn_mask is not correct.") + else: + raise RuntimeError("attn_mask's dimension {} is not supported".format(attn_mask.dim())) + # attn_mask's dim is 3 now. + + # convert ByteTensor key_padding_mask to bool + if key_padding_mask is not None and key_padding_mask.dtype == flow.uint8: + warnings.warn( + "Byte tensor for key_padding_mask in nn.MultiheadAttention is deprecated. " + "Use bool tensor instead." + ) + key_padding_mask = key_padding_mask.to(flow.bool) + + assert bias_k is None, "Only support bias_k is None" + assert bias_v is None, "Only support bias_v is None" + + q = q.contiguous().view(tgt_len, bsz * num_heads, head_dim).transpose(0, 1) + if k is not None: + k = k.contiguous().view(-1, bsz * num_heads, head_dim).transpose(0, 1) + if v is not None: + v = v.contiguous().view(-1, bsz * num_heads, head_dim).transpose(0, 1) + + if static_k is not None: + assert static_k.size(0) == bsz * num_heads + assert static_k.size(2) == head_dim + k = static_k + + if static_v is not None: + assert static_v.size(0) == bsz * num_heads + assert static_v.size(2) == head_dim + v = static_v + + src_len = k.size(1) + + if key_padding_mask is not None: + assert key_padding_mask.size(0) == bsz + assert key_padding_mask.size(1) == src_len + + attn_output_weights = flow.bmm(q, k.transpose(1, 2)) + assert list(attn_output_weights.size()) == [bsz * num_heads, tgt_len, src_len] + + if attn_mask is not None: + if attn_mask.dtype == flow.bool: + attn_output_weights.masked_fill_(attn_mask, float("-inf")) + else: + attn_output_weights += attn_mask + + if key_padding_mask is not None: + attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len) + attn_output_weights = attn_output_weights.masked_fill( + key_padding_mask.unsqueeze(1).unsqueeze(2), + float("-inf"), + ) + attn_output_weights = attn_output_weights.view(bsz * num_heads, tgt_len, src_len) + + attn_output_weights = F.softmax(attn_output_weights, dim=-1) + attn_output_weights = F.dropout(attn_output_weights, p=dropout_p, training=training) + + attn_output = flow.bmm(attn_output_weights, v) + assert list(attn_output.size()) == [bsz * num_heads, tgt_len, head_dim] + attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim) + attn_output = F.linear(attn_output, out_proj_weight, out_proj_bias) + + if need_weights: + # average attention weights over heads + attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len) + return attn_output, attn_output_weights.sum(dim=1) / num_heads + else: + return attn_output, None diff --git a/projects/CLIP/clip/simple_tokenizer.py b/projects/CLIP/clip/simple_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..f7ce129b61efab871a74aa63b4f56c3db57f4799 --- /dev/null +++ b/projects/CLIP/clip/simple_tokenizer.py @@ -0,0 +1,160 @@ +# -------------------------------------------------------- +# Borrow code from: +# https://github.com/openai/CLIP/tree/main/clip/simple_tokenizer.py +# -------------------------------------------------------- + +import gzip +import html +import os +from functools import lru_cache + +import ftfy +import regex as re + +from libai.utils.file_utils import download_file + + +@lru_cache() +def default_bpe(): + default_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz" + ) + if not os.path.exists(default_path): + download_file( + default_path, + "https://oneflow-static.oss-cn-beijing.aliyuncs.com/libai/clip/bpe_simple_vocab_16e6.txt.gz", # noqa: E501 + ) + return default_path + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you + want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing + around 5K for decent coverage. + This is a significant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = ( + list(range(ord("!"), ord("~") + 1)) + + list(range(ord("¡"), ord("¬") + 1)) + + list(range(ord("®"), ord("ÿ") + 1)) + ) + cs = bs[:] + n = 0 + for b in range(2 ** 8): + if b not in bs: + bs.append(b) + cs.append(2 ** 8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r"\s+", " ", text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + def __init__(self, bpe_path: str = default_bpe()): + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = gzip.open(bpe_path).read().decode("utf-8").split("\n") + merges = merges[1 : 49152 - 256 - 2 + 1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v + "" for v in vocab] + for merge in merges: + vocab.append("".join(merge)) + vocab.extend(["<|startoftext|>", "<|endoftext|>"]) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = {"<|startoftext|>": "<|startoftext|>", "<|endoftext|>": "<|endoftext|>"} + self.pat = re.compile( + r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", # noqa + re.IGNORECASE, + ) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + (token[-1] + "",) + pairs = get_pairs(word) + + if not pairs: + return token + "" + + while True: + bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf"))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except: # noqa + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word) - 1 and word[i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = " ".join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = "".join(self.byte_encoder[b] for b in token.encode("utf-8")) + bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(" ")) + return bpe_tokens + + def decode(self, tokens): + text = "".join([self.decoder[token] for token in tokens]) + text = ( + bytearray([self.byte_decoder[c] for c in text]) + .decode("utf-8", errors="replace") + .replace("", " ") + ) + return text diff --git a/projects/CLIP/requirements.txt b/projects/CLIP/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..e1bc89c06fc1fa14b75fed45f873f673fd204334 --- /dev/null +++ b/projects/CLIP/requirements.txt @@ -0,0 +1,7 @@ +ftfy +regex +tqdm +oneflow +flowvision +torch +torchvision diff --git a/projects/CLIP/tests/test_clip.py b/projects/CLIP/tests/test_clip.py new file mode 100644 index 0000000000000000000000000000000000000000..4a63940f9aec9f7cab86865f88b5015f547035a3 --- /dev/null +++ b/projects/CLIP/tests/test_clip.py @@ -0,0 +1,72 @@ +import os +import sys +import unittest + +import oneflow as flow + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) +from clip.model import CLIP, ModifiedResNet, Transformer # noqa: E402 + + +class TestCLIP(unittest.TestCase): + def test_modified_resnet(self): + net = ModifiedResNet([3, 4, 6, 3], 120, 16).to_global( + sbp=flow.sbp.broadcast, placement=flow.placement("cuda", ranks=[0]) + ) + + x = flow.rand( + 32, 3, 224, 224, sbp=flow.sbp.split(0), placement=flow.placement("cuda", ranks=[0]) + ) + y = net(x) + assert isinstance(y, flow.Tensor) + + def test_transformer(self): + mask = flow.ones( + 12, 12, sbp=flow.sbp.broadcast, placement=flow.placement("cuda", ranks=[0]) + ) + mask = flow.tril(mask) # zero out the lower diagonal + + # [1, 1, s, s] + mask = mask.unsqueeze(0).unsqueeze(1).expand(16, 1, 12, 12) + + net = Transformer(128, 10, 16, mask) + x = flow.rand( + 16, 12, 128, sbp=flow.sbp.split(0), placement=flow.placement("cuda", ranks=[0]) + ) + y = net(x) + assert isinstance(y, flow.Tensor) + + def test_clip(self): + # clip with resnet + net = CLIP( + embed_dim=10, + # vision + image_resolution=224, + vision_layers=6, + vision_width=120, + vision_patch_size=16, + # text + context_length=24, + vocab_size=3000, + transformer_width=128, + transformer_heads=16, + transformer_layers=10, + ) + img = flow.rand( + 16, 3, 224, 224, sbp=flow.sbp.split(0), placement=flow.placement("cuda", ranks=[0]) + ) + text = flow.ones( + 16, + 24, + dtype=flow.int, + sbp=flow.sbp.split(0), + placement=flow.placement("cuda", ranks=[0]), + ) + + logits_img, logits_text = net(img, text) + print(logits_img) + print(logits_text) + + +if __name__ == "__main__": + unittest.main() diff --git a/projects/CLIP/tests/test_multi_head_attn.py b/projects/CLIP/tests/test_multi_head_attn.py new file mode 100644 index 0000000000000000000000000000000000000000..4d3dd85832c64d8faeadcac5935aaacfe4edbeaf --- /dev/null +++ b/projects/CLIP/tests/test_multi_head_attn.py @@ -0,0 +1,91 @@ +import os +import sys +import unittest + +import numpy as np +import oneflow as flow +import torch +from torch.nn.functional import multi_head_attention_forward as multi_head_attention_forward_torch + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) +from clip.ops import multi_head_attention_forward as multi_head_attention_forward_flow # noqa: E402 + + +class TestMultiHeadAttention(unittest.TestCase): + def test_with_torch(self): + k_proj_weight = np.random.normal(size=(32, 32)) + k_proj_bias = np.random.normal(size=(32)) + + q_proj_weight = np.random.normal(size=(32, 32)) + q_proj_bias = np.random.normal(size=(32)) + + v_proj_weight = np.random.normal(size=(32, 32)) + v_proj_bias = np.random.normal(size=(32)) + + c_proj_weight = np.random.normal(size=(64, 32)) + c_proj_bias = np.random.normal(size=(64)) + + x = np.random.normal(size=(65, 16, 32)) + + x_torch = torch.from_numpy(x) + torch_out, _ = multi_head_attention_forward_torch( + query=x_torch, + key=x_torch, + value=x_torch, + embed_dim_to_check=x_torch.shape[-1], + num_heads=8, + q_proj_weight=torch.from_numpy(q_proj_weight), + k_proj_weight=torch.from_numpy(k_proj_weight), + v_proj_weight=torch.from_numpy(v_proj_weight), + in_proj_weight=None, + in_proj_bias=torch.cat( + [ + torch.from_numpy(q_proj_bias), + torch.from_numpy(k_proj_bias), + torch.from_numpy(v_proj_bias), + ] + ), + bias_k=None, + bias_v=None, + add_zero_attn=False, + dropout_p=0, + out_proj_weight=torch.from_numpy(c_proj_weight), + out_proj_bias=torch.from_numpy(c_proj_bias), + use_separate_proj_weight=True, + training=True, + need_weights=False, + ) + + x_flow = flow.from_numpy(x).cuda() + flow_out, _ = multi_head_attention_forward_flow( + query=x_flow, + key=x_flow, + value=x_flow, + embed_dim_to_check=x_flow.shape[-1], + num_heads=8, + q_proj_weight=flow.from_numpy(q_proj_weight).cuda(), + k_proj_weight=flow.from_numpy(k_proj_weight).cuda(), + v_proj_weight=flow.from_numpy(v_proj_weight).cuda(), + in_proj_weight=None, + in_proj_bias=flow.cat( + [ + flow.from_numpy(q_proj_bias).cuda(), + flow.from_numpy(k_proj_bias).cuda(), + flow.from_numpy(v_proj_bias).cuda(), + ] + ), + bias_k=None, + bias_v=None, + dropout_p=0, + out_proj_weight=flow.from_numpy(c_proj_weight).cuda(), + out_proj_bias=flow.from_numpy(c_proj_bias).cuda(), + use_separate_proj_weight=True, + training=True, + need_weights=False, + ) + + assert np.allclose(torch_out.numpy(), flow_out.numpy()) + + +if __name__ == "__main__": + unittest.main() diff --git a/projects/Couplets/configs/config.py b/projects/Couplets/configs/config.py new file mode 100644 index 0000000000000000000000000000000000000000..2b1f4bfa11b99c8c240f67bc8419f1724e984ddc --- /dev/null +++ b/projects/Couplets/configs/config.py @@ -0,0 +1,85 @@ +import os +import sys + +dir_path = os.path.abspath(os.path.dirname(__file__)) +dir_path = "/".join(dir_path.split("/")[:-1]) +sys.path.append(dir_path) + +from omegaconf import OmegaConf # noqa + +from dataset.dataset import CoupletsDataset # noqa +from modeling.model import Seq2Seq # noqa + +from libai.config import get_config # noqa +from libai.config import LazyCall # noqa +from libai.data.build import build_nlp_train_loader, build_nlp_test_loader # noqa + +optim = get_config("common/optim.py").optim +graph = get_config("common/models/graph.py").graph +train = get_config("common/train.py").train + +dataloader = OmegaConf.create() +dataloader.train = LazyCall(build_nlp_train_loader)( + dataset=[ + LazyCall(CoupletsDataset)( + path="data_test/couplets", + is_train=True, + maxlen=64, + ) + ], + num_workers=4, +) +dataloader.test = [ + LazyCall(build_nlp_test_loader)( + dataset=LazyCall(CoupletsDataset)( + path="data_test/couplets", + is_train=False, + maxlen=64, + ), + num_workers=4, + ) +] + +transformer_cfg = dict( + vocab_size=9027, + max_position_embeddings=64, + hidden_size=512, + intermediate_size=512, + hidden_layers=6, + num_attention_heads=8, + embedding_dropout_prob=0.1, + hidden_dropout_prob=0.1, + attention_dropout_prob=0.1, + initializer_range=0.02, + layernorm_epsilon=1e-5, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=True, +) +model = LazyCall(Seq2Seq)(cfg=transformer_cfg) + +train.update( + dict( + rdma_enabled=False, + recompute_grad=dict(enabled=False), + amp=dict(enabled=False), + output_dir="output/couplet/", + train_micro_batch_size=128, + test_micro_batch_size=32, + train_epoch=20, + train_iter=0, + log_period=10, + warmup_ratio=0.01, + dist=dict( + data_parallel_size=1, + tensor_parallel_size=1, + pipeline_parallel_size=1, + pipeline_stage_id=None, + pipeline_num_layers=model.cfg.hidden_layers * 2, + ), + evaluation=dict( + enabled=False, + ), + ) +) diff --git a/projects/Couplets/dataset/dataset.py b/projects/Couplets/dataset/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..56b5b29eabe26ddff81a7ed099536f91ee15d477 --- /dev/null +++ b/projects/Couplets/dataset/dataset.py @@ -0,0 +1,61 @@ +import os + +import oneflow as flow +from dataset.mask import make_padding_mask, make_sequence_mask +from oneflow.utils.data import Dataset +from tokenizer.tokenizer import CoupletsTokenizer + +from libai.data.structures import DistTensorData, Instance + + +class CoupletsDataset(Dataset): + def __init__(self, path, is_train=True, maxlen=64): + if is_train: + datapath = os.path.join(path, "train") + else: + datapath = os.path.join(path, "test") + + src = [] + with open(f"{datapath}/in.txt", "r") as f_src: + for line in f_src.readlines(): + src.append(line.strip("\n")) + tgt = [] + with open(f"{datapath}/out.txt", "r") as f_tgt: + for line in f_tgt.readlines(): + tgt.append(line.strip("\n")) + self.data = list(zip(src, tgt)) + self.tokenizer = CoupletsTokenizer(f"{path}/vocab.txt") + self.maxlen = maxlen + self.unk_id = self.tokenizer.unk_id + self.pad_id = self.tokenizer.pad_id + self.bos_id = self.tokenizer.bos_id + self.eos_id = self.tokenizer.eos_id + + def __len__(self): + return len(self.data) + + def text2ids(self, text): + tokens = self.tokenizer.tokenize(text) + ids = self.tokenizer.convert_tokens_to_ids(tokens) + ids = ids[: self.maxlen - 2] + ids = [self.bos_id] + ids + [self.eos_id] + ids = ids + [self.pad_id] * (self.maxlen - len(ids)) + return ids + + def __getitem__(self, index): + sample = self.data[index] + src_ids = self.text2ids(sample[0]) + tgt_ids = self.text2ids(sample[1]) + encoder_self_attn_mask = make_padding_mask(src_ids, src_ids, self.pad_id) + decoder_self_attn_mask = make_padding_mask( + tgt_ids, tgt_ids, self.pad_id + ) * make_sequence_mask(tgt_ids) + cross_attn_mask = make_padding_mask(tgt_ids, src_ids, self.pad_id) + + return Instance( + encoder_input_ids=DistTensorData(flow.tensor(src_ids, dtype=flow.long)), + decoder_input_ids=DistTensorData(flow.tensor(tgt_ids, dtype=flow.long)), + encoder_attn_mask=DistTensorData(flow.tensor(encoder_self_attn_mask, dtype=flow.long)), + decoder_attn_mask=DistTensorData(flow.tensor(decoder_self_attn_mask, dtype=flow.long)), + encoder_decoder_attn_mask=DistTensorData(flow.tensor(cross_attn_mask, dtype=flow.long)), + ) diff --git a/projects/Couplets/dataset/mask.py b/projects/Couplets/dataset/mask.py new file mode 100644 index 0000000000000000000000000000000000000000..fc10e92720cff4a5beaf36c9a25963fc2ee9e2ec --- /dev/null +++ b/projects/Couplets/dataset/mask.py @@ -0,0 +1,14 @@ +import numpy as np + + +def make_padding_mask(q_ids, kv_ids, pad_id): + q = (np.array(q_ids) != pad_id).reshape(-1, 1) + kv = (np.array(kv_ids) != pad_id).reshape(1, -1) + padding_mask = (q * kv).astype(float) + return padding_mask + + +def make_sequence_mask(ids): + seqlen = len(ids) + sequence_mask = np.triu(np.ones((seqlen, seqlen))).transpose() + return sequence_mask diff --git a/projects/Couplets/distribute_infer.py b/projects/Couplets/distribute_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..7b37bf0961771eff3e5187bfb0d8bb59a6a73e03 --- /dev/null +++ b/projects/Couplets/distribute_infer.py @@ -0,0 +1,110 @@ +import os +import sys + +dir_path = os.path.abspath(os.path.dirname(__file__)) # noqa +sys.path.append(dir_path) # noqa + +import oneflow as flow # noqa +from dataset.mask import make_sequence_mask # noqa +from tokenizer.tokenizer import CoupletsTokenizer # noqa + +from libai.data.structures import DistTensorData # noqa +from libai.inference.basic import BasePipeline # noqa +from libai.utils import distributed as dist # noqa + + +def get_global_tensor(rawdata): + t = flow.tensor(rawdata, dtype=flow.long).unsqueeze(0) + dtd = DistTensorData(t) + dtd.to_global() + return dtd.tensor + + +class CoupletPipeline(BasePipeline): + def _parse_parameters(self, **pipeline_parameters): + preprocess_params = {**pipeline_parameters} + forward_params = {} + postprocess_params = {} + + return preprocess_params, forward_params, postprocess_params + + def update_cfg( + self, + data_parallel=1, + tensor_parallel=1, + pipeline_parallel=1, + pipeline_stage_id=None, + pipeline_num_layers=None, + ): + super().update_cfg( + data_parallel, + tensor_parallel, + pipeline_parallel, + pipeline_stage_id, + pipeline_num_layers, + ) + self.cfg.vocab_file = "data_test/couplets/vocab.txt" + + def build_tokenizer(self, cfg): + return CoupletsTokenizer(cfg.vocab_file) + + def generate(self, sentence): + # Encode + sentence = " ".join([word for word in sentence]) + tokens_list = self.tokenizer.tokenize(sentence) + encoder_ids_list = ( + [self.tokenizer.bos_id] + + self.tokenizer.convert_tokens_to_ids(tokens_list) + + [self.tokenizer.eos_id] + ) + seq_len = len(encoder_ids_list) + encoder_input_ids = get_global_tensor(encoder_ids_list) + encoder_states = self.model.encode(encoder_input_ids, None) + + # Decode + decoder_ids_list = [self.tokenizer.bos_id] + decoder_input_ids = get_global_tensor(decoder_ids_list) + for i in range(seq_len + 10): + mask_array = make_sequence_mask(decoder_ids_list) + decoder_attn_mask = get_global_tensor(mask_array) + logits = self.model.decode(decoder_input_ids, decoder_attn_mask, encoder_states, None) + prob = logits[:, -1] + _, next_word = flow.max(prob, dim=1) + next_word = next_word.item() + decoder_ids_list = decoder_ids_list + [next_word] + decoder_input_ids = get_global_tensor(decoder_ids_list) + if next_word == self.tokenizer.eos_id: + break + result_tokens_list = self.tokenizer.convert_ids_to_tokens(decoder_ids_list) + + return "".join(result_tokens_list).replace("", "").replace("", "") + + def preprocess(self, sentence) -> dict: + input_dict = {"sentence": sentence} + return input_dict + + def forward(self, input_dict) -> dict: + model_output = self.generate(input_dict["sentence"]) + model_out_dict = {"下联": model_output} + return model_out_dict + + def postprocess(self, model_out_dict) -> dict: + return model_out_dict + + +if __name__ == "__main__": + + pipeline = CoupletPipeline( + "projects/Couplets/configs/config.py", + data_parallel=1, + tensor_parallel=1, + pipeline_parallel=4, + pipeline_stage_id=None, + pipeline_num_layers=12, + model_path="output/couplet/model_final/model", + mode="libai", + ) + + out = pipeline("滚滚长江东逝水") + if dist.is_main_process(): + print(out) diff --git a/projects/Couplets/infer.py b/projects/Couplets/infer.py new file mode 100644 index 0000000000000000000000000000000000000000..864d197071a307d05813c0a3e22f48bdf1faba46 --- /dev/null +++ b/projects/Couplets/infer.py @@ -0,0 +1,72 @@ +import os +import sys + +dir_path = os.path.abspath(os.path.dirname(__file__)) +sys.path.append(dir_path) + +import oneflow as flow # noqa +from dataset.mask import make_padding_mask, make_sequence_mask # noqa +from modeling.model import Seq2Seq # noqa +from tokenizer.tokenizer import CoupletsTokenizer # noqa + +from libai.config import LazyConfig # noqa +from libai.data.structures import DistTensorData # noqa +from libai.engine.default import DefaultTrainer # noqa +from libai.utils.checkpoint import Checkpointer # noqa + + +def get_global_tensor(rawdata): + t = flow.tensor(rawdata, dtype=flow.long).unsqueeze(0) + dtd = DistTensorData(t) + dtd.to_global() + return dtd.tensor + + +class GeneratorForEager: + def __init__(self, config_file, checkpoint_file, vocab_file): + cfg = LazyConfig.load(config_file) + self.model = DefaultTrainer.build_model(cfg).eval() + Checkpointer(self.model).load(checkpoint_file) + self.tokenizer = CoupletsTokenizer(vocab_file) + + def infer(self, sentence): + # Encode + sentence = " ".join([word for word in sentence]) + tokens_list = self.tokenizer.tokenize(sentence) + encoder_ids_list = ( + [self.tokenizer.bos_id] + + self.tokenizer.convert_tokens_to_ids(tokens_list) + + [self.tokenizer.eos_id] + ) + seq_len = len(encoder_ids_list) + encoder_input_ids = get_global_tensor(encoder_ids_list) + encoder_states = self.model.encode(encoder_input_ids, None) + + # Decode + decoder_ids_list = [self.tokenizer.bos_id] + decoder_input_ids = get_global_tensor(decoder_ids_list) + for i in range(seq_len + 10): + mask_array = make_sequence_mask(decoder_ids_list) + decoder_attn_mask = get_global_tensor(mask_array) + logits = self.model.decode(decoder_input_ids, decoder_attn_mask, encoder_states, None) + prob = logits[:, -1] + _, next_word = flow.max(prob, dim=1) + next_word = next_word.item() + decoder_ids_list = decoder_ids_list + [next_word] + decoder_input_ids = get_global_tensor(decoder_ids_list) + if next_word == self.tokenizer.eos_id: + break + result_tokens_list = self.tokenizer.convert_ids_to_tokens(decoder_ids_list) + + return "".join(result_tokens_list).replace("", "").replace("", "") + + +if __name__ == "__main__": + config_file = "output/couplet/config.yaml" + checkpoint_file = "output/couplet/model_final" + vocab_file = "data_test/couplets/vocab.txt" + generator = GeneratorForEager(config_file, checkpoint_file, vocab_file) + + sentence = input("上联：\n") + result = generator.infer(sentence) + print("下联：\n" + result) diff --git a/projects/Couplets/modeling/model.py b/projects/Couplets/modeling/model.py new file mode 100644 index 0000000000000000000000000000000000000000..5911de8b592ca8069141e356bf77e97f13537144 --- /dev/null +++ b/projects/Couplets/modeling/model.py @@ -0,0 +1,148 @@ +import oneflow as flow +from oneflow import nn + +from libai.layers.cross_entropy import ParallelCrossEntropyLoss +from libai.utils import distributed as dist + +from .transformer_model import TransformerModel + + +class Seq2SeqLoss(nn.Module): + def __init__(self): + super().__init__() + self.lm_loss = ParallelCrossEntropyLoss() + + def forward(self, logits, lm_labels): + logits = logits[:, :-1, :] + lm_labels = lm_labels[:, 1:] + lm_loss = self.lm_loss(logits, lm_labels) + lm_loss = lm_loss.mean() + return lm_loss + + +class Seq2Seq(nn.Module): + def __init__(self, cfg): + super().__init__() + self.language_model = TransformerModel(cfg) + self.loss_func = Seq2SeqLoss() + + def forward( + self, + encoder_input_ids, + decoder_input_ids, + encoder_attn_mask, + decoder_attn_mask, + encoder_decoder_attn_mask, + ): + logits = self.language_model( + encoder_input_ids, + decoder_input_ids, + encoder_attn_mask, + decoder_attn_mask, + encoder_decoder_attn_mask, + ) + + if self.training: + loss = self.loss_func(logits, decoder_input_ids) + return {"total_loss": loss} + + logits = logits.view(-1, logits.shape[-1]) + return {"prediction_scores": logits} + + def encode( + self, + encoder_input_ids, + encoder_attn_mask, + ): + encoder_input_embeddings = self.language_model.embedding(encoder_input_ids) + if encoder_attn_mask is not None: + encoder_extended_attn_mask = self.language_model.extended_attn_mask(encoder_attn_mask) + encoder_states = self.language_model.encoder( + encoder_input_embeddings, + encoder_extended_attn_mask, + ) + else: + encoder_states = self.language_model.encoder( + encoder_input_embeddings, + None, + ) + return encoder_states + + def decode( + self, + decoder_input_ids, + decoder_attn_mask, + encoder_states, + encoder_decoder_attn_mask, + ): + decoder_input_embeddings = self.language_model.embedding(decoder_input_ids) + decoder_extended_attn_mask = self.language_model.extended_attn_mask(decoder_attn_mask) + if encoder_decoder_attn_mask is not None: + encoder_decoder_extended_attn_mask = self.language_model.extended_attn_mask( + encoder_decoder_attn_mask + ) + decoder_states = self.language_model.decoder( + decoder_input_embeddings, + decoder_extended_attn_mask, + encoder_states, + encoder_decoder_extended_attn_mask, + ) + else: + decoder_states = self.language_model.decoder( + decoder_input_embeddings, + decoder_extended_attn_mask, + encoder_states, + None, + ) + logits = self.language_model.lm_head(decoder_states) + return logits + + @staticmethod + def set_pipeline_stage_id(model): + dist_utils = dist.get_dist_util() + + from .transformer_model import ExtendedMask, TransformerEmbedding, TransformerLayer + + # Set pipeline parallelism stage_id + if hasattr(model.language_model.lm_head, "config"): + # Old API in OneFlow 0.8 + for module_block in model.modules(): + # module.origin can get the original module + if isinstance(module_block.origin, TransformerEmbedding): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, ExtendedMask): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, TransformerLayer): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + + # Set the lm_head stage id + model.language_model.lm_head.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + else: + for module_block in model.modules(): + if isinstance(module_block.to(nn.Module), TransformerEmbedding): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), ExtendedMask): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + + # Set the lm_head stage id + model.language_model.lm_head.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) diff --git a/projects/Couplets/modeling/transformer_model.py b/projects/Couplets/modeling/transformer_model.py new file mode 100644 index 0000000000000000000000000000000000000000..8132d56e5dea2ea15141fd296c95ea7067c0901e --- /dev/null +++ b/projects/Couplets/modeling/transformer_model.py @@ -0,0 +1,289 @@ +import math + +import oneflow as flow +from oneflow import nn + +from libai.config import configurable +from libai.layers import ( + LayerNorm, + Linear, + SinePositionalEmbedding, + TransformerLayer, + VocabEmbedding, +) +from libai.models.utils import init_method_normal, scaled_init_method_normal +from libai.utils import distributed as dist + + +class ExtendedMask(flow.nn.Module): + def forward(self, x): + return x.unsqueeze(1) + + +class TransformerEmbedding(nn.Module): + def __init__( + self, + vocab_size, + hidden_size, + max_sequence_length, + embedding_dropout_prob, + init_method=nn.init.xavier_normal_, + ): + super().__init__() + + self.hidden_size = hidden_size + self.vocab_size = vocab_size + + self.word_embedding = VocabEmbedding(vocab_size, hidden_size, init_method=init_method) + self.positional_encoding = SinePositionalEmbedding(max_sequence_length, hidden_size) + self.position_ids = flow.arange( + max_sequence_length, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ).unsqueeze(0) + self.embedding_dropout = nn.Dropout(embedding_dropout_prob) + + def forward(self, input_ids): + seq_length = input_ids.size()[1] + + word_embeddings = self.word_embedding(input_ids) + position_ids = ( + self.position_ids[:, :seq_length].expand_as(input_ids).to_global(sbp=input_ids.sbp) + ) + positional_encodings = self.positional_encoding(position_ids) + embeddings = word_embeddings * math.sqrt(self.hidden_size) + positional_encodings + embeddings = self.embedding_dropout(embeddings) + return embeddings + + +class TransformerEncoder(nn.Module): + def __init__( + self, + hidden_size=512, + ffn_hidden_size=512, + hidden_layers=6, + num_attention_heads=8, + is_decoder=False, + initializer_range=0.02, + attention_dropout_prob=0.1, + output_dropout_prob=0.1, + layernorm_epsilon=1e-5, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=True, + ): + super().__init__() + self.encoder_layers = nn.ModuleList( + [ + TransformerLayer( + hidden_size=hidden_size, + ffn_hidden_size=ffn_hidden_size, + num_attention_heads=num_attention_heads, + is_decoder=is_decoder, + attention_dropout_prob=attention_dropout_prob, + output_dropout_prob=output_dropout_prob, + layernorm_epsilon=layernorm_epsilon, + bias_gelu_fusion=bias_gelu_fusion, + bias_dropout_fusion=bias_dropout_fusion, + scale_mask_softmax_fusion=scale_mask_softmax_fusion, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + init_method=init_method_normal(initializer_range), + output_layer_init_method=scaled_init_method_normal( + initializer_range, hidden_layers + ), + layer_idx=i, + ) + for i in range(hidden_layers) + ] + ) + self.encoder_final_layernorm = LayerNorm( + (hidden_size,), eps=layernorm_epsilon, layer_idx=hidden_layers - 1 + ) + + def forward(self, encoder_input_embeddings, encoder_extended_attn_mask): + enc_hidden_states = encoder_input_embeddings + for layer in self.encoder_layers: + enc_hidden_states = layer(enc_hidden_states, encoder_extended_attn_mask) + encoder_states = self.encoder_final_layernorm(enc_hidden_states) + return encoder_states + + +class TransformerDecoder(nn.Module): + def __init__( + self, + hidden_size=512, + ffn_hidden_size=512, + hidden_layers=6, + num_attention_heads=8, + is_decoder=True, + initializer_range=0.02, + attention_dropout_prob=0.1, + output_dropout_prob=0.1, + layernorm_epsilon=1e-5, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=True, + ): + super().__init__() + self.decoder_layers = nn.ModuleList( + [ + TransformerLayer( + hidden_size=hidden_size, + ffn_hidden_size=ffn_hidden_size, + num_attention_heads=num_attention_heads, + is_decoder=is_decoder, + attention_dropout_prob=attention_dropout_prob, + output_dropout_prob=output_dropout_prob, + layernorm_epsilon=layernorm_epsilon, + bias_gelu_fusion=bias_gelu_fusion, + bias_dropout_fusion=bias_dropout_fusion, + scale_mask_softmax_fusion=scale_mask_softmax_fusion, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + init_method=init_method_normal(initializer_range), + output_layer_init_method=scaled_init_method_normal( + initializer_range, hidden_layers + ), + layer_idx=i, + ) + for i in range(hidden_layers, 2 * hidden_layers) + ] + ) + self.decoder_final_layernorm = LayerNorm( + (hidden_size,), eps=layernorm_epsilon, layer_idx=2 * hidden_layers - 1 + ) + + def forward( + self, + decoder_input_embeddings, + decoder_extended_attn_mask, + encoder_states, + encoder_decoder_extended_attn_mask, + ): + dec_hidden_states = decoder_input_embeddings + for layer in self.decoder_layers: + dec_hidden_states = layer( + dec_hidden_states, + decoder_extended_attn_mask, + encoder_states, + encoder_decoder_extended_attn_mask, + ) + decoder_states = self.decoder_final_layernorm(dec_hidden_states) + return decoder_states + + +class TransformerModel(nn.Module): + @configurable + def __init__( + self, + vocab_size, + max_position_embeddings, + hidden_size=512, + intermediate_size=512, + hidden_layers=6, + num_attention_heads=8, + embedding_dropout_prob=0.1, + hidden_dropout_prob=0.1, + attention_dropout_prob=0.1, + initializer_range=0.02, + layernorm_epsilon=1e-5, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=True, + ): + super().__init__() + + self.embedding = TransformerEmbedding( + vocab_size, + hidden_size, + max_position_embeddings, + embedding_dropout_prob, + init_method=init_method_normal(initializer_range), + ) + + self.extended_attn_mask = ExtendedMask() + + self.encoder = TransformerEncoder( + hidden_size=hidden_size, + ffn_hidden_size=intermediate_size, + hidden_layers=hidden_layers, + num_attention_heads=num_attention_heads, + is_decoder=False, + initializer_range=0.02, + attention_dropout_prob=attention_dropout_prob, + output_dropout_prob=hidden_dropout_prob, + layernorm_epsilon=layernorm_epsilon, + bias_gelu_fusion=bias_gelu_fusion, + bias_dropout_fusion=bias_dropout_fusion, + scale_mask_softmax_fusion=scale_mask_softmax_fusion, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + ) + + self.decoder = TransformerDecoder( + hidden_size=hidden_size, + ffn_hidden_size=intermediate_size, + hidden_layers=hidden_layers, + num_attention_heads=num_attention_heads, + is_decoder=True, + initializer_range=0.02, + attention_dropout_prob=attention_dropout_prob, + output_dropout_prob=hidden_dropout_prob, + layernorm_epsilon=layernorm_epsilon, + bias_gelu_fusion=bias_gelu_fusion, + bias_dropout_fusion=bias_dropout_fusion, + scale_mask_softmax_fusion=scale_mask_softmax_fusion, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + ) + + self.lm_head = Linear( + hidden_size, + vocab_size, + layer_idx=-1, + ) + + @classmethod + def from_config(cls, cfg): + return { + "vocab_size": cfg.vocab_size, + "max_position_embeddings": cfg.max_position_embeddings, + "hidden_size": cfg.hidden_size, + "intermediate_size": cfg.intermediate_size, + "hidden_layers": cfg.hidden_layers, + "num_attention_heads": cfg.num_attention_heads, + "embedding_dropout_prob": cfg.embedding_dropout_prob, + "hidden_dropout_prob": cfg.hidden_dropout_prob, + "attention_dropout_prob": cfg.attention_dropout_prob, + "initializer_range": cfg.initializer_range, + "layernorm_epsilon": cfg.layernorm_epsilon, + "bias_gelu_fusion": cfg.bias_gelu_fusion, + "bias_dropout_fusion": cfg.bias_dropout_fusion, + "scale_mask_softmax_fusion": cfg.scale_mask_softmax_fusion, + "apply_query_key_layer_scaling": cfg.apply_query_key_layer_scaling, + } + + def forward( + self, + encoder_input_ids, + decoder_input_ids, + encoder_attn_mask, + decoder_attn_mask, + encoder_decoder_attn_mask, + ): + encoder_input_embeddings = self.embedding(encoder_input_ids) + decoder_input_embeddings = self.embedding(decoder_input_ids) + encoder_extended_attn_mask = self.extended_attn_mask(encoder_attn_mask) + decoder_extended_attn_mask = self.extended_attn_mask(decoder_attn_mask) + encoder_decoder_extended_attn_mask = self.extended_attn_mask(encoder_decoder_attn_mask) + + encoder_states = self.encoder(encoder_input_embeddings, encoder_extended_attn_mask) + decoder_states = self.decoder( + decoder_input_embeddings, + decoder_extended_attn_mask, + encoder_states, + encoder_decoder_extended_attn_mask, + ) + logits = self.lm_head(decoder_states) + return logits diff --git a/projects/Couplets/readme.md b/projects/Couplets/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..9e31b92eb0e0779d0191b6a9d9c73ae5fd39a54e --- /dev/null +++ b/projects/Couplets/readme.md @@ -0,0 +1,191 @@ +## Couplets in LiBai + +Contributor{Yulin Zhuang: [https://github.com/ZylOo0](https://github.com/ZylOo0)} + +This is the LiBai implementation of Couplets + +## Supported parallel mode and task +Based on [libai.layers](https://libai.readthedocs.io/en/latest/modules/libai.layers.html), Couplets model is automatically configured with the following parallelism mode. + + + + + + + + + + + + + + + + + + + + + + +
Model Data ParallelTensor ParallelPipeline Parallel
Couplets training ✔✔✔
Couplets inference - ✔✔
+ +## Setup env + +Install in LiBai, refer to [LiBai install doc](https://libai.readthedocs.io/en/latest/tutorials/get_started/Installation.html) + +## Prepare Datasets + +Download datasets and unzip data: +```shell +wget http://oneflow-static.oss-cn-beijing.aliyuncs.com/libai/couplets/couplets.zip +unzip couplets.zip +``` + +you will get dataset like this: +``` +couplets +├── test +│ ├── in.txt +│ └── out.txt +├── train +│ ├── in.txt +│ └── out.txt +└── vocab.txt +``` + +## Training + +- Set dataset path in `configs/config.py` + + ```python + dataloader.train = LazyCall(build_nlp_train_loader)( + dataset=[ + LazyCall(CoupletsDataset)( + path="data_test/couplets", # set to your data_path + is_train=True, + maxlen=64, + ) + ], + num_workers=4, + ) + dataloader.test = [ + LazyCall(build_nlp_test_loader)( + dataset=LazyCall(CoupletsDataset)( + path="data_test/couplets", # set to your data_path + is_train=False, + maxlen=64, + ), + num_workers=4, + ) + ] + ``` +- set model cfg in `configs/config.py` according to your needs + ```python + transformer_cfg = dict( + vocab_size=9027, + max_position_embeddings=64, + hidden_size=512, # modify it according to your needs + intermediate_size=512, # modify it according to your needs + hidden_layers=6, # modify it according to your needs + num_attention_heads=8, # modify it according to your needs + embedding_dropout_prob=0.1, + hidden_dropout_prob=0.1, + attention_dropout_prob=0.1, + initializer_range=0.02, + layernorm_epsilon=1e-5, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=True, + ) + ``` +- set dist config in `configs/config.py` according to your needs, refer to [LiBai distribute doc](https://libai.readthedocs.io/en/latest/tutorials/basics/Distributed_Configuration.html) for more details + ```python + dist=dict( + data_parallel_size=1, # modify it according to your needs + tensor_parallel_size=1, # modify it according to your needs + pipeline_parallel_size=1, # modify it according to your needs + pipeline_stage_id=None, # modify it according to your needs + pipeline_num_layers=model.cfg.hidden_layers * 2, + ), + ``` + +- Following [quick_run](https://libai.readthedocs.io/en/latest/tutorials/get_started/quick_run.html) in LiBai, run training command in LiBai **root** dir + ```shell + # cd path to libai + bash tools/train.sh tools/train_net.py projects/Couplets/configs/config.py 4 + ``` + +- After finish training, you will get trained model in path `output/couplet` + ```shell + output/couplet + ├── config.yaml + ├── last_checkpoint + ├── log.txt + ├── log.txt.rank1 + ├── log.txt.rank2 + ├── log.txt.rank3 + ├── metrics.json + ├── model_0004999 + ├── model_0009999 + ├── model_0014999 + ├── model_0019999 + ├── model_0024999 + └── model_final + ``` + +## inference + +- for inference in one gpu: + ``` + # modify path in projects/Couplets/infer.py + # config_file = "output/couplet/config.yaml" + # checkpoint_file = "output/couplet/model_final" + # vocab_file = "data_test/couplets/vocab.txt" + + python projects/Couplets/infer.py + ``` + +- for distributed inference: + + set your data path and model in `projects/Couplets/distribute_infer.py` + ```python + # line 46 + self.cfg.vocab_file = "data_test/couplets/vocab.txt" + + # line 97 ~106 + pipeline = CoupletPipeline( + # you can also use path output/couplet/config.yaml to replace config.py + "projects/Couplets/configs/config.py", + data_parallel=1, + tensor_parallel=1, # modify it according to your needs + pipeline_parallel=4, # modify it according to your needs + pipeline_stage_id=None, # modify it according to your needs + pipeline_num_layers=12, # modify it according to your needs + model_path="output/couplet/model_final/model", # modify it according to your needs + mode="libai", + ) + ``` + ``` + bash tools/infer.sh projects/Couplets/distribute_infer.py 4 + ``` + +## Results + +```shell +上联： +天朗气清风和畅 +下联： +水流花海月圆融 + +上联： +千秋月色君长看 +下联： +一夜风流人在天 + +上联： +梦里不知身是客 +下联： +此间何处是家乡 +``` \ No newline at end of file diff --git a/projects/Couplets/tokenizer/tokenizer.py b/projects/Couplets/tokenizer/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..755c9c1128e9c210b7422be6206da6da1ccc4a44 --- /dev/null +++ b/projects/Couplets/tokenizer/tokenizer.py @@ -0,0 +1,46 @@ +import collections + + +def load_vocab(vocab_file): + vocab = collections.OrderedDict() + index = 0 + with open(vocab_file, "r", encoding="utf-8") as reader: + while True: + token = reader.readline() + token = token.strip("\n") + if not token: + break + vocab[token] = index + index += 1 + return vocab + + +class CoupletsTokenizer: + def __init__(self, vocab_file): + self.vocab = load_vocab(vocab_file) + self.inv_vocab = {v: k for k, v in self.vocab.items()} + self.vocab_tokens = self.vocab.keys() + self.unk_id = self.vocab[""] + self.pad_id = self.vocab[""] + self.bos_id = self.vocab[""] + self.eos_id = self.vocab[""] + + def tokenize(self, text): + tokens_list = text.split() + return tokens_list + + def convert_tokens_to_ids(self, tokens_list): + ids_list = [] + for token in tokens_list: + if token not in self.vocab_tokens: + token = "" + token_id = self.vocab[token] + ids_list.append(token_id) + return ids_list + + def convert_ids_to_tokens(self, ids_list): + tokens_list = [] + for token_id in ids_list: + token = self.inv_vocab[token_id] + tokens_list.append(token) + return tokens_list diff --git a/projects/DALLE2/configs/dalle2_config.py b/projects/DALLE2/configs/dalle2_config.py new file mode 100644 index 0000000000000000000000000000000000000000..fde1d37fe7c0ffa09648c3583e14b6ed3ba1ce89 --- /dev/null +++ b/projects/DALLE2/configs/dalle2_config.py @@ -0,0 +1,68 @@ +from libai.config import LazyCall +from configs.common.train import train +from dalle2.models import DiffusionPrior, DiffusionPriorNetwork, Unet, Decoder, DALLE2 +from dalle2._clip import OpenAIClipAdapter +from omegaconf import DictConfig + +clip = LazyCall(OpenAIClipAdapter)(name="") +swinir = DictConfig({"swinir_path": None}) + +prior = LazyCall(DiffusionPrior)( + net=LazyCall(DiffusionPriorNetwork)( + dim=768, + depth=24, + num_timesteps=1000, + max_text_len=77, + num_time_embeds=1, + num_image_embeds=1, + num_text_embeds=1, + dim_head=64, + heads=32, + ff_mult=4, + attn_dropout=0.05, + ff_dropout=0.05, + normformer=True, + ), + clip=clip, + image_embed_dim=768, + timesteps=1000, + cond_drop_prob=0.1, + loss_type="l2", + condition_on_text_encodings=True, +) + +unet1 = LazyCall(Unet)( + dim=320, + image_embed_dim=768, + text_embed_dim=768, + cond_dim=512, + channels=3, + dim_mults=(1, 2, 3, 4), + num_resnet_blocks=4, + attn_heads=8, + attn_dim_head=64, + sparse_attn=True, + memory_efficient=True, + cond_on_text_encodings=True, + self_attn=[False, True, True, True], +) + +decoder = LazyCall(Decoder)( + unet=(unet1,), + image_sizes=[ + 64, + ], + clip=None, + channels=3, + timesteps=1000, + loss_type="l2", + beta_schedule=["cosine"], + learned_variance=True, +) + +model = LazyCall(DALLE2)( + prior=prior, + decoder=decoder, + prior_weight_path="", + decoder_weight_path="", +) diff --git a/projects/DALLE2/dalle2/__init__.py b/projects/DALLE2/dalle2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..688e204f21a5a916f7622eed29e9a19164385543 --- /dev/null +++ b/projects/DALLE2/dalle2/__init__.py @@ -0,0 +1,2 @@ +from .models import Unet, DALLE2, DiffusionPriorNetwork, DiffusionPrior, Decoder +from ._clip import OpenAIClipAdapter, import_flow_clip diff --git a/projects/DALLE2/dalle2/_clip.py b/projects/DALLE2/dalle2/_clip.py new file mode 100644 index 0000000000000000000000000000000000000000..1150bc1cdc1746d7292604796e5d211dc6a9082d --- /dev/null +++ b/projects/DALLE2/dalle2/_clip.py @@ -0,0 +1,116 @@ +import os +import sys +from collections import namedtuple + +import oneflow as flow +from oneflow import nn + +from .models import l2norm + + +def import_flow_clip(fn): + def wrapper(*args, **kwargs): + sys.path.append( + os.path.join(os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")), "CLIP") + ) + fn(*args, **kwargs) + sys.path.pop() + + return wrapper + + +EmbeddedText = namedtuple("EmbedTextReturn", ["text_embed", "text_encodings"]) +EmbeddedImage = namedtuple("EmbedImageReturn", ["image_embed", "image_encodings"]) + + +class BaseClipAdapter(nn.Module): + def __init__(self, clip, **kwargs): + super().__init__() + self.clip = clip + self.overrides = kwargs + + @property + def dim_latent(self): + raise NotImplementedError + + @property + def image_size(self): + raise NotImplementedError + + @property + def image_channels(self): + raise NotImplementedError + + @property + def max_text_len(self): + raise NotImplementedError + + def embed_text(self, text): + raise NotImplementedError + + def embed_image(self, image): + raise NotImplementedError + + +class OpenAIClipAdapter(BaseClipAdapter): + @import_flow_clip + def __init__(self, name="ViT-L/14"): + import clip + + openai_clip, preprocess = clip.load(name) + super().__init__(openai_clip) + self.eos_id = 49407 # for handling 0 being also '!' + + text_attention_final = self.find_layer("ln_final") + self.handle = text_attention_final.register_forward_hook(self._hook) + self.clip_normalize = preprocess.transforms[-1] + self.cleared = False + + def find_layer(self, layer): + modules = dict([*self.clip.named_modules()]) + return modules.get(layer, None) + + def clear(self): + if self.cleared: + return + + self.handle() + + def _hook(self, _, inputs, outputs): + self.text_encodings = outputs + + @property + def dim_latent(self): + return 512 + + @property + def image_size(self): + return self.clip.visual.input_resolution + + @property + def image_channels(self): + return 3 + + @property + def max_text_len(self): + return self.clip.context_length + + @flow.no_grad() + def embed_text(self, text): + text = text[..., : self.max_text_len] + + assert not self.cleared + text_mask = text != 0 # v0.15.4 + + text_embed = self.clip.encode_text(text) + text_encodings = self.text_encodings + del self.text_encodings + return l2norm(text_embed.float()), text_encodings.float(), text_mask + + @flow.no_grad() + def embed_image(self, image): + assert not self.cleared + image = self.validate_and_resize_image(image) + image = self.clip_normalize(image) + image_embed = self.clip.encode_image(image) + return EmbeddedImage(l2norm(image_embed.float()), None) diff --git a/projects/DALLE2/dalle2/dalle2_loader.py b/projects/DALLE2/dalle2/dalle2_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..5aac611bc509eb78d1e1464b9b437c7a6a615639 --- /dev/null +++ b/projects/DALLE2/dalle2/dalle2_loader.py @@ -0,0 +1,96 @@ +import logging + +import oneflow as flow +from oneflow.framework.check_point_v2 import _broadcast_py_object + +import libai.utils.distributed as dist +from libai.models.build import build_model +from libai.models.utils.model_utils.base_loader import ( + ModelLoaderHuggerFace, + _load_state_dict_into_model, +) + +logger = logging.getLogger("libai.dalle2." + __name__) + + +class Dalle2ModelLoader(ModelLoaderHuggerFace): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + self.base_model_prefix_1 = "" + self.base_model_prefix_2 = "" + + def _convert_state_dict(self, state_dict, module="prior"): + old_keys = [] + new_keys = [] + if module == "prior": + for k in state_dict.keys(): + if "clip." in k: + continue + old_keys.append(k) + if k.endswith(".g"): + k = k[:-1] + "weight" + elif k.startswith("net.causal_transformer"): + if k.endswith("gamma"): + k = k[:-5] + "weight" + elif k.endswith("beta"): + k = k[:-4] + "bias" + new_keys.append("prior." + k) + elif module == "decoder": + for k in state_dict.keys(): + if "clip." in k: + continue + old_keys.append(k) + if k.endswith(".g"): + k = k[:-1] + "weight" + elif "cross_attn" in k: + if k.endswith("gamma"): + k = k[:-5] + "weight" + elif k.endswith("beta"): + k = k[:-4] + "bias" + new_keys.append("decoder." + k) + ret_state_dict = {} + for old_key, new_key in zip(old_keys, new_keys): + ret_state_dict[new_key] = state_dict.pop(old_key) + return ret_state_dict + + def load(self): + if dist.is_main_process(): + # prior + logger.info("loading torch model prior...") + torch_state_dict = self._load_torch_state_dict(self.libai_cfg.model.prior_weight_path)[ + "ema_model" + ] + logger.info("converting torch model prior into oneflow model...") + flow_state_dict = self._convert_tensors(torch_state_dict) + prior_state_dict = self._convert_state_dict(flow_state_dict) + # decoder + logger.info("loading torch model decoder...") + torch_state_dict = self._load_torch_state_dict(self.libai_cfg.model.decoder_weight_path) + flow_state_dict = self._convert_tensors(torch_state_dict) + logger.info("converting torch model decoder into oneflow model...") + decoder_state_dict = self._convert_state_dict(flow_state_dict, module="decoder") + flow_state_dict = {**prior_state_dict, **decoder_state_dict} + else: + flow_state_dict = None + + logger.info("building LiBai model...") + self.libai_cfg = _broadcast_py_object(self.libai_cfg, src=0) + self.model = build_model(self.model) + self.model._apply(dist.convert_to_distributed_default_setting) + self.model = self.model.eval() + + flow.cuda.empty_cache() + # State_dict to global + logger.info("transfering state_dict local to global...") + flow_state_dict = self._state_dict_to_global(flow_state_dict, mode="pytorch") # oom + # Load + # ( + # model, + # missing_keys, + # unexpected_keys, + # mismatched_keys, + # error_msgs, + # ) = self._load_pretrained_model(self.model, flow_state_dict, self.pretrained_model_path) + logger.info("loading model weights into LiBai...") + _load_state_dict_into_model(self.model, flow_state_dict, "") + return self.model diff --git a/projects/DALLE2/dalle2/einops_exts.py b/projects/DALLE2/dalle2/einops_exts.py new file mode 100644 index 0000000000000000000000000000000000000000..83bc5ea2f4080872fc562f90b109fc52b1359c75 --- /dev/null +++ b/projects/DALLE2/dalle2/einops_exts.py @@ -0,0 +1,168 @@ +# From https://github.com/arogozhnikov/einops/blob/master/einops/layers/oneflow.py + +import re +from functools import wraps + +import oneflow as flow +from einops import rearrange, reduce, repeat +from einops._backends import AbstractBackend +from einops.layers import RearrangeMixin +from oneflow import nn + + +class Rearrange(RearrangeMixin, flow.nn.Module): + def forward(self, input): + return self._apply_recipe(input) + + +class OneFlowBackend(AbstractBackend): + framework_name = "oneflow" + + def __init__(self): + import oneflow as flow + + self.flow = flow + + def is_appropriate_type(self, tensor): + return isinstance(tensor, self.flow.Tensor) + + def from_numpy(self, x): + variable = self.flow.from_numpy(x) + if self.is_float_type(variable): + # attach grad only to floating types + variable.requires_grad = True + return variable + + def to_numpy(self, x): + return x.detach().cpu().numpy() + + def arange(self, start, stop): + return self.flow.arange(start, stop, dtype=self.flow.int64) + + def reduce(self, x, operation, reduced_axes): + for axis in sorted(reduced_axes, reverse=True): + if operation == "min": + x, _ = x.min(dim=axis) + elif operation == "max": + x, _ = x.max(dim=axis) + elif operation in ["sum", "mean", "prod"]: + x = getattr(x, operation)(dim=axis) + else: + raise NotImplementedError("Unknown reduction ", operation) + return x + + def transpose(self, x, axes): + return x.permute(axes) + + def stack_on_zeroth_dimension(self, tensors: list): + return self.flow.stack(tensors) + + def add_axes(self, x, n_axes, pos2len): + repeats = [-1] * n_axes + for axis_position, axis_length in pos2len.items(): + x = self.add_axis(x, axis_position) + repeats[axis_position] = axis_length + return x.expand(*repeats) + + def tile(self, x, repeats): + return x.repeat(repeats) + + def add_axis(self, x, new_position): + return self.flow.unsqueeze(x, new_position) + + def is_float_type(self, x): + return x.dtype in [self.flow.float16, self.flow.float32, self.flow.float64] + + def einsum(self, pattern, *x): + return self.flow.einsum(pattern, *x) + + +# From https://github.com/lucidrains/einops-exts/tree/main/einops_exts + + +class EinopsToAndFrom(nn.Module): + def __init__(self, from_einops, to_einops, fn): + super().__init__() + self.from_einops = from_einops + self.to_einops = to_einops + self.fn = fn + + def forward(self, x, **kwargs): + shape = x.shape + reconstitute_kwargs = dict(tuple(zip(self.from_einops.split(" "), shape))) + x = rearrange(x, f"{self.from_einops} -> {self.to_einops}") + x = self.fn(x, **kwargs) + x = rearrange(x, f"{self.to_einops} -> {self.from_einops}", **reconstitute_kwargs) + return x + + +# checking shape +# @nils-werner +# https://github.com/arogozhnikov/einops/issues/168#issuecomment-1042933838 + + +def check_shape(tensor, pattern, **kwargs): + return rearrange(tensor, f"{pattern} -> {pattern}", **kwargs) + + +# do same einops operations on a list of tensors + + +def _many(fn): + @wraps(fn) + def inner(tensors, pattern, **kwargs): + return (fn(tensor, pattern, **kwargs) for tensor in tensors) + + return inner + + +# do einops with unflattening of anonymously named dimensions +# (...flattened) -> ...flattened + + +def _with_anon_dims(fn): + @wraps(fn) + def inner(tensor, pattern, **kwargs): + regex = r"(\.\.\.[a-zA-Z]+)" + matches = re.findall(regex, pattern) + + def get_anon_dim_name(t): + return t.lstrip("...") + + dim_prefixes = tuple(map(get_anon_dim_name, set(matches))) + + update_kwargs_dict = dict() + + for prefix in dim_prefixes: + assert prefix in kwargs, f'dimension list "{prefix}" was not passed in' + dim_list = kwargs[prefix] + assert isinstance( + dim_list, (list, tuple) + ), f'dimension list "{prefix}" needs to be a tuple of list of dimensions' + dim_names = list(map(lambda ind: f"{prefix}{ind}", range(len(dim_list)))) + update_kwargs_dict[prefix] = dict(zip(dim_names, dim_list)) + + def sub_with_anonymous_dims(t): + dim_name_prefix = get_anon_dim_name(t.groups()[0]) + return " ".join(update_kwargs_dict[dim_name_prefix].keys()) + + pattern_new = re.sub(regex, sub_with_anonymous_dims, pattern) + + for prefix, update_dict in update_kwargs_dict.items(): + del kwargs[prefix] + kwargs.update(update_dict) + + return fn(tensor, pattern_new, **kwargs) + + return inner + + +# generate all helper functions + +rearrange_many = _many(rearrange) +repeat_many = _many(repeat) +reduce_many = _many(reduce) + +rearrange_with_anon_dims = _with_anon_dims(rearrange) +repeat_with_anon_dims = _with_anon_dims(repeat) +reduce_with_anon_dims = _with_anon_dims(reduce) diff --git a/projects/DALLE2/dalle2/model_weights/download_utils.py b/projects/DALLE2/dalle2/model_weights/download_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4b7a8f4db4833f8a560bfdc5d79204b694a5faf8 --- /dev/null +++ b/projects/DALLE2/dalle2/model_weights/download_utils.py @@ -0,0 +1,37 @@ +import logging +import os + +from libai.utils.file_utils import download_file + +logger = logging.getLogger(__name__) # pylint: disable=invalid-name + +url_map = { + "prior": "https://huggingface.co/nousr/conditioned-prior/resolve/main/vit-l-14/prior_aes_finetune.pth", # noqa + "decoder": "https://huggingface.co/laion/DALLE2-PyTorch/resolve/main/decoder/1.5B_laion2B/latest.pth", # noqa + "clip": "https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt", # noqa + "bpe_vocab": "https://oneflow-static.oss-cn-beijing.aliyuncs.com/libai/clip/bpe_simple_vocab_16e6.txt.gz", # noqa + "swinir": "https://github.com/JingyunLiang/SwinIR/releases/download/v0.0/", +} + + +def _download_if_not_exist(path, name): + if os.path.exists(path): + logger.info((f"using {name}'s weight at {path}")) + return + if name == "swinir": + os.makedirs(os.path.dirname(path), exist_ok=True) + download_file(path, url_map[name] + os.path.basename(path)) + return + os.makedirs(os.path.dirname(path), exist_ok=True) + download_file(path, url_map[name]) + + +def download_dalle2_weights(cfg): + if not os.path.exists("./dalle2/data/bpe_simple_vocab_16e6.txt.gz"): + os.makedirs("./dalle2/data", exist_ok=True) + download_file("./dalle2/data/bpe_simple_vocab_16e6.txt.gz", url_map["bpe_vocab"]) + + _download_if_not_exist(cfg.swinir.swinir_path, "swinir") + _download_if_not_exist(cfg.model.prior_weight_path, "prior") + _download_if_not_exist(cfg.model.decoder_weight_path, "decoder") + _download_if_not_exist(cfg.model.prior.clip.name, "clip") diff --git a/projects/DALLE2/dalle2/models.py b/projects/DALLE2/dalle2/models.py new file mode 100644 index 0000000000000000000000000000000000000000..efa873019caa9b64fdffc9109ed5dddfd61c93fb --- /dev/null +++ b/projects/DALLE2/dalle2/models.py @@ -0,0 +1,2541 @@ +import math +import random +from contextlib import contextmanager +from functools import partial, wraps + +import flowvision.transforms as T +import kornia.augmentation as K +import numpy as np +import oneflow as flow +import oneflow.nn.functional as F +from einops import rearrange, reduce, repeat +from kornia.filters import gaussian_blur2d +from omegaconf import ListConfig +from oneflow import einsum, nn +from oneflow.nn import Conv2d, ConvTranspose2d, GroupNorm +from oneflow.nn.functional import layer_norm +from resize_right import resize +from tqdm.auto import tqdm + +from libai.layers import Embedding, LayerNorm, Linear +from libai.utils import distributed as dist + +from .einops_exts import EinopsToAndFrom, Rearrange, check_shape, rearrange_many, repeat_many +from .rotary_embedding_flow import RotaryEmbedding +from .tokenizer import SimpleTokenizer +from .vqgan_vae import NullVQGanVAE, VQGanVAE + +# rotary embeddings + + +# constants +def get_default_sbp(): + return dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]) + + +def get_default_placement(): + return dist.get_layer_placement(0) + + +NAT = 1.0 / math.log(2.0) + +random.seed(666) +np.random.seed(6666) +# helper functions + + +def exists(val): + return val is not None + + +def identity(t, *args, **kwargs): + return t + + +def first(arr, d=None): + if len(arr) == 0: + return d + return arr[0] + + +def maybe(fn): + @wraps(fn) + def inner(x): + if not exists(x): + return x + return fn(x) + + return inner + + +def default(val, d): + if exists(val): + return val + return d() if callable(d) else d + + +def cast_tuple(val, length=None): + if isinstance(val, list) or isinstance(val, ListConfig): + val = tuple(val) + + out = val if isinstance(val, tuple) else ((val,) * default(length, 1)) + + if exists(length): + assert len(out) == length + + return out + + +def module_device(module): + return next(module.parameters()).device + + +def zero_init_(m): + nn.init.zeros_(m.weight) + if exists(m.bias): + nn.init.zeros_(m.bias) + + +@contextmanager +def null_context(*args, **kwargs): + yield + + +def eval_decorator(fn): + def inner(model, *args, **kwargs): + was_training = model.training + model.eval() + out = fn(model, *args, **kwargs) + model.train(was_training) + return out + + return inner + + +def is_list_str(x): + if not isinstance(x, (list, tuple)): + return False + return all([type(el) == str for el in x]) + + +def pad_tuple_to_length(t, length, fillvalue=None): + remain_length = length - len(t) + if remain_length <= 0: + return t + return (*t, *((fillvalue,) * remain_length)) + + +# for controlling freezing of CLIP + + +def set_module_requires_grad_(module, requires_grad): + for param in module.parameters(): + param.requires_grad = requires_grad + + +def freeze_all_layers_(module): + set_module_requires_grad_(module, False) + + +def unfreeze_all_layers_(module): + set_module_requires_grad_(module, True) + + +def freeze_model_and_make_eval_(model): + model.eval() + freeze_all_layers_(model) + + +# tensor helpers + + +def log(t, eps=1e-12): + return flow.log(t.clamp(min=eps)) + + +def l2norm(t): + return F.normalize(t, dim=-1) + + +def resize_image_to(image, target_image_size): + orig_image_size = image.shape[-1] + + if orig_image_size == target_image_size: + return image + + scale_factors = target_image_size / orig_image_size + return resize(image, scale_factors=scale_factors) + + +# image normalization functions +# ddpms expect images to be in the range of -1 to 1 +# but CLIP may otherwise + + +def normalize_neg_one_to_one(img): + return img * 2 - 1 + + +def unnormalize_zero_to_one(normed_img): + return (normed_img + 1) * 0.5 + + +# classifier free guidance functions + + +def prob_mask_like(shape, prob, placement=None, sbp=None): + placement = placement or get_default_placement() + sbp = sbp or get_default_sbp() + if prob == 1: + return flow.ones(shape, dtype=flow.bool, placement=placement, sbp=sbp) + elif prob == 0: + return flow.zeros(shape, dtype=flow.bool, placement=placement, sbp=sbp) + else: + return flow.zeros(shape, placement=placement, sbp=sbp).float().uniform_(0, 1) < prob + + +# gaussian diffusion helper functions + + +def extract(a, t, x_shape): + b, *_ = t.shape + out = a.gather(len(a.shape) - 1, t) + return out.reshape(b, *((1,) * (len(x_shape) - 1))) + + +def meanflat(x): + return x.mean(dim=tuple(range(1, len(x.shape)))) + + +def normal_kl(mean1, logvar1, mean2, logvar2): + return 0.5 * ( + -1.0 + + logvar2 + - logvar1 + + flow.exp(logvar1 - logvar2) + + ((mean1 - mean2) ** 2) * flow.exp(-logvar2) + ) + + +def approx_standard_normal_cdf(x): + return 0.5 * (1.0 + flow.tanh(((2.0 / math.pi) ** 0.5) * (x + 0.044715 * (x ** 3)))) + + +def discretized_gaussian_log_likelihood(x, *, means, log_scales, thres=0.999): + assert x.shape == means.shape == log_scales.shape + + # attempting to correct nan gradients when learned variance is turned on + # in the setting of deepspeed fp16 + eps = 1e-12 if x.dtype == flow.float32 else 1e-3 + + centered_x = x - means + inv_stdv = flow.exp(-log_scales) + plus_in = inv_stdv * (centered_x + 1.0 / 255.0) + cdf_plus = approx_standard_normal_cdf(plus_in) + min_in = inv_stdv * (centered_x - 1.0 / 255.0) + cdf_min = approx_standard_normal_cdf(min_in) + log_cdf_plus = log(cdf_plus) + log_one_minus_cdf_min = log(1.0 - cdf_min) + cdf_delta = cdf_plus - cdf_min + + log_probs = flow.where( + x < -thres, log_cdf_plus, flow.where(x > thres, log_one_minus_cdf_min, log(cdf_delta, eps)) + ) + + return log_probs + + +def cosine_beta_schedule(timesteps, s=0.008): + """ + cosine schedule + as proposed in https://openreview.net/forum?id=-NEXDKk8gZ + """ + steps = timesteps + 1 + x = flow.linspace(0, timesteps, steps, dtype=flow.float64) + alphas_cumprod = flow.cos(((x / timesteps) + s) / (1 + s) * np.pi * 0.5) ** 2 + alphas_cumprod = alphas_cumprod / first(alphas_cumprod) + betas = 1 - (alphas_cumprod[1:] / alphas_cumprod[:-1]) + return flow.clip(betas, 0, 0.999) + + +def linear_beta_schedule(timesteps): + scale = 1000 / timesteps + beta_start = scale * 0.0001 + beta_end = scale * 0.02 + return flow.linspace(beta_start, beta_end, timesteps, dtype=flow.float64) + + +def quadratic_beta_schedule(timesteps): + scale = 1000 / timesteps + beta_start = scale * 0.0001 + beta_end = scale * 0.02 + return flow.linspace(beta_start ** 0.5, beta_end ** 0.5, timesteps, dtype=flow.float64) ** 2 + + +def sigmoid_beta_schedule(timesteps): + scale = 1000 / timesteps + beta_start = scale * 0.0001 + beta_end = scale * 0.02 + betas = flow.linspace(-6, 6, timesteps, dtype=flow.float64) + return flow.sigmoid(betas) * (beta_end - beta_start) + beta_start + + +class NoiseScheduler(nn.Module): + def __init__( + self, *, beta_schedule, timesteps, loss_type, p2_loss_weight_gamma=0.0, p2_loss_weight_k=1 + ): + super().__init__() + + if beta_schedule == "cosine": + betas = cosine_beta_schedule(timesteps) + elif beta_schedule == "linear": + betas = linear_beta_schedule(timesteps) + elif beta_schedule == "quadratic": + betas = quadratic_beta_schedule(timesteps) + elif beta_schedule == "jsd": + betas = 1.0 / flow.linspace(timesteps, 1, timesteps) + elif beta_schedule == "sigmoid": + betas = sigmoid_beta_schedule(timesteps) + else: + raise NotImplementedError() + + betas = betas + alphas = 1.0 - betas + alphas_cumprod = flow.cumprod(alphas, dim=0) + alphas_cumprod_prev = F.pad(alphas_cumprod[:-1], (1, 0), value=1.0) + + (timesteps,) = betas.shape + self.num_timesteps = int(timesteps) + + if loss_type == "l1": + loss_fn = F.l1_loss + elif loss_type == "l2": + loss_fn = flow.nn.MSELoss() + elif loss_type == "huber": + loss_fn = F.smooth_l1_loss + else: + raise NotImplementedError() + + self.loss_type = loss_type + self.loss_fn = loss_fn + + # register buffer helper function to cast double back to float + + def register_buffer(name, val): + self.register_buffer(name, val.to(flow.float32)) + + register_buffer("betas", betas) + register_buffer("alphas_cumprod", alphas_cumprod) + register_buffer("alphas_cumprod_prev", alphas_cumprod_prev) + + # calculations for diffusion q(x_t | x_{t-1}) and others + + register_buffer("sqrt_alphas_cumprod", flow.sqrt(alphas_cumprod)) + register_buffer("sqrt_one_minus_alphas_cumprod", flow.sqrt(1.0 - alphas_cumprod)) + register_buffer("log_one_minus_alphas_cumprod", flow.log(1.0 - alphas_cumprod)) + register_buffer("sqrt_recip_alphas_cumprod", flow.sqrt(1.0 / alphas_cumprod)) + register_buffer("sqrt_recipm1_alphas_cumprod", flow.sqrt(1.0 / alphas_cumprod - 1)) + + # calculations for posterior q(x_{t-1} | x_t, x_0) + + posterior_variance = betas * (1.0 - alphas_cumprod_prev) / (1.0 - alphas_cumprod) + + # above: equal to 1. / (1. / (1. - alpha_cumprod_tm1) + alpha_t / beta_t) + + register_buffer("posterior_variance", posterior_variance) + + register_buffer( + "posterior_log_variance_clipped", flow.log(posterior_variance.clamp(min=1e-20)) + ) + register_buffer( + "posterior_mean_coef1", betas * flow.sqrt(alphas_cumprod_prev) / (1.0 - alphas_cumprod) + ) + register_buffer( + "posterior_mean_coef2", + (1.0 - alphas_cumprod_prev) * flow.sqrt(alphas) / (1.0 - alphas_cumprod), + ) + + # p2 loss reweighting + + self.has_p2_loss_reweighting = p2_loss_weight_gamma > 0.0 + register_buffer( + "p2_loss_weight", + (p2_loss_weight_k + alphas_cumprod / (1 - alphas_cumprod)) ** -p2_loss_weight_gamma, + ) + + def q_posterior(self, x_start, x_t, t): + posterior_mean = ( + extract(self.posterior_mean_coef1, t, x_t.shape) * x_start + + extract(self.posterior_mean_coef2, t, x_t.shape) * x_t + ) + posterior_variance = extract(self.posterior_variance, t, x_t.shape) + posterior_log_variance_clipped = extract(self.posterior_log_variance_clipped, t, x_t.shape) + return posterior_mean, posterior_variance, posterior_log_variance_clipped + + def q_sample(self, x_start, t, noise=None): + noise = default(noise, lambda: flow.randn_like(x_start)) + + return ( + extract(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start + + extract(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * noise + ) + + def predict_start_from_noise(self, x_t, t, noise): + return ( + extract(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t + - extract(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) * noise + ) + + def p2_reweigh_loss(self, loss, times): + if not self.has_p2_loss_reweighting: + return loss + return loss * extract(self.p2_loss_weight, times, loss.shape) + + +# diffusion prior + + +class ChanLayerNorm(nn.Module): + def __init__(self, dim, eps=1e-5): + super().__init__() + self.eps = eps + self.weight = nn.Parameter(flow.ones(1, dim, 1, 1)) + + def forward(self, x): + # var = flow.var(x, dim = 1, unbiased = False, keepdim = True) + # mean = flow.mean(x, dim = 1, keepdim = True) + # return (x - mean) / (var + self.eps).sqrt() * self.weight + x = x.permute(0, 2, 3, 1) + out = layer_norm(x, normalized_shape=(x.shape[-1:]), eps=self.eps) + return out.permute(0, 3, 1, 2) * self.weight + + +class Residual(nn.Module): + def __init__(self, fn): + super().__init__() + self.fn = fn + + def forward(self, x, **kwargs): + return self.fn(x, **kwargs) + x + + +# mlp + + +class MLP(nn.Module): + def __init__( + self, + dim_in, + dim_out, + *, + expansion_factor=2.0, + depth=2, + norm=False, + ): + super().__init__() + hidden_dim = int(expansion_factor * dim_out) + + def norm_fn(): + return LayerNorm(hidden_dim) if norm else nn.Identity() + + layers = [nn.Sequential(Linear(dim_in, hidden_dim), nn.SiLU(), norm_fn())] + + for _ in range(depth - 1): + layers.append(nn.Sequential(Linear(hidden_dim, hidden_dim), nn.SiLU(), norm_fn())) + + layers.append(Linear(hidden_dim, dim_out)) + self.net = nn.Sequential(*layers) + + def forward(self, x): + return self.net(x.float()) + + +# relative positional bias for causal transformer + + +class RelPosBias(nn.Module): + def __init__( + self, + heads=8, + num_buckets=32, + max_distance=128, + ): + super().__init__() + self.num_buckets = num_buckets + self.max_distance = max_distance + self.relative_attention_bias = Embedding(num_buckets, heads) + + @staticmethod + def _relative_position_bucket(relative_position, num_buckets=32, max_distance=128): + n = -relative_position + n = flow.max(n, flow.zeros_like(n)).long() + + max_exact = num_buckets // 2 + is_small = n < max_exact + + val_if_large = ( + max_exact + + ( + flow.log(n.float() / max_exact) + / math.log(max_distance / max_exact) + * (num_buckets - max_exact) + ).long() + ) + val_if_large = flow.min(val_if_large, flow.zeros_like(val_if_large) + num_buckets - 1) + return flow.where(is_small, n, val_if_large) + + def forward(self, i, j): + q_pos = flow.arange(i, dtype=flow.long) + k_pos = flow.arange(j, dtype=flow.long) + rel_pos = rearrange(k_pos, "j -> 1 j") - rearrange(q_pos, "i -> i 1") + rp_bucket = self._relative_position_bucket( + rel_pos, num_buckets=self.num_buckets, max_distance=self.max_distance + ) + values = self.relative_attention_bias( + rp_bucket.to_global(sbp=get_default_sbp(), placement=get_default_placement()) + ) + return rearrange(values, "i j h -> h i j") + + +# feedforward + + +class SwiGLU(nn.Module): + """used successfully in https://arxiv.org/abs/2204.0231""" + + def forward(self, x): + x, gate = x.chunk(2, dim=-1) + return x * F.silu(gate) + + +def FeedForward(dim, mult=4, dropout=0.0, post_activation_norm=False): + """post-activation norm https://arxiv.org/abs/2110.09456""" + + inner_dim = int(mult * dim) + return nn.Sequential( + LayerNorm(dim), + Linear(dim, inner_dim * 2, bias=False, parallel="col"), + SwiGLU(), + LayerNorm(inner_dim) if post_activation_norm else nn.Identity(), + nn.Dropout(dropout), + Linear(inner_dim, dim, bias=False, parallel="row"), + ) + + +# attention + + +class Attention(nn.Module): + def __init__(self, dim, *, dim_head=64, heads=8, dropout=0.0, causal=False, rotary_emb=None): + super().__init__() + self.scale = dim_head ** -0.5 + self.heads = heads + inner_dim = dim_head * heads + + self.causal = causal + self.norm = LayerNorm(dim) + self.dropout = nn.Dropout(dropout) + + self.null_kv = nn.Parameter(flow.randn(2, dim_head)) + self.to_q = Linear(dim, inner_dim, bias=False, parallel="col") + self.to_kv = Linear(dim, dim_head * 2, bias=False, parallel="col") + + self.rotary_emb = rotary_emb + + self.to_out = nn.Sequential( + Linear(inner_dim, dim, bias=False, parallel="row"), LayerNorm(dim) + ) + + def forward(self, x, mask=None, attn_bias=None): + b, n = x.shape[:2] + + x = self.norm(x) + q, k, v = (self.to_q(x), *self.to_kv(x).chunk(2, dim=-1)) + + q = rearrange(q, "b n (h d) -> b h n d", h=self.heads) + q = q * self.scale + + # rotary embeddings + + if exists(self.rotary_emb): + q, k = map(self.rotary_emb.rotate_queries_or_keys, (q, k)) + + nk, nv = repeat_many(self.null_kv.unbind(dim=-2), "d -> b 1 d", b=b) + + k = flow.cat((nk, k), dim=-2) + v = flow.cat((nv, v), dim=-2) + + # calculate query / key similarities + sim = einsum("b h i d, b j d -> b h i j", q, k) + # relative positional encoding (T5 style) + + if exists(attn_bias): + sim = sim + attn_bias + + # masking + + max_neg_value = -3.4028e38 # flow.finfo(sim.dtype).max + + if exists(mask): + mask = F.pad(mask, (1, 0), value=1) + mask = rearrange(mask, "b j -> b 1 1 j") + sim = sim.masked_fill(1 - mask, max_neg_value) # ~mask + if self.causal: + i, j = sim.shape[-2:] + causal_mask = flow.ones( + (i, j), placement=get_default_placement(), sbp=get_default_sbp(), dtype=flow.int32 + ).triu(j - i + 1) + sim = sim.masked_fill(causal_mask, max_neg_value) + # attention + + attn = sim.softmax(dim=-1) + attn = self.dropout(attn) + + # aggregate values + + out = einsum("b h i j, b j d -> b h i d", attn, v) + + out = rearrange(out, "b h n d -> b n (h d)") + return self.to_out(out) + + +class CausalTransformer(nn.Module): + def __init__( + self, + *, + dim, + depth, + dim_head=64, + heads=8, + ff_mult=4, + norm_out=True, + attn_dropout=0.0, + ff_dropout=0.0, + final_proj=True, + normformer=False, + rotary_emb=True, + ): + super().__init__() + self.rel_pos_bias = RelPosBias(heads=heads) + + rotary_emb = RotaryEmbedding(dim=min(32, dim_head)) if rotary_emb else None + + self.layers = nn.ModuleList([]) + for _ in range(depth): + self.layers.append( + nn.ModuleList( + [ + Attention( + dim=dim, + causal=True, + dim_head=dim_head, + heads=heads, + dropout=attn_dropout, + rotary_emb=rotary_emb, + ), + FeedForward( + dim=dim, + mult=ff_mult, + dropout=ff_dropout, + post_activation_norm=normformer, + ), + ] + ) + ) + self.norm = LayerNorm(dim) if norm_out else nn.Identity() + self.project_out = Linear(dim, dim, bias=False) if final_proj else nn.Identity() + + def forward( + self, + x, + mask=None, + ): + n = x.shape[1] + + attn_bias = self.rel_pos_bias(n, n + 1) + + for attn, ff in self.layers: + x = attn(x, mask=mask, attn_bias=attn_bias) + x + x = ff(x) + x + + out = self.norm(x) + return self.project_out(out) + + +class DiffusionPriorNetwork(nn.Module): + def __init__( + self, + dim, + num_timesteps=None, + num_time_embeds=1, + num_image_embeds=1, + num_text_embeds=1, + max_text_len=256, + **kwargs, + ): + super().__init__() + self.num_time_embeds = num_time_embeds + self.num_image_embeds = num_image_embeds + self.num_text_embeds = num_text_embeds + + self.to_text_embeds = nn.Sequential( + Linear(dim, dim * num_text_embeds) if num_text_embeds > 1 else nn.Identity(), + Rearrange("b (n d) -> b n d", n=num_text_embeds), + ) + + self.to_time_embeds = nn.Sequential( + Embedding(num_timesteps, dim * num_time_embeds) + if exists(num_timesteps) + else nn.Sequential( + SinusoidalPosEmb(dim), MLP(dim, dim * num_time_embeds) + ), # also offer a continuous version of timestep embeddings, with a 2 layer MLP + Rearrange("b (n d) -> b n d", n=num_time_embeds), + ) + + self.to_image_embeds = nn.Sequential( + Linear(dim, dim * num_image_embeds) if num_image_embeds > 1 else nn.Identity(), + Rearrange("b (n d) -> b n d", n=num_image_embeds), + ) + + self.learned_query = nn.Parameter(flow.randn(dim)) + self.causal_transformer = CausalTransformer(dim=dim, **kwargs) + + def forward_with_cond_scale(self, *args, cond_scale=1.0, **kwargs): + logits = self.forward(*args, **kwargs) + + if cond_scale == 1: + return logits + + null_logits = self.forward(*args, cond_drop_prob=1.0, **kwargs) + return null_logits + (logits - null_logits) * cond_scale + + def forward( + self, + image_embed, + diffusion_timesteps, + *, + text_embed, + text_encodings=None, + mask=None, + cond_drop_prob=0.0, + ): + batch, dim, dtype = *image_embed.shape, image_embed.dtype + + num_time_embeds, num_image_embeds, num_text_embeds = ( + self.num_time_embeds, + self.num_image_embeds, + self.num_text_embeds, + ) + + text_embed = self.to_text_embeds(text_embed) + image_embed = self.to_image_embeds(image_embed) + + # make text encodings optional + # although the paper seems to suggest it is present <-- + + if not exists(text_encodings): + text_encodings = flow.empty( + (batch, 0, dim), + placement=get_default_placement(), + sbp=get_default_sbp(), + dtype=dtype, + ) + + if not exists(mask): + mask = flow.ones( + (batch, text_encodings.shape[-2]), + placement=get_default_placement(), + sbp=get_default_sbp(), + dtype=flow.bool, + ) + + # classifier free guidance + + keep_mask = prob_mask_like((batch,), 1 - cond_drop_prob) + keep_mask = rearrange(keep_mask, "b -> b 1").to_global( + placement=get_default_placement(), sbp=get_default_sbp() + ) + + mask &= keep_mask + + keep_mask = repeat(keep_mask, "b 1 -> b n", n=num_text_embeds) + mask = flow.cat((mask, keep_mask), dim=1) + + if exists(mask): + attend_padding = ( + 1 + num_time_embeds + num_image_embeds + ) # 1 for learned queries + number of image embeds + time embeds + mask = F.pad(mask.to(flow.int32), (0, attend_padding), value=1) + + time_embed = self.to_time_embeds( + diffusion_timesteps.to_global(placement=get_default_placement(), sbp=get_default_sbp()) + ) + + learned_queries = repeat(self.learned_query, "d -> b 1 d", b=batch) + + tokens = flow.cat( + (text_encodings, text_embed, time_embed, image_embed, learned_queries), dim=-2 + ) + + # attend + + tokens = self.causal_transformer(tokens, mask=mask) + + # get learned query, which should predict the image embedding (per DDPM timestep) + + pred_image_embed = tokens[..., -1, :] + + return pred_image_embed + + +class DiffusionPrior(nn.Module): + def __init__( + self, + net, + *, + clip=None, + image_embed_dim=None, + image_size=None, + image_channels=3, + timesteps=1000, + cond_drop_prob=0.0, + loss_type="l2", + predict_x_start=True, + beta_schedule="cosine", + condition_on_text_encodings=True, + sampling_clamp_l2norm=False, + training_clamp_l2norm=False, + init_image_embed_l2norm=False, + image_embed_scale=None, + clip_adapter_overrides=dict(), + ): + super().__init__() + + self.noise_scheduler = NoiseScheduler( + beta_schedule=beta_schedule, timesteps=timesteps, loss_type=loss_type + ) + + if exists(clip): + assert ( + image_channels == clip.image_channels + ), f"channels of image ({image_channels}) should be equal to the channels " + "that CLIP accepts ({clip.image_channels})" + + freeze_model_and_make_eval_(clip) + self.clip = clip + else: + assert exists( + image_embed_dim + ), "latent dimension must be given, if training prior network without CLIP given" + self.clip = None + + self.net = net + self.image_embed_dim = default(image_embed_dim, lambda: clip.dim_latent) + self.channels = default(image_channels, lambda: clip.image_channels) + + self.cond_drop_prob = cond_drop_prob + self.can_classifier_guidance = cond_drop_prob > 0.0 + self.condition_on_text_encodings = condition_on_text_encodings + + self.predict_x_start = predict_x_start + + self.image_embed_scale = default(image_embed_scale, self.image_embed_dim ** 0.5) + + # whether to force an l2norm, similar to clipping denoised, when sampling + self.sampling_clamp_l2norm = sampling_clamp_l2norm + self.training_clamp_l2norm = training_clamp_l2norm + self.init_image_embed_l2norm = init_image_embed_l2norm + + # device tracker + self.register_buffer("_dummy", flow.tensor([True]), persistent=False) + + def p_mean_variance(self, x, t, text_cond, clip_denoised=False, cond_scale=1.0): + assert not ( + cond_scale != 1.0 and not self.can_classifier_guidance + ), "the model was not trained with conditional dropout, " + "and thus one cannot use classifier free guidance (cond_scale anything other than 1)" + + pred = self.net.forward_with_cond_scale(x, t, cond_scale=cond_scale, **text_cond) + + if self.predict_x_start: + x_recon = pred + else: + x_recon = self.noise_scheduler.predict_start_from_noise(x, t=t, noise=pred) + + if clip_denoised and not self.predict_x_start: + x_recon.clamp_(-1.0, 1.0) + + if self.predict_x_start and self.sampling_clamp_l2norm: + x_recon = l2norm(x_recon) * self.image_embed_scale + + model_mean, posterior_variance, posterior_log_variance = self.noise_scheduler.q_posterior( + x_start=x_recon, x_t=x, t=t + ) + return model_mean, posterior_variance, posterior_log_variance + + @flow.no_grad() + def p_sample(self, x, t, text_cond=None, clip_denoised=True, cond_scale=1.0): + b = x.shape[0] + model_mean, _, model_log_variance = self.p_mean_variance( + x=x, t=t, text_cond=text_cond, clip_denoised=clip_denoised, cond_scale=cond_scale + ) + noise = flow.randn(*x.shape, placement=get_default_placement(), sbp=get_default_sbp()) + # no noise when t == 0 + nonzero_mask = (1 - (t == 0).float()).reshape(b, *((1,) * (len(x.shape) - 1))) + return model_mean + nonzero_mask * (0.5 * model_log_variance).exp() * noise + + @flow.no_grad() + def p_sample_loop(self, shape, text_cond, cond_scale=1.0): + b = shape[0] + image_embed = flow.randn(*shape, placement=get_default_placement(), sbp=get_default_sbp()) + + if self.init_image_embed_l2norm: + image_embed = l2norm(image_embed) * self.image_embed_scale + + for i in tqdm( + reversed(range(0, self.noise_scheduler.num_timesteps)), + desc="sampling loop time step", + total=self.noise_scheduler.num_timesteps, + ): + times = flow.full( + (b,), i, placement=get_default_placement(), sbp=get_default_sbp(), dtype=flow.long + ) + image_embed = self.p_sample( + image_embed, times, text_cond=text_cond, cond_scale=cond_scale + ) + + return image_embed + + def p_losses(self, image_embed, times, text_cond, noise=None): + noise = default( + noise, + lambda: flow.randn( + *image_embed.shape, placement=get_default_placement(), sbp=get_default_sbp() + ), + ) + + image_embed_noisy = self.noise_scheduler.q_sample(x_start=image_embed, t=times, noise=noise) + + pred = self.net(image_embed_noisy, times, cond_drop_prob=self.cond_drop_prob, **text_cond) + + if self.predict_x_start and self.training_clamp_l2norm: + pred = l2norm(pred) * self.image_embed_scale + + target = noise if not self.predict_x_start else image_embed + + loss = self.noise_scheduler.loss_fn(pred, target) + return loss + + @flow.no_grad() + @eval_decorator + def sample_batch_size(self, batch_size, text_cond, cond_scale=1.0): + shape = (batch_size, self.image_embed_dim) + + img = flow.randn(*shape, placement=get_default_placement(), sbp=get_default_sbp()) + + for i in tqdm( + reversed(range(0, self.noise_scheduler.num_timesteps)), + desc="sampling loop time step", + total=self.noise_scheduler.num_timesteps, + ): + img = self.p_sample( + img, + flow.full( + (batch_size,), + i, + placement=get_default_placement(), + sbp=get_default_sbp(), + dtype=flow.long, + ), + text_cond=text_cond, + cond_scale=cond_scale, + ) + return img + + @flow.no_grad() + @eval_decorator + def sample( + self, + text, + num_samples_per_batch=2, + cond_scale=1.0, + text_embed=None, + text_encodings=None, + text_mask=None, + ): + # in the paper, what they did was + # sample 2 image embeddings, choose the top 1 similarity, as judged by CLIP + text = repeat(text, "b ... -> (b r) ...", r=num_samples_per_batch) + + batch_size = text.shape[0] + image_embed_dim = self.image_embed_dim + + if text_embed is None: + assert self.clip is not None + text_embed, text_encodings, text_mask = self.clip.embed_text(text) + + text_cond = dict(text_embed=text_embed) + + if self.condition_on_text_encodings: + text_cond = {**text_cond, "text_encodings": text_encodings, "mask": text_mask} + + image_embeds = self.p_sample_loop( + (batch_size, image_embed_dim), text_cond=text_cond, cond_scale=cond_scale + ) + + # retrieve original unscaled image embed + + image_embeds /= self.image_embed_scale + + text_embeds = text_cond["text_embed"] + + text_embeds = rearrange(text_embeds, "(b r) d -> b r d", r=num_samples_per_batch) + image_embeds = rearrange(image_embeds, "(b r) d -> b r d", r=num_samples_per_batch) + + text_image_sims = einsum("b r d, b r d -> b r", l2norm(text_embeds), l2norm(image_embeds)) + top_sim_indices = text_image_sims.topk(k=1)[1] # .indices + + top_sim_indices = repeat(top_sim_indices, "b 1 -> b 1 d", d=image_embed_dim) + + top_image_embeds = image_embeds.gather(1, top_sim_indices) + return rearrange(top_image_embeds, "b 1 d -> b d") + + def forward( + self, + text=None, + image=None, + text_embed=None, # allow for training on preprocessed CLIP text and image embeddings + image_embed=None, + text_encodings=None, # as well as CLIP text encodings + text_mask=None, + *args, + **kwargs, + ): + assert exists(text) ^ exists(text_embed), "either text or text embedding must be supplied" + assert exists(image) ^ exists(image_embed), "either text or text embedding must be supplied" + assert not ( + self.condition_on_text_encodings and (not exists(text_encodings) and not exists(text)) + ), "text encodings must be present if you specified to condition on it on initialization" + + if exists(image): + image_embed, _ = self.clip.embed_image(image) + + # calculate text conditionings, based on what is passed in + + if exists(text): + text_embed, text_encodings, text_mask = self.clip.embed_text(text) + + text_cond = dict(text_embed=text_embed) + + if self.condition_on_text_encodings: + assert exists( + text_encodings + ), "text encodings must be present for diffusion prior if specified" + text_cond = {**text_cond, "text_encodings": text_encodings, "mask": text_mask} + + # timestep conditioning from ddpm + + batch = image_embed.shape[0] + times = flow.randint( + 0, + self.noise_scheduler.num_timesteps, + (batch,), + placement=get_default_placement(), + sbp=get_default_sbp(), + dtype=flow.long, + ) + + # scale image embed (Katherine) + + image_embed *= self.image_embed_scale + + # calculate forward loss + + return self.p_losses(image_embed, times, text_cond=text_cond, *args, **kwargs) + + +# decoder + + +def ConvTransposeUpsample(dim, dim_out=None): + dim_out = default(dim_out, dim) + return ConvTranspose2d(dim, dim_out, 4, 2, 1) + + +def NearestUpsample(dim, dim_out=None): + dim_out = default(dim_out, dim) + return nn.Sequential( + nn.Upsample(scale_factor=2, mode="nearest"), Conv2d(dim, dim_out, 3, padding=1) + ) + + +def Downsample(dim, *, dim_out=None): + dim_out = default(dim_out, dim) + return Conv2d(dim, dim_out, 4, 2, 1) + + +class SinusoidalPosEmb(nn.Module): + def __init__(self, dim): + super().__init__() + self.dim = dim + + def forward(self, x): + half_dim = self.dim // 2 + emb = math.log(10000) / (half_dim - 1) + emb = flow.exp( + flow.arange(half_dim, placement=get_default_placement(), sbp=get_default_sbp()) * -emb + ) + emb = rearrange(x, "i -> i 1") * rearrange(emb, "j -> 1 j") + return flow.cat((emb.sin(), emb.cos()), dim=-1) + + +class Block(nn.Module): + def __init__(self, dim, dim_out, groups=8): + super().__init__() + self.project = Conv2d(dim, dim_out, 3, padding=1) + self.norm = GroupNorm(groups, dim_out) + self.act = nn.SiLU() + + def forward(self, x, scale_shift=None): + x = self.project(x) + x = self.norm(x) + + if exists(scale_shift): + scale, shift = scale_shift + x = x * (scale + 1) + shift + + x = self.act(x) + return x + + +class ResnetBlock(nn.Module): + def __init__(self, dim, dim_out, *, cond_dim=None, time_cond_dim=None, groups=8): + super().__init__() + + self.time_mlp = None + + if exists(time_cond_dim): + self.time_mlp = nn.Sequential(nn.SiLU(), Linear(time_cond_dim, dim_out * 2)) + + self.cross_attn = None + + if exists(cond_dim): + self.cross_attn = EinopsToAndFrom( + "b c h w", "b (h w) c", CrossAttention(dim=dim_out, context_dim=cond_dim) + ) + + self.block1 = Block(dim, dim_out, groups=groups) + self.block2 = Block(dim_out, dim_out, groups=groups) + self.res_conv = Conv2d(dim, dim_out, 1) if dim != dim_out else nn.Identity() + + def forward(self, x, time_emb=None, cond=None): + + scale_shift = None + if exists(self.time_mlp) and exists(time_emb): + time_emb = self.time_mlp(time_emb) + time_emb = rearrange(time_emb, "b c -> b c 1 1") + scale_shift = time_emb.chunk(2, dim=1) + + h = self.block1(x, scale_shift=scale_shift) + + if exists(self.cross_attn): + assert exists(cond) + h = self.cross_attn(h, context=cond) + h + + h = self.block2(h) + return h + self.res_conv(x) + + +class CrossAttention(nn.Module): + def __init__( + self, dim, *, context_dim=None, dim_head=64, heads=8, dropout=0.0, norm_context=False + ): + super().__init__() + self.scale = dim_head ** -0.5 + self.heads = heads + inner_dim = dim_head * heads + + context_dim = default(context_dim, dim) + + self.norm = LayerNorm(dim) + self.norm_context = LayerNorm(context_dim) if norm_context else nn.Identity() + self.dropout = nn.Dropout(dropout) + + self.null_kv = nn.Parameter( + flow.randn(2, dim_head, placement=get_default_placement(), sbp=get_default_sbp()) + ) + self.to_q = Linear(dim, inner_dim, bias=False, parallel="col") + self.to_kv = Linear(context_dim, inner_dim * 2, bias=False, parallel="col") + + self.to_out = nn.Sequential( + Linear(inner_dim, dim, bias=False, parallel="row"), LayerNorm(dim) + ) + + def forward(self, x, context, mask=None): + b, n = x.shape[:2] + + x = self.norm(x) + context = self.norm_context(context) + + q, k, v = (self.to_q(x), *self.to_kv(context).chunk(2, dim=-1)) + + q, k, v = rearrange_many((q, k, v), "b n (h d) -> b h n d", h=self.heads) + + # add null key / value for classifier free guidance in prior net + + nk, nv = repeat_many(self.null_kv.unbind(dim=-2), "d -> b h 1 d", h=self.heads, b=b) + + k = flow.cat((nk, k), dim=-2) + v = flow.cat((nv, v), dim=-2) + + q = q * self.scale + + sim = einsum("b h i d, b h j d -> b h i j", q, k) + max_neg_value = -3.4028e38 # -flow.finfo(sim.dtype).max + + if exists(mask): + mask = F.pad(mask, (1, 0), value=1) + mask = rearrange(mask, "b j -> b 1 1 j") + sim = sim.masked_fill(1 - mask, max_neg_value) + + attn = sim.softmax(dim=-1) + + out = einsum("b h i j, b h j d -> b h i d", attn, v) + out = rearrange(out, "b h n d -> b n (h d)") + return self.to_out(out) + + +class LinearAttention(nn.Module): + def __init__(self, dim, dim_head=32, heads=8): + super().__init__() + self.scale = dim_head ** -0.5 + self.heads = heads + inner_dim = dim_head * heads + self.norm = ChanLayerNorm(dim) + + self.nonlin = nn.GELU() + self.to_qkv = Conv2d(dim, inner_dim * 3, 1, bias=False) + + self.to_out = nn.Sequential(Conv2d(inner_dim, dim, 1, bias=False), ChanLayerNorm(dim)) + + def forward(self, fmap): + h, x, y = self.heads, *fmap.shape[-2:] + + fmap = self.norm(fmap) + q, k, v = self.to_qkv(fmap).chunk(3, dim=1) + q, k, v = rearrange_many((q, k, v), "b (h c) x y -> (b h) (x y) c", h=h) + + q = q.softmax(dim=-1) + k = k.softmax(dim=-2) + + q = q * self.scale + + context = einsum("b n d, b n e -> b d e", k, v) + out = einsum("b n d, b d e -> b n e", q, context) + out = rearrange(out, "(b h) (x y) d -> b (h d) x y", h=h, x=x, y=y) + + out = self.nonlin(out) + return self.to_out(out) + + +class CrossEmbedLayer(nn.Module): + def __init__(self, dim_in, kernel_sizes, dim_out=None, stride=2): + super().__init__() + assert all([*map(lambda t: (t % 2) == (stride % 2), kernel_sizes)]) + dim_out = default(dim_out, dim_in) + + kernel_sizes = sorted(kernel_sizes) + num_scales = len(kernel_sizes) + + # calculate the dimension at each scale + dim_scales = [int(dim_out / (2 ** i)) for i in range(1, num_scales)] + dim_scales = [*dim_scales, dim_out - sum(dim_scales)] + + self.convs = nn.ModuleList([]) + for kernel, dim_scale in zip(kernel_sizes, dim_scales): + self.convs.append( + Conv2d(dim_in, dim_scale, kernel, stride=stride, padding=(kernel - stride) // 2) + ) + + def forward(self, x): + fmaps = tuple(map(lambda conv: conv(x), self.convs)) + return flow.cat(fmaps, dim=1) + + +class Unet(nn.Module): + def __init__( + self, + dim, + *, + image_embed_dim=None, + text_embed_dim=None, + cond_dim=None, + num_image_tokens=4, + num_time_tokens=2, + out_dim=None, + dim_mults=(1, 2, 4, 8), + channels=3, + channels_out=None, + self_attn=False, + attn_dim_head=32, + attn_heads=16, + lowres_cond=False, # + sparse_attn=False, + attend_at_middle=True, + cond_on_text_encodings=False, + max_text_len=256, + cond_on_image_embeds=False, + add_image_embeds_to_time=True, # + init_dim=None, + init_conv_kernel_size=7, + resnet_groups=8, + num_resnet_blocks=2, + init_cross_embed_kernel_sizes=(3, 7, 15), + cross_embed_downsample=False, + cross_embed_downsample_kernel_sizes=(2, 4), + memory_efficient=False, + scale_skip_connection=False, + nearest_upsample=False, + final_conv_kernel_size=1, + **kwargs, + ): + super().__init__() + # save locals to take care of some hyperparameters for cascading DDPM + + self._locals = locals() + del self._locals["self"] + del self._locals["__class__"] + + # for eventual cascading diffusion + + self.lowres_cond = lowres_cond + + # determine dimensions + + self.channels = channels + self.channels_out = default(channels_out, channels) + + init_channels = channels if not lowres_cond else channels * 2 + init_dim = default(init_dim, dim) + + self.init_conv = CrossEmbedLayer( + init_channels, dim_out=init_dim, kernel_sizes=init_cross_embed_kernel_sizes, stride=1 + ) + + dims = [init_dim, *map(lambda m: dim * m, dim_mults)] + in_out = list(zip(dims[:-1], dims[1:])) + + num_stages = len(in_out) + + # time, image embeddings, and optional text encoding + + cond_dim = default(cond_dim, dim) + time_cond_dim = dim * 4 + + self.to_time_hiddens = nn.Sequential( + SinusoidalPosEmb(dim), Linear(dim, time_cond_dim, parallel="col"), nn.GELU() + ) + + self.to_time_tokens = nn.Sequential( + Linear(time_cond_dim, cond_dim * num_time_tokens, parallel="row"), + Rearrange("b (r d) -> b r d", r=num_time_tokens), + ) + + self.to_time_cond = nn.Sequential(Linear(time_cond_dim, time_cond_dim, parallel="row")) + + self.image_to_tokens = ( + nn.Sequential( + Linear(image_embed_dim, cond_dim * num_image_tokens), + Rearrange("b (n d) -> b n d", n=num_image_tokens), + ) + if cond_on_image_embeds and image_embed_dim != cond_dim + else nn.Identity() + ) + + self.to_image_hiddens = ( + nn.Sequential(Linear(image_embed_dim, time_cond_dim), nn.GELU()) + if cond_on_image_embeds and add_image_embeds_to_time + else None + ) + + self.norm_cond = LayerNorm(cond_dim) + self.norm_mid_cond = LayerNorm(cond_dim) + + # text encoding conditioning (optional) + + self.text_to_cond = None + + if cond_on_text_encodings: + assert exists( + text_embed_dim + ), "text_embed_dim must be given to the unet if cond_on_text_encodings is True" + self.text_to_cond = Linear(text_embed_dim, cond_dim) + + # finer control over whether to condition on image embeddings and text encodings + # so one can have the latter unets in the cascading DDPMs only focus on super-resoluting + + self.cond_on_text_encodings = cond_on_text_encodings + self.cond_on_image_embeds = cond_on_image_embeds + + # for classifier free guidance + + self.null_image_embed = nn.Parameter(flow.randn(1, num_image_tokens, cond_dim)) + self.null_image_hiddens = nn.Parameter(flow.randn(1, time_cond_dim)) + + self.max_text_len = max_text_len + self.null_text_embed = nn.Parameter(flow.randn(1, max_text_len, cond_dim)) + + # whether to scale skip connection, adopted in Imagen + + self.skip_connect_scale = 1.0 if not scale_skip_connection else (2 ** -0.5) + + # attention related params + + attn_kwargs = dict(heads=attn_heads, dim_head=attn_dim_head) + + self_attn = cast_tuple(self_attn, num_stages) + + def create_self_attn(dim): + return EinopsToAndFrom("b c h w", "b (h w) c", Residual(Attention(dim, **attn_kwargs))) + + # resnet block klass + + resnet_groups = cast_tuple(resnet_groups, num_stages) + top_level_resnet_group = first(resnet_groups) + + num_resnet_blocks = cast_tuple(num_resnet_blocks, num_stages) + + # downsample klass + + downsample_klass = Downsample + if cross_embed_downsample: + downsample_klass = partial( + CrossEmbedLayer, kernel_sizes=cross_embed_downsample_kernel_sizes + ) + + # upsample klass + + upsample_klass = ConvTransposeUpsample if not nearest_upsample else NearestUpsample + + # give memory efficient unet an initial resnet block + + self.init_resnet_block = ( + ResnetBlock( + init_dim, init_dim, time_cond_dim=time_cond_dim, groups=top_level_resnet_group + ) + if memory_efficient + else None + ) + + # layers + + self.downs = nn.ModuleList([]) + self.ups = nn.ModuleList([]) + num_resolutions = len(in_out) + + skip_connect_dims = [] # keeping track of skip connection dimensions + + for ind, ((dim_in, dim_out), groups, layer_num_resnet_blocks, layer_self_attn) in enumerate( + zip(in_out, resnet_groups, num_resnet_blocks, self_attn) + ): + is_first = ind == 0 + is_last = ind >= (num_resolutions - 1) + layer_cond_dim = cond_dim if not is_first else None + + dim_layer = dim_out if memory_efficient else dim_in + skip_connect_dims.append(dim_layer) + + attention = nn.Identity() + if layer_self_attn: + attention = create_self_attn(dim_layer) + elif sparse_attn: + attention = Residual(LinearAttention(dim_layer, **attn_kwargs)) + + self.downs.append( + nn.ModuleList( + [ + downsample_klass(dim_in, dim_out=dim_out) if memory_efficient else None, + ResnetBlock( + dim_layer, dim_layer, time_cond_dim=time_cond_dim, groups=groups + ), + nn.ModuleList( + [ + ResnetBlock( + dim_layer, + dim_layer, + cond_dim=layer_cond_dim, + time_cond_dim=time_cond_dim, + groups=groups, + ) + for _ in range(layer_num_resnet_blocks) + ] + ), + attention, + downsample_klass(dim_layer, dim_out=dim_out) + if not is_last and not memory_efficient + else Conv2d(dim_layer, dim_out, 1), + ] + ) + ) + + mid_dim = dims[-1] + + self.mid_block1 = ResnetBlock( + mid_dim, + mid_dim, + cond_dim=cond_dim, + time_cond_dim=time_cond_dim, + groups=resnet_groups[-1], + ) + self.mid_attn = create_self_attn(mid_dim) + self.mid_block2 = ResnetBlock( + mid_dim, + mid_dim, + cond_dim=cond_dim, + time_cond_dim=time_cond_dim, + groups=resnet_groups[-1], + ) + + for ind, ((dim_in, dim_out), groups, layer_num_resnet_blocks, layer_self_attn) in enumerate( + zip( + reversed(in_out), + reversed(resnet_groups), + reversed(num_resnet_blocks), + reversed(self_attn), + ) + ): + is_last = ind >= (len(in_out) - 1) + layer_cond_dim = cond_dim if not is_last else None + + skip_connect_dim = skip_connect_dims.pop() + + attention = nn.Identity() + if layer_self_attn: + attention = create_self_attn(dim_out) + elif sparse_attn: + attention = Residual(LinearAttention(dim_out, **attn_kwargs)) + + self.ups.append( + nn.ModuleList( + [ + ResnetBlock( + dim_out + skip_connect_dim, + dim_out, + cond_dim=layer_cond_dim, + time_cond_dim=time_cond_dim, + groups=groups, + ), + nn.ModuleList( + [ + ResnetBlock( + dim_out + skip_connect_dim, + dim_out, + cond_dim=layer_cond_dim, + time_cond_dim=time_cond_dim, + groups=groups, + ) + for _ in range(layer_num_resnet_blocks) + ] + ), + attention, + upsample_klass(dim_out, dim_in) + if not is_last or memory_efficient + else nn.Identity(), + ] + ) + ) + + self.final_resnet_block = ResnetBlock( + dim * 2, dim, time_cond_dim=time_cond_dim, groups=top_level_resnet_group + ) + self.to_out = Conv2d( + dim, + self.channels_out, + kernel_size=final_conv_kernel_size, + padding=final_conv_kernel_size // 2, + ) + + # if the current settings for the unet are not correct + # for cascading DDPM, then reinit the unet with the right settings + def cast_model_parameters( + self, *, lowres_cond, channels, channels_out, cond_on_image_embeds, cond_on_text_encodings + ): + if ( + lowres_cond == self.lowres_cond + and channels == self.channels + and cond_on_image_embeds == self.cond_on_image_embeds + and cond_on_text_encodings == self.cond_on_text_encodings + and channels_out == self.channels_out + ): + return self + + updated_kwargs = dict( + lowres_cond=lowres_cond, + channels=channels, + channels_out=channels_out, + cond_on_image_embeds=cond_on_image_embeds, + cond_on_text_encodings=cond_on_text_encodings, + ) + + return self.__class__(**{**self._locals, **updated_kwargs}) + + def forward_with_cond_scale(self, *args, cond_scale=1.0, **kwargs): + logits = self.forward(*args, **kwargs) + + if cond_scale == 1: + return logits + + null_logits = self.forward( + *args, text_cond_drop_prob=1.0, image_cond_drop_prob=1.0, **kwargs + ) + return null_logits + (logits - null_logits) * cond_scale + + def forward( + self, + x, + time, + *, + image_embed, + lowres_cond_img=None, + text_encodings=None, + text_mask=None, + image_cond_drop_prob=0.0, + text_cond_drop_prob=0.0, + blur_sigma=None, + blur_kernel_size=None, + ): + batch_size = x.shape[0] + + # add low resolution conditioning, if present + + assert not ( + self.lowres_cond and not exists(lowres_cond_img) + ), "low resolution conditioning image must be present" + + if exists(lowres_cond_img): + x = flow.cat((x, lowres_cond_img), dim=1) + + # initial convolution + + x = self.init_conv(x) + r = x.clone() # final residual + # time conditioning + + time_hiddens = self.to_time_hiddens(time) + + time_tokens = self.to_time_tokens(time_hiddens) + t = self.to_time_cond(time_hiddens) + + # conditional dropout + + image_keep_mask = prob_mask_like((batch_size,), 1 - image_cond_drop_prob) + text_keep_mask = prob_mask_like((batch_size,), 1 - text_cond_drop_prob) + + text_keep_mask = rearrange(text_keep_mask, "b -> b 1 1") + + # image embedding to be summed to time embedding + # discovered by @mhh0318 in the paper + + if exists(image_embed) and exists(self.to_image_hiddens): + image_hiddens = self.to_image_hiddens(image_embed) + image_keep_mask_hidden = rearrange(image_keep_mask, "b -> b 1") + null_image_hiddens = self.null_image_hiddens.to(image_hiddens.dtype) + + image_hiddens = flow.where(image_keep_mask_hidden, image_hiddens, null_image_hiddens) + + t = t + image_hiddens + + # mask out image embedding depending on condition dropout + # for classifier free guidance + + image_tokens = None + + if self.cond_on_image_embeds: + image_keep_mask_embed = rearrange(image_keep_mask, "b -> b 1 1") + image_tokens = self.image_to_tokens(image_embed) + null_image_embed = self.null_image_embed.to( + image_tokens.dtype + ) # for some reason pyflow AMP not working + + image_tokens = flow.where(image_keep_mask_embed, image_tokens, null_image_embed) + + # take care of text encodings (optional) + + text_tokens = None + + if exists(text_encodings) and self.cond_on_text_encodings: + text_tokens = self.text_to_cond(text_encodings) + text_tokens = text_tokens[:, : self.max_text_len] + + text_tokens_len = text_tokens.shape[1] + remainder = self.max_text_len - text_tokens_len + + if remainder > 0: + text_tokens = F.pad(text_tokens, (0, 0, 0, remainder)) + + if exists(text_mask): + if remainder > 0: + # text_mask = F.pad(text_mask, (0, remainder), value = False) + text_mask = F.pad(text_mask.to(flow.int32), (0, remainder), value=0) + + text_mask = rearrange(text_mask, "b n -> b n 1") + text_keep_mask = text_mask & text_keep_mask + + null_text_embed = self.null_text_embed.to( + text_tokens.dtype + ) # for some reason pyflow AMP not working + + text_tokens = flow.where(text_keep_mask, text_tokens, null_text_embed) + + # main conditioning tokens (c) + c = time_tokens + + if exists(image_tokens): + c = flow.cat((c, image_tokens), dim=-2) + + # text and image conditioning tokens (mid_c), to save on compute, + # only do cross attention based conditioning on the inner most layers of the Unet + + mid_c = c if not exists(text_tokens) else flow.cat((c, text_tokens), dim=-2) + + # normalize conditioning tokens + c = self.norm_cond(c) + mid_c = self.norm_mid_cond(mid_c) + # initial resnet block + + if exists(self.init_resnet_block): + x = self.init_resnet_block(x, t) + # go through the layers of the unet, down and up + + hiddens = [] + for pre_downsample, init_block, resnet_blocks, attn, post_downsample in self.downs: + if exists(pre_downsample): + x = pre_downsample(x) + x = init_block(x, t, c) + for resnet_block in resnet_blocks: + x = resnet_block(x, t, c) + hiddens.append(x) + x = attn(x) + hiddens.append(x) + if exists(post_downsample): + x = post_downsample(x) + x = self.mid_block1(x, t, mid_c) + + if exists(self.mid_attn): + x = self.mid_attn(x) + + x = self.mid_block2(x, t, mid_c) + + def connect_skip(fmap): + return flow.cat((fmap, hiddens.pop() * self.skip_connect_scale), dim=1) + + for init_block, resnet_blocks, attn, upsample in self.ups: + x = connect_skip(x) + x = init_block(x, t, c) + + for resnet_block in resnet_blocks: + x = connect_skip(x) + x = resnet_block(x, t, c) + + x = attn(x) + x = upsample(x) + x = flow.cat((x, r), dim=1) + x = self.final_resnet_block(x, t) + return self.to_out(x) + + +class LowresConditioner(nn.Module): + def __init__( + self, + downsample_first=True, + blur_sigma=0.6, + blur_kernel_size=3, + ): + super().__init__() + self.downsample_first = downsample_first + self.blur_sigma = blur_sigma + self.blur_kernel_size = blur_kernel_size + + def forward( + self, + cond_fmap, + *, + target_image_size, + downsample_image_size=None, + blur_sigma=None, + blur_kernel_size=None, + ): + if self.training and self.downsample_first and exists(downsample_image_size): + cond_fmap = resize_image_to(cond_fmap, downsample_image_size) + + if self.training: + # when training, blur the low resolution conditional image + blur_sigma = default(blur_sigma, self.blur_sigma) + blur_kernel_size = default(blur_kernel_size, self.blur_kernel_size) + + # allow for drawing a random sigma between lo and hi float values + if isinstance(blur_sigma, tuple): + blur_sigma = tuple(map(float, blur_sigma)) + blur_sigma = random.uniform(*blur_sigma) + + # allow for drawing a random kernel size between lo and hi int values + if isinstance(blur_kernel_size, tuple): + blur_kernel_size = tuple(map(int, blur_kernel_size)) + kernel_size_lo, kernel_size_hi = blur_kernel_size + blur_kernel_size = random.randrange(kernel_size_lo, kernel_size_hi + 1) + + cond_fmap = gaussian_blur2d( + cond_fmap, cast_tuple(blur_kernel_size, 2), cast_tuple(blur_sigma, 2) + ) + + cond_fmap = resize_image_to(cond_fmap, target_image_size) + + return cond_fmap + + +class Decoder(nn.Module): + def __init__( + self, + unet, + *, + clip=None, + image_size=None, + channels=3, + vae=tuple(), + timesteps=1000, + image_cond_drop_prob=0.1, + text_cond_drop_prob=0.5, + loss_type="l2", + beta_schedule=None, + predict_x_start=False, + predict_x_start_for_latent_diffusion=False, + image_sizes=None, # for cascading ddpm, image size at each stage + random_crop_sizes=None, + lowres_downsample_first=True, + blur_sigma=0.6, # cascading ddpm - blur sigma + blur_kernel_size=3, # cascading ddpm - blur kernel size + clip_denoised=True, + clip_x_start=True, + clip_adapter_overrides=dict(), + learned_variance=True, + learned_variance_constrain_frac=False, + vb_loss_weight=0.001, + unconditional=False, # set to True for generating images without conditioning + auto_normalize_img=True, + use_dynamic_thres=False, # from the Imagen paper + dynamic_thres_percentile=0.9, + p2_loss_weight_gamma=0.0, + p2_loss_weight_k=1, + ): + super().__init__() + + # clip + + self.clip = None + if exists(clip): + assert not unconditional, "clip must not be given if doing unconditional image training" + assert ( + channels == clip.image_channels + ), f"channels of image ({channels}) should be equal to the" + " channels that CLIP accepts ({clip.image_channels})" + + freeze_model_and_make_eval_(clip) + self.clip = clip + + # determine image size, with image_size and image_sizes taking precedence + + if exists(image_size) or exists(image_sizes): + assert exists(image_size) ^ exists( + image_sizes + ), "only one of image_size or image_sizes must be given" + image_size = default(image_size, lambda: image_sizes[-1]) + elif exists(clip): + image_size = clip.image_size + else: + raise ("either image_size, image_sizes, or clip must be given to decoder") + + # channels + + self.channels = channels + + # verify conditioning method + + unets = cast_tuple(unet) + num_unets = len(unets) + + self.unconditional = unconditional + + # automatically take care of ensuring that first unet is unconditional while the rest + # of the unets are conditioned on the low resolution image produced by previous unet + + vaes = pad_tuple_to_length( + cast_tuple(vae), len(unets), fillvalue=NullVQGanVAE(channels=self.channels) + ) + + # whether to use learned variance, defaults to True for the first unet in the cascade + + learned_variance = pad_tuple_to_length( + cast_tuple(learned_variance), len(unets), fillvalue=False + ) + self.learned_variance = learned_variance + # whether to constrain the output of the network (the interpolation fraction) from 0 to 1 + self.learned_variance_constrain_frac = learned_variance_constrain_frac + self.vb_loss_weight = vb_loss_weight + + # construct unets and vaes + + self.unets = nn.ModuleList([]) + self.vaes = nn.ModuleList([]) + + for ind, (one_unet, one_vae, one_unet_learned_var) in enumerate( + zip(unets, vaes, learned_variance) + ): + assert isinstance(one_unet, Unet) + assert isinstance(one_vae, (VQGanVAE, NullVQGanVAE)) + + is_first = ind == 0 + latent_dim = one_vae.encoded_dim if exists(one_vae) else None + + unet_channels = default(latent_dim, self.channels) + unet_channels_out = unet_channels * (1 if not one_unet_learned_var else 2) + + one_unet = one_unet.cast_model_parameters( + lowres_cond=not is_first, + cond_on_image_embeds=not unconditional and is_first, + cond_on_text_encodings=not unconditional and one_unet.cond_on_text_encodings, + channels=unet_channels, + channels_out=unet_channels_out, + ) + + self.unets.append(one_unet) + self.vaes.append(one_vae.copy_for_eval()) + + # determine from unets whether conditioning on text encoding is needed + + self.condition_on_text_encodings = any([unet.cond_on_text_encodings for unet in self.unets]) + + # create noise schedulers per unet + + if not exists(beta_schedule): + beta_schedule = ( + "cosine", + *(("cosine",) * max(num_unets - 2, 0)), + *(("linear",) * int(num_unets > 1)), + ) + + beta_schedule = cast_tuple(beta_schedule, num_unets) + p2_loss_weight_gamma = cast_tuple(p2_loss_weight_gamma, num_unets) + + self.noise_schedulers = nn.ModuleList([]) + + for unet_beta_schedule, unet_p2_loss_weight_gamma in zip( + beta_schedule, p2_loss_weight_gamma + ): + noise_scheduler = NoiseScheduler( + beta_schedule=unet_beta_schedule, + timesteps=timesteps, + loss_type=loss_type, + p2_loss_weight_gamma=unet_p2_loss_weight_gamma, + p2_loss_weight_k=p2_loss_weight_k, + ) + + self.noise_schedulers.append(noise_scheduler) + + # unet image sizes + + image_sizes = default(image_sizes, (image_size,)) + image_sizes = tuple(sorted(set(image_sizes))) + + assert len(self.unets) == len( + image_sizes + ), "you did not supply the correct number of u-nets " + f"({len(self.unets)}) for resolutions {image_sizes}" + self.image_sizes = image_sizes + self.sample_channels = cast_tuple(self.channels, len(image_sizes)) + + # random crop sizes (for super-resoluting unets at the end of cascade?) + + self.random_crop_sizes = cast_tuple(random_crop_sizes, len(image_sizes)) + + # predict x0 config + + self.predict_x_start = ( + cast_tuple(predict_x_start, len(unets)) + if not predict_x_start_for_latent_diffusion + else tuple(map(lambda t: isinstance(t, VQGanVAE), self.vaes)) + ) + + # cascading ddpm related stuff + + lowres_conditions = tuple(map(lambda t: t.lowres_cond, self.unets)) + assert lowres_conditions == ( + False, + *((True,) * (len(self.unets) - 1)), + ), "the first unet must be unconditioned (by low resolution image), " + "and the rest of the unets must have `lowres_cond` set to True" + + self.to_lowres_cond = LowresConditioner( + downsample_first=lowres_downsample_first, + blur_sigma=blur_sigma, + blur_kernel_size=blur_kernel_size, + ) + + # classifier free guidance + + self.image_cond_drop_prob = image_cond_drop_prob + self.text_cond_drop_prob = text_cond_drop_prob + self.can_classifier_guidance = image_cond_drop_prob > 0.0 or text_cond_drop_prob > 0.0 + + # whether to clip when sampling + + self.clip_denoised = clip_denoised + self.clip_x_start = clip_x_start + + # dynamic thresholding settings, if clipping denoised during sampling + + self.use_dynamic_thres = use_dynamic_thres + self.dynamic_thres_percentile = dynamic_thres_percentile + + # normalize and unnormalize image functions + + self.normalize_img = normalize_neg_one_to_one if auto_normalize_img else identity + self.unnormalize_img = unnormalize_zero_to_one if auto_normalize_img else identity + + # device tracker + + self.register_buffer("_dummy", flow.Tensor([True]), persistent=False) + + def get_unet(self, unet_number): + assert 0 < unet_number <= len(self.unets) + index = unet_number - 1 + return self.unets[index] + + @contextmanager + def one_unet_in_gpu(self, unet_number=None, unet=None): + assert exists(unet_number) ^ exists(unet) + + if exists(unet_number): + unet = self.get_unet(unet_number) + + self.cuda() + devices = [module_device(unet) for unet in self.unets] + self.unets.cpu() + unet.cuda() + yield + for unet, device in zip(self.unets, devices): + unet.to(device) + + def p_mean_variance( + self, + unet, + x, + t, + image_embed, + noise_scheduler, + text_encodings=None, + text_mask=None, + lowres_cond_img=None, + clip_denoised=True, + predict_x_start=False, + learned_variance=False, + cond_scale=1.0, + model_output=None, + ): + assert not ( + cond_scale != 1.0 and not self.can_classifier_guidance + ), "the decoder was not trained with conditional dropout, " + "and thus one cannot use classifier free guidance (cond_scale anything other than 1)" + + pred = default( + model_output, + lambda: unet.forward_with_cond_scale( + x, + t, + image_embed=image_embed, + text_encodings=text_encodings, + text_mask=text_mask, + cond_scale=cond_scale, + lowres_cond_img=lowres_cond_img, + ), + ) + + if learned_variance: + pred, var_interp_frac_unnormalized = pred.chunk(2, dim=1) + if predict_x_start: + x_recon = pred + else: + x_recon = noise_scheduler.predict_start_from_noise(x, t=t, noise=pred) + + if clip_denoised: + # s is the threshold amount + # static thresholding would just be s = 1 + s = 1.0 + if self.use_dynamic_thres: + s = flow.quantile( + rearrange(x_recon, "b ... -> b (...)").abs(), + self.dynamic_thres_percentile, + dim=-1, + ) + + s.clamp_(min=1.0) + s = s.view(-1, *((1,) * (x_recon.ndim - 1))) + + # clip by threshold, depending on whether static or dynamic + x_recon = x_recon.clamp(-s, s) / s + + model_mean, posterior_variance, posterior_log_variance = noise_scheduler.q_posterior( + x_start=x_recon, x_t=x, t=t + ) + + if learned_variance: + # if learned variance, posterio variance and posterior log variance are + # predicted by the network by an interpolation of the max and min log beta values + # eq 15 - https://arxiv.org/abs/2102.09672 + min_log = extract(noise_scheduler.posterior_log_variance_clipped, t, x.shape) + max_log = extract(flow.log(noise_scheduler.betas), t, x.shape) + var_interp_frac = unnormalize_zero_to_one(var_interp_frac_unnormalized) + + if self.learned_variance_constrain_frac: + var_interp_frac = var_interp_frac.sigmoid() + + posterior_log_variance = var_interp_frac * max_log + (1 - var_interp_frac) * min_log + posterior_variance = posterior_log_variance.exp() + + return model_mean, posterior_variance, posterior_log_variance + + @flow.no_grad() + def p_sample( + self, + unet, + x, + t, + image_embed, + noise_scheduler, + text_encodings=None, + text_mask=None, + cond_scale=1.0, + lowres_cond_img=None, + predict_x_start=False, + learned_variance=False, + clip_denoised=True, + ): + b = x.shape[0] + model_mean, _, model_log_variance = self.p_mean_variance( + unet, + x=x, + t=t, + image_embed=image_embed, + text_encodings=text_encodings, + text_mask=text_mask, + cond_scale=cond_scale, + lowres_cond_img=lowres_cond_img, + clip_denoised=clip_denoised, + predict_x_start=predict_x_start, + noise_scheduler=noise_scheduler, + learned_variance=learned_variance, + ) + noise = flow.randn(*x.shape, placement=get_default_placement(), sbp=get_default_sbp()) + # no noise when t == 0 + nonzero_mask = (1 - (t == 0).float()).reshape(b, *((1,) * (len(x.shape) - 1))) + return model_mean + nonzero_mask * (0.5 * model_log_variance).exp() * noise + + @flow.no_grad() + def p_sample_loop( + self, + unet, + shape, + image_embed, + noise_scheduler, + predict_x_start=False, + learned_variance=False, + clip_denoised=True, + lowres_cond_img=None, + text_encodings=None, + text_mask=None, + cond_scale=1, + is_latent_diffusion=False, + ): + + b = shape[0] + img = flow.randn(*shape, placement=get_default_placement(), sbp=get_default_sbp()) + + if not is_latent_diffusion: + lowres_cond_img = maybe(self.normalize_img)(lowres_cond_img) + + for i in tqdm( + reversed(range(0, noise_scheduler.num_timesteps)), + desc="sampling loop time step", + total=noise_scheduler.num_timesteps, + ): + img = self.p_sample( + unet, + img, + flow.full( + (b,), + i, + placement=get_default_placement(), + sbp=get_default_sbp(), + dtype=flow.long, + ), + image_embed=image_embed, + text_encodings=text_encodings, + text_mask=text_mask, + cond_scale=cond_scale, + lowres_cond_img=lowres_cond_img, + predict_x_start=predict_x_start, + noise_scheduler=noise_scheduler, + learned_variance=learned_variance, + clip_denoised=clip_denoised, + ) + + unnormalize_img = self.unnormalize_img(img) + return unnormalize_img + + def p_losses( + self, + unet, + x_start, + times, + *, + image_embed, + noise_scheduler, + lowres_cond_img=None, + text_encodings=None, + text_mask=None, + predict_x_start=False, + noise=None, + learned_variance=False, + clip_denoised=False, + is_latent_diffusion=False, + ): + noise = default( + noise, + lambda: flow.randn( + *x_start.shape, placement=get_default_placement(), sbp=get_default_sbp() + ), + ) + + # normalize to [-1, 1] + + if not is_latent_diffusion: + x_start = self.normalize_img(x_start) + lowres_cond_img = maybe(self.normalize_img)(lowres_cond_img) + + # get x_t + + x_noisy = noise_scheduler.q_sample(x_start=x_start, t=times, noise=noise) + + model_output = unet( + x_noisy, + times, + image_embed=image_embed, + text_encodings=text_encodings, + text_mask=text_mask, + lowres_cond_img=lowres_cond_img, + image_cond_drop_prob=self.image_cond_drop_prob, + text_cond_drop_prob=self.text_cond_drop_prob, + ) + + if learned_variance: + pred, _ = model_output.chunk(2, dim=1) + else: + pred = model_output + + target = noise if not predict_x_start else x_start + + loss = noise_scheduler.loss_fn(pred, target, reduction="none") + loss = reduce(loss, "b ... -> b (...)", "mean") + + loss = noise_scheduler.p2_reweigh_loss(loss, times) + + loss = loss.mean() + + if not learned_variance: + # return simple loss if not using learned variance + return loss + + # most of the code below is transcribed from + # https://github.com/hojonathanho/diffusion/blob/master/diffusion_tf/diffusion_utils_2.py + # the Improved DDPM paper then further modified it so that the mean is detached + # (shown a couple lines before), and weighted to be smaller than the l1 or l2 "simple" loss + # it is questionable whether this is really needed, looking at some of the figures in the + # paper, but may as well stay faithful to their implementation + + # if learning the variance, also include the extra weight kl loss + + true_mean, _, true_log_variance_clipped = noise_scheduler.q_posterior( + x_start=x_start, x_t=x_noisy, t=times + ) + model_mean, _, model_log_variance = self.p_mean_variance( + unet, + x=x_noisy, + t=times, + image_embed=image_embed, + noise_scheduler=noise_scheduler, + clip_denoised=clip_denoised, + learned_variance=True, + model_output=model_output, + ) + + # kl loss with detached model predicted mean, for stability reasons as in paper + + detached_model_mean = model_mean.detach() + + kl = normal_kl( + true_mean, true_log_variance_clipped, detached_model_mean, model_log_variance + ) + kl = meanflat(kl) * NAT + + decoder_nll = -discretized_gaussian_log_likelihood( + x_start, means=detached_model_mean, log_scales=0.5 * model_log_variance + ) + decoder_nll = meanflat(decoder_nll) * NAT + + # at the first timestep return the decoder NLL, + # otherwise KL(q(x_{t-1}|x_t,x_0) || p(x_{t-1}|x_t)) + + vb_losses = flow.where(times == 0, decoder_nll, kl) + + # weight the vb loss smaller, for stability, as in the paper (recommended 0.001) + + vb_loss = vb_losses.mean() * self.vb_loss_weight + + return loss + vb_loss + + @flow.no_grad() + @eval_decorator + def sample( + self, + image_embed=None, + text=None, + text_mask=None, + text_encodings=None, + batch_size=1, + cond_scale=1.0, + stop_at_unet_number=None, + distributed=False, + ): + assert self.unconditional or exists( + image_embed + ), "image embed must be present on sampling from decoder unless if trained unconditionally" + + if not self.unconditional: + batch_size = image_embed.shape[0] + + if exists(text) and not exists(text_encodings) and not self.unconditional: + assert exists(self.clip) + _, text_encodings, text_mask = self.clip.embed_text(text) + + assert not ( + self.condition_on_text_encodings and not exists(text_encodings) + ), "text or text encodings must be passed into decoder if specified" + assert not ( + not self.condition_on_text_encodings and exists(text_encodings) + ), "decoder specified not to be conditioned on text, yet it is presented" + + img = None + + for ( + unet_number, + unet, + vae, + channel, + image_size, + predict_x_start, + learned_variance, + noise_scheduler, + ) in tqdm( + zip( + range(1, len(self.unets) + 1), + self.unets, + self.vaes, + self.sample_channels, + self.image_sizes, + self.predict_x_start, + self.learned_variance, + self.noise_schedulers, + ) + ): + + context = null_context() + + with context: + lowres_cond_img = None + shape = (batch_size, channel, image_size, image_size) + + if unet.lowres_cond: + lowres_cond_img = self.to_lowres_cond(img, target_image_size=image_size) + + is_latent_diffusion = isinstance(vae, VQGanVAE) + image_size = vae.get_encoded_fmap_size(image_size) + shape = (batch_size, vae.encoded_dim, image_size, image_size) + + lowres_cond_img = maybe(vae.encode)(lowres_cond_img) + + img = self.p_sample_loop( + unet, + shape, + image_embed=image_embed, + text_encodings=text_encodings, + text_mask=text_mask, + cond_scale=cond_scale, + predict_x_start=predict_x_start, + learned_variance=learned_variance, + clip_denoised=not is_latent_diffusion, + lowres_cond_img=lowres_cond_img, + is_latent_diffusion=is_latent_diffusion, + noise_scheduler=noise_scheduler, + ) + + img = vae.decode(img) + + if exists(stop_at_unet_number) and stop_at_unet_number == unet_number: + break + + return img + + def forward( + self, + image, + text=None, + image_embed=None, + text_encodings=None, + text_mask=None, + unet_number=None, + return_lowres_cond_image=False, + ): + assert not ( + len(self.unets) > 1 and not exists(unet_number) + ), f"you must specify which unet you want trained, from a range of 1 to {len(self.unets)}," + " if you are training cascading DDPM (multiple unets)" + unet_number = default(unet_number, 1) + unet_index = unet_number - 1 + + unet = self.get_unet(unet_number) + + vae = self.vaes[unet_index] + noise_scheduler = self.noise_schedulers[unet_index] + target_image_size = self.image_sizes[unet_index] + predict_x_start = self.predict_x_start[unet_index] + random_crop_size = self.random_crop_sizes[unet_index] + learned_variance = self.learned_variance[unet_index] + b, _, h, w, _, = ( + *image.shape, + image.device, + ) + + check_shape(image, "b c h w", c=self.channels) + assert h >= target_image_size and w >= target_image_size + + times = flow.randint( + 0, + noise_scheduler.num_timesteps, + (b,), + placement=get_default_placement(), + sbp=get_default_sbp(), + dtype=flow.long, + ) + + if not exists(image_embed) and not self.unconditional: + assert exists(self.clip), "if you want to derive CLIP image embeddings automatically, " + "you must supply `clip` to the decoder on init" + image_embed, _ = self.clip.embed_image(image) + + if exists(text) and not exists(text_encodings) and not self.unconditional: + assert exists( + self.clip + ), "if you are passing in raw text, you need to supply `clip` to the decoder" + _, text_encodings, text_mask = self.clip.embed_text(text) + + assert not ( + self.condition_on_text_encodings and not exists(text_encodings) + ), "text or text encodings must be passed into decoder if specified" + assert not ( + not self.condition_on_text_encodings and exists(text_encodings) + ), "decoder specified not to be conditioned on text, yet it is presented" + + lowres_cond_img = ( + self.to_lowres_cond( + image, + target_image_size=target_image_size, + downsample_image_size=self.image_sizes[unet_index - 1], + ) + if unet_number > 1 + else None + ) + image = resize_image_to(image, target_image_size) + + if exists(random_crop_size): + aug = K.RandomCrop((random_crop_size, random_crop_size), p=1.0) + # make sure low res conditioner and image both get augmented the same way + image = aug(image) + lowres_cond_img = aug(lowres_cond_img, params=aug._params) + + is_latent_diffusion = not isinstance(vae, NullVQGanVAE) + + vae.eval() + with flow.no_grad(): + image = vae.encode(image) + lowres_cond_img = maybe(vae.encode)(lowres_cond_img) + + losses = self.p_losses( + unet, + image, + times, + image_embed=image_embed, + text_encodings=text_encodings, + text_mask=text_mask, + lowres_cond_img=lowres_cond_img, + predict_x_start=predict_x_start, + learned_variance=learned_variance, + is_latent_diffusion=is_latent_diffusion, + noise_scheduler=noise_scheduler, + ) + + if not return_lowres_cond_image: + return losses + + return losses, lowres_cond_img + + +# main class + + +class DALLE2(nn.Module): + def __init__(self, *, prior, decoder, prior_num_samples=2, **kwargs): + super().__init__() + # assert isinstance(prior, DiffusionPrior) + # assert isinstance(decoder, Decoder) + self.prior = prior + self.decoder = decoder + self.tokenizer = SimpleTokenizer() + + self.prior_num_samples = prior_num_samples + self.decoder_need_text_cond = self.decoder.condition_on_text_encodings + + self.to_pil = T.ToPILImage() + + @flow.no_grad() + @eval_decorator + def forward(self, text, cond_scale=1.0, prior_cond_scale=1.0, return_pil_images=False): + device = module_device(self) + one_text = isinstance(text, str) or (not is_list_str(text) and text.shape[0] == 1) + + if isinstance(text, str) or is_list_str(text): + text = [text] if not isinstance(text, (list, tuple)) else text + text = self.tokenizer.tokenize(text).to(device) + + image_embed = self.prior.sample( + text, num_samples_per_batch=self.prior_num_samples, cond_scale=prior_cond_scale + ) + + text_cond = text if self.decoder_need_text_cond else None + images = self.decoder.sample(image_embed, text=text_cond, cond_scale=cond_scale) + + if return_pil_images: + images = list(map(self.to_pil, images.unbind(dim=0))) + + if one_text: + return first(images) + + return images diff --git a/projects/DALLE2/dalle2/rotary_embedding_flow.py b/projects/DALLE2/dalle2/rotary_embedding_flow.py new file mode 100644 index 0000000000000000000000000000000000000000..997fbf523894974f0522548f1be5a0a618597162 --- /dev/null +++ b/projects/DALLE2/dalle2/rotary_embedding_flow.py @@ -0,0 +1,133 @@ +from inspect import isfunction +from math import pi + +import oneflow as flow +from einops import rearrange, repeat +from oneflow import einsum, nn + +from libai.utils import distributed as dist + +# helper functions + + +def exists(val): + return val is not None + + +def broadcat(tensors, dim=-1): + num_tensors = len(tensors) + shape_lens = set(list(map(lambda t: len(t.shape), tensors))) + assert len(shape_lens) == 1, "tensors must all have the same number of dimensions" + shape_len = list(shape_lens)[0] + + dim = (dim + shape_len) if dim < 0 else dim + dims = list(zip(*map(lambda t: list(t.shape), tensors))) + + expandable_dims = [(i, val) for i, val in enumerate(dims) if i != dim] + assert all( + [*map(lambda t: len(set(t[1])) <= 2, expandable_dims)] + ), "invalid dimensions for broadcastable concatentation" + max_dims = list(map(lambda t: (t[0], max(t[1])), expandable_dims)) + expanded_dims = list(map(lambda t: (t[0], (t[1],) * num_tensors), max_dims)) + expanded_dims.insert(dim, (dim, dims[dim])) + expandable_shapes = list(zip(*map(lambda t: t[1], expanded_dims))) + tensors = list(map(lambda t: t[0].expand(*t[1]), zip(tensors, expandable_shapes))) + return flow.cat(tensors, dim=dim) + + +# rotary embedding helper functions + + +def rotate_half(x): + x = rearrange(x, "... (d r) -> ... d r", r=2) + x1, x2 = [i.squeeze(-1) for i in x.chunk(2, -1)] # x.unbind(dim = -1) + x = flow.stack((-x2, x1), dim=-1) + return rearrange(x, "... d r -> ... (d r)") + + +def apply_rotary_emb(freqs, t, start_index=0): + freqs = freqs.to(t.dtype) + rot_dim = freqs.shape[-1] + end_index = start_index + rot_dim + assert ( + rot_dim <= t.shape[-1] + ), f"feature dimension {t.shape[-1]} is not of sufficient size to \ + rotate in all the positions {rot_dim}" + t_left, t, t_right = t[..., :start_index], t[..., start_index:end_index], t[..., end_index:] + t = (t * freqs.cos()) + (rotate_half(t) * freqs.sin()) + return flow.cat((t_left, t, t_right), dim=-1) + + +# learned rotation helpers + + +def apply_learned_rotations(rotations, t, start_index=0, freq_ranges=None): + if exists(freq_ranges): + rotations = einsum("..., f -> ... f", rotations, freq_ranges) + rotations = rearrange(rotations, "... r f -> ... (r f)") + + rotations = repeat(rotations, "... n -> ... (n r)", r=2) + return apply_rotary_emb(rotations, t, start_index=start_index) + + +# classes + + +class RotaryEmbedding(nn.Module): + def __init__( + self, + dim, + custom_freqs=None, + freqs_for="lang", + theta=10000, + max_freq=10, + num_freqs=1, + learned_freq=False, + ): + super().__init__() + if exists(custom_freqs): + freqs = custom_freqs + elif freqs_for == "lang": + freqs = 1.0 / (theta ** (flow.arange(0, dim, 2)[: (dim // 2)].float() / dim)) + elif freqs_for == "pixel": + freqs = flow.linspace(1.0, max_freq / 2, dim // 2) * pi + elif freqs_for == "constant": + freqs = flow.ones(num_freqs).float() + else: + raise ValueError(f"unknown modality {freqs_for}") + + self.cache = dict() + + if learned_freq: + self.freqs = nn.Parameter(freqs) + else: + self.register_buffer("freqs", freqs) + + def rotate_queries_or_keys(self, t, seq_dim=-2): + seq_len = t.shape[seq_dim] + freqs = self.forward( + lambda: flow.arange( + seq_len, + placement=dist.get_layer_placement(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ), + cache_key=seq_len, + ) + return apply_rotary_emb(freqs, t) + + def forward(self, t, cache_key=None): + if exists(cache_key) and cache_key in self.cache: + return self.cache[cache_key] + + if isfunction(t): + t = t() + + freqs = self.freqs + + freqs = flow.einsum("..., f -> ... f", t.to(freqs.dtype), freqs) + freqs = repeat(freqs, "... n -> ... (n r)", r=2) + + if exists(cache_key): + self.cache[cache_key] = freqs + + return freqs diff --git a/projects/DALLE2/dalle2/tokenizer.py b/projects/DALLE2/dalle2/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..71bd2f17e86afdb016036c5e71e7640dfd0cf182 --- /dev/null +++ b/projects/DALLE2/dalle2/tokenizer.py @@ -0,0 +1,219 @@ +# take from https://github.com/openai/CLIP/blob/main/clip/simple_tokenizer.py +# to give users a quick easy start to training DALL-E without doing BPE + +import gzip +import html +import os +from functools import lru_cache +from pathlib import Path + +import ftfy +import oneflow as flow +import regex as re + +from .utils import import_or_print_error + +# OpenAI simple tokenizer + + +@lru_cache() +def default_bpe(): + return os.path.join( + os.path.dirname(os.path.abspath(__file__)), "data/bpe_simple_vocab_16e6.txt.gz" + ) + + +@lru_cache() +def bytes_to_unicode(): + bs = ( + list(range(ord("!"), ord("~") + 1)) + + list(range(ord("¡"), ord("¬") + 1)) + + list(range(ord("®"), ord("ÿ") + 1)) + ) + cs = bs[:] + n = 0 + for b in range(2 ** 8): + if b not in bs: + bs.append(b) + cs.append(2 ** 8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r"\s+", " ", text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + def __init__(self, bpe_path=default_bpe()): + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = ( + gzip.open(bpe_path).read().decode("utf-8").split("\n") + ) # Path(bpe_path).read_text(encoding='utf8').split('\n') + merges = merges[1 : 49152 - 256 - 2 + 1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v + "" for v in vocab] + for merge in merges: + vocab.append("".join(merge)) + vocab.extend(["<|startoftext|>", "<|endoftext|>"]) + + self.vocab_size = 49408 + + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = {"<|startoftext|>": "<|startoftext|>", "<|endoftext|>": "<|endoftext|>"} + self.pat = re.compile( + r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", # noqa + re.IGNORECASE, + ) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + (token[-1] + "",) + pairs = get_pairs(word) + + if not pairs: + return token + "" + + while True: + bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf"))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except ValueError: + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word) - 1 and word[i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = " ".join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = "".join(self.byte_encoder[b] for b in token.encode("utf-8")) + bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(" ")) + return bpe_tokens + + def decode(self, tokens, remove_start_end=True, pad_tokens=set()): + if flow.is_tensor(tokens): + tokens = tokens.tolist() + + if remove_start_end: + tokens = [token for token in tokens if token not in (49406, 40407, 0)] + text = "".join([self.decoder[token] for token in tokens if token not in pad_tokens]) + text = ( + bytearray([self.byte_decoder[c] for c in text]) + .decode("utf-8", errors="replace") + .replace("", " ") + ) + return text + + def tokenize(self, texts, context_length=256, truncate_text=False): + if isinstance(texts, str): + texts = [texts] + + all_tokens = [self.encode(text) for text in texts] + result = flow.zeros(len(all_tokens), context_length, dtype=flow.long) + + for i, tokens in enumerate(all_tokens): + if len(tokens) > context_length: + if truncate_text: + tokens = tokens[:context_length] + else: + raise RuntimeError( + f"Input {texts[i]} is too long for context length {context_length}" + ) + result[i, : len(tokens)] = flow.tensor(tokens) + + return result + + +# tokenizer = SimpleTokenizer() + +# YTTM tokenizer + + +class YttmTokenizer: + def __init__(self, bpe_path=None): + bpe_path = Path(bpe_path) + assert bpe_path.exists(), f"BPE json path {str(bpe_path)} does not exist" + + self.yttm = import_or_print_error( + "youtokentome", "you need to install youtokentome by `pip install youtokentome`" + ) + + tokenizer = self.yttm.BPE(model=str(bpe_path)) + self.tokenizer = tokenizer + self.vocab_size = tokenizer.vocab_size() + + def decode(self, tokens, pad_tokens=set()): + if flow.is_tensor(tokens): + tokens = tokens.tolist() + + return self.tokenizer.decode(tokens, ignore_ids=pad_tokens.union({0})) + + def encode(self, texts): + encoded = self.tokenizer.encode(texts, output_type=self.yttm.OutputType.ID) + return list(map(flow.tensor, encoded)) + + def tokenize(self, texts, context_length=256, truncate_text=False): + if isinstance(texts, str): + texts = [texts] + + all_tokens = self.encode(texts) + + result = flow.zeros(len(all_tokens), context_length, dtype=flow.long) + for i, tokens in enumerate(all_tokens): + if len(tokens) > context_length: + if truncate_text: + tokens = tokens[:context_length] + else: + raise RuntimeError( + f"Input {texts[i]} is too long for context length {context_length}" + ) + result[i, : len(tokens)] = flow.tensor(tokens) + + return result diff --git a/projects/DALLE2/dalle2/utils.py b/projects/DALLE2/dalle2/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..aecfca3dc6c12c1cd44987e86f81926e85594ce1 --- /dev/null +++ b/projects/DALLE2/dalle2/utils.py @@ -0,0 +1,42 @@ +import importlib +import time + +# helper functions + + +def exists(val): + return val is not None + + +# time helpers + + +class Timer: + def __init__(self): + self.reset() + + def reset(self): + self.last_time = time.time() + + def elapsed(self): + return time.time() - self.last_time + + +# print helpers + + +def print_ribbon(s, symbol="=", repeat=40): + flank = symbol * repeat + return f"{flank} {s} {flank}" + + +# import helpers + + +def import_or_print_error(pkg_name, err_str=None): + try: + return importlib.import_module(pkg_name) + except ModuleNotFoundError: + if exists(err_str): + print(err_str) + exit() diff --git a/projects/DALLE2/dalle2/vector_quantize_flow.py b/projects/DALLE2/dalle2/vector_quantize_flow.py new file mode 100644 index 0000000000000000000000000000000000000000..08bbbd7584e9bb8335dd08f431a08b7ab3190e49 --- /dev/null +++ b/projects/DALLE2/dalle2/vector_quantize_flow.py @@ -0,0 +1,618 @@ +# from https://github.com/lucidrains/vector_quantize_pytorch/vector_quantize_pytorch.py + +import oneflow as flow +import oneflow.nn.functional as F +from einops import rearrange, repeat +from oneflow import einsum, nn + +from libai.utils import distributed + + +def exists(val): + return val is not None + + +def default(val, d): + return val if exists(val) else d + + +def noop(*args, **kwargs): + pass + + +def l2norm(t): + return F.normalize(t, p=2, dim=-1) + + +def log(t, eps=1e-20): + return flow.log(t.clamp(min=eps)) + + +def uniform_init(*shape): + t = flow.empty(shape) + nn.init.kaiming_uniform_(t) + return t + + +def gumbel_noise(t): + noise = flow.zeros_like(t).uniform_(0, 1) + return -log(-log(noise)) + + +def gumbel_sample(t, temperature=1.0, dim=-1): + if temperature == 0: + return t.argmax(dim=dim) + + return ((t / temperature) + gumbel_noise(t)).argmax(dim=dim) + + +def ema_inplace(moving_avg, new, decay): + moving_avg.data.mul_(decay).add_(new, alpha=(1 - decay)) + + +def laplace_smoothing(x, n_categories, eps=1e-5): + return (x + eps) / (x.sum() + n_categories * eps) + + +def sample_vectors(samples, num): + num_samples, device = samples.shape[0], samples.device + if num_samples >= num: + indices = flow.randperm(num_samples, device=device)[:num] + else: + indices = flow.randint(0, num_samples, (num,), device=device) + + return samples[indices] + + +def batched_sample_vectors(samples, num): + return flow.stack([sample_vectors(sample, num) for sample in samples.unbind(dim=0)], dim=0) + + +def pad_shape(shape, size, dim=0): + return [size if i == dim else s for i, s in enumerate(shape)] + + +def sample_multinomial(total_count, probs): + device = probs.device + probs = probs.cpu() + + total_count = probs.new_full((), total_count) + remainder = probs.new_ones(()) + sample = flow.empty_like(probs, dtype=flow.long) + + for i, p in enumerate(probs): + s = flow.binomial(total_count, p / remainder) + sample[i] = s + total_count -= s + remainder -= p + + return sample.to(device) + + +def all_gather_sizes(x, dim): + size = flow.tensor(x.shape[dim], dtype=flow.long, device=x.device) + all_sizes = [flow.empty_like(size) for _ in range(distributed.get_world_size())] + distributed.all_gather(all_sizes, size) + + return flow.stack(all_sizes) + + +def all_gather_variably_sized(x, sizes, dim=0): + rank = distributed.get_rank() + all_x = [] + + for i, size in enumerate(sizes): + t = x if i == rank else x.new_empty(pad_shape(x.shape, size, dim)) + distributed.broadcast(t, src=i, async_op=True) + all_x.append(t) + + distributed.barrier() + return all_x + + +def sample_vectors_distributed(local_samples, num): + rank = distributed.get_rank() + all_num_samples = all_gather_sizes(local_samples, dim=0) + + if rank == 0: + samples_per_rank = sample_multinomial(num, all_num_samples / all_num_samples.sum()) + else: + samples_per_rank = flow.empty_like(all_num_samples) + + distributed.broadcast(samples_per_rank, src=0) + samples_per_rank = samples_per_rank.tolist() + + local_samples = batched_sample_vectors(local_samples, samples_per_rank[rank]) + all_samples = all_gather_variably_sized(local_samples, samples_per_rank, dim=0) + return flow.cat(all_samples, dim=0) + + +def batched_bincount(x, *, minlength): + batch, dtype, device = x.shape[0], x.dtype, x.device + target = flow.zeros(batch, minlength, dtype=dtype, device=device) + values = flow.ones_like(x) + target.scatter_add_(-1, x, values) + return target + + +def kmeans( + samples, + num_clusters, + num_iters=10, + use_cosine_sim=False, + sample_fn=batched_sample_vectors, + all_reduce_fn=noop, +): + num_codebooks, dim, dtype, _ = ( + samples.shape[0], + samples.shape[-1], + samples.dtype, + samples.device, + ) + + means = sample_fn(samples, num_clusters) + + for _ in range(num_iters): + if use_cosine_sim: + dists = samples @ rearrange(means, "h n d -> h d n") + else: + dists = -flow.cdist(samples, means, p=2) + + buckets = flow.argmax(dists, dim=-1) + bins = batched_bincount(buckets, minlength=num_clusters) + all_reduce_fn(bins) + + zero_mask = bins == 0 + bins_min_clamped = bins.masked_fill(zero_mask, 1) + + new_means = buckets.new_zeros(num_codebooks, num_clusters, dim, dtype=dtype) + + new_means.scatter_add_(1, repeat(buckets, "h n -> h n d", d=dim), samples) + new_means = new_means / rearrange(bins_min_clamped, "... -> ... 1") + all_reduce_fn(new_means) + + if use_cosine_sim: + new_means = l2norm(new_means) + + means = flow.where(rearrange(zero_mask, "... -> ... 1"), means, new_means) + + return means, bins + + +def batched_embedding(indices, embeds): + batch, dim = indices.shape[1], embeds.shape[-1] + indices = repeat(indices, "h b n -> h b n d", d=dim) + embeds = repeat(embeds, "h c d -> h b c d", b=batch) + return embeds.gather(2, indices) + + +# regularization losses + + +def orthgonal_loss_fn(t): + # eq (2) from https://arxiv.org/abs/2112.00384 + h, n = t.shape[:2] + normed_codes = l2norm(t) + identity = repeat(flow.eye(n, device=t.device), "i j -> h i j", h=h) + cosine_sim = einsum("h i d, h j d -> h i j", normed_codes, normed_codes) + return ((cosine_sim - identity) ** 2).sum() / (h * n ** 2) + + +# distance types + + +class EuclideanCodebook(nn.Module): + def __init__( + self, + dim, + codebook_size, + num_codebooks=1, + kmeans_init=False, + kmeans_iters=10, + decay=0.8, + eps=1e-5, + threshold_ema_dead_code=2, + use_ddp=False, + learnable_codebook=False, + sample_codebook_temp=0, + ): + super().__init__() + self.decay = decay + init_fn = uniform_init if not kmeans_init else flow.zeros + embed = init_fn(num_codebooks, codebook_size, dim) + + self.codebook_size = codebook_size + self.num_codebooks = num_codebooks + + self.kmeans_iters = kmeans_iters + self.eps = eps + self.threshold_ema_dead_code = threshold_ema_dead_code + self.sample_codebook_temp = sample_codebook_temp + + self.sample_fn = sample_vectors_distributed if use_ddp else batched_sample_vectors + self.all_reduce_fn = distributed.all_reduce if use_ddp else noop + + self.register_buffer("initted", flow.Tensor([not kmeans_init])) + self.register_buffer("cluster_size", flow.zeros(num_codebooks, codebook_size)) + self.register_buffer("embed_avg", embed.clone()) + + self.learnable_codebook = learnable_codebook + if learnable_codebook: + self.embed = nn.Parameter(embed) + else: + self.register_buffer("embed", embed) + + def init_embed_(self, data): + if self.initted: + return + + embed, cluster_size = kmeans( + data, + self.codebook_size, + self.kmeans_iters, + sample_fn=self.sample_fn, + all_reduce_fn=self.all_reduce_fn, + ) + + self.embed.data.copy_(embed) + self.embed_avg.data.copy_(embed.clone()) + self.cluster_size.data.copy_(cluster_size) + self.initted.data.copy_(flow.Tensor([True])) + + def replace(self, batch_samples, batch_mask): + batch_samples = l2norm(batch_samples) + + for ind, (samples, mask) in enumerate( + zip(batch_samples.unbind(dim=0), batch_mask.unbind(dim=0)) + ): + if not flow.any(mask): + continue + + sampled = self.sample_fn(rearrange(samples, "... -> 1 ..."), mask.sum().item()) + self.embed.data[ind][mask] = rearrange(sampled, "1 ... -> ...") + + def expire_codes_(self, batch_samples): + if self.threshold_ema_dead_code == 0: + return + + expired_codes = self.cluster_size < self.threshold_ema_dead_code + + if not flow.any(expired_codes): + return + + batch_samples = rearrange(batch_samples, "h ... d -> h (...) d") + self.replace(batch_samples, batch_mask=expired_codes) + + def forward(self, x): + needs_codebook_dim = x.ndim < 4 + + x = x.float() + + if needs_codebook_dim: + x = rearrange(x, "... -> 1 ...") + + shape, dtype = x.shape, x.dtype + flatten = rearrange(x, "h ... d -> h (...) d") + + self.init_embed_(flatten) + + embed = self.embed if not self.learnable_codebook else self.embed.detach() + + dist = -flow.cdist(flatten, embed, p=2) + + embed_ind = gumbel_sample(dist, dim=-1, temperature=self.sample_codebook_temp) + embed_onehot = F.one_hot(embed_ind, self.codebook_size).type(dtype) + embed_ind = embed_ind.view(*shape[:-1]) + + quantize = batched_embedding(embed_ind, self.embed) + + if self.training: + cluster_size = embed_onehot.sum(dim=1) + + self.all_reduce_fn(cluster_size) + ema_inplace(self.cluster_size, cluster_size, self.decay) + + embed_sum = einsum("h n d, h n c -> h c d", flatten, embed_onehot) + self.all_reduce_fn(embed_sum) + + cluster_size = ( + laplace_smoothing(self.cluster_size, self.codebook_size, self.eps) + * self.cluster_size.sum() + ) + + embed_normalized = self.embed_avg / rearrange(cluster_size, "... -> ... 1") + self.embed.data.copy_(embed_normalized) + self.expire_codes_(x) + + if needs_codebook_dim: + quantize, embed_ind = map(lambda t: rearrange(t, "1 ... -> ..."), (quantize, embed_ind)) + + return quantize, embed_ind + + +class CosineSimCodebook(nn.Module): + def __init__( + self, + dim, + codebook_size, + num_codebooks=1, + kmeans_init=False, + kmeans_iters=10, + decay=0.8, + eps=1e-5, + threshold_ema_dead_code=2, + use_ddp=False, + learnable_codebook=False, + sample_codebook_temp=0.0, + ): + super().__init__() + self.decay = decay + + if not kmeans_init: + embed = l2norm(uniform_init(num_codebooks, codebook_size, dim)) + else: + embed = flow.zeros(num_codebooks, codebook_size, dim) + + self.codebook_size = codebook_size + self.num_codebooks = num_codebooks + + self.kmeans_iters = kmeans_iters + self.eps = eps + self.threshold_ema_dead_code = threshold_ema_dead_code + self.sample_codebook_temp = sample_codebook_temp + + self.sample_fn = sample_vectors_distributed if use_ddp else batched_sample_vectors + self.all_reduce_fn = distributed.all_reduce if use_ddp else noop + + self.register_buffer("initted", flow.Tensor([not kmeans_init])) + self.register_buffer("cluster_size", flow.zeros(num_codebooks, codebook_size)) + + self.learnable_codebook = learnable_codebook + if learnable_codebook: + self.embed = nn.Parameter(embed) + else: + self.register_buffer("embed", embed) + + def init_embed_(self, data): + if self.initted: + return + + embed, cluster_size = kmeans( + data, + self.codebook_size, + self.kmeans_iters, + use_cosine_sim=True, + sample_fn=self.sample_fn, + all_reduce_fn=self.all_reduce_fn, + ) + + self.embed.data.copy_(embed) + self.cluster_size.data.copy_(cluster_size) + self.initted.data.copy_(flow.Tensor([True])) + + def replace(self, batch_samples, batch_mask): + batch_samples = l2norm(batch_samples) + + for ind, (samples, mask) in enumerate( + zip(batch_samples.unbind(dim=0), batch_mask.unbind(dim=0)) + ): + if not flow.any(mask): + continue + + sampled = self.sample_fn(rearrange(samples, "... -> 1 ..."), mask.sum().item()) + self.embed.data[ind][mask] = rearrange(sampled, "1 ... -> ...") + + def expire_codes_(self, batch_samples): + if self.threshold_ema_dead_code == 0: + return + + expired_codes = self.cluster_size < self.threshold_ema_dead_code + + if not flow.any(expired_codes): + return + + batch_samples = rearrange(batch_samples, "h ... d -> h (...) d") + self.replace(batch_samples, batch_mask=expired_codes) + + def forward(self, x): + needs_codebook_dim = x.ndim < 4 + + x = x.float() + + if needs_codebook_dim: + x = rearrange(x, "... -> 1 ...") + + shape, dtype = x.shape, x.dtype + + flatten = rearrange(x, "h ... d -> h (...) d") + flatten = l2norm(flatten) + + self.init_embed_(flatten) + + embed = self.embed if not self.learnable_codebook else self.embed.detach() + embed = l2norm(embed) + + dist = einsum("h n d, h c d -> h n c", flatten, embed) + embed_ind = gumbel_sample(dist, dim=-1, temperature=self.sample_codebook_temp) + embed_onehot = F.one_hot(embed_ind, self.codebook_size).type(dtype) + embed_ind = embed_ind.view(*shape[:-1]) + + quantize = batched_embedding(embed_ind, self.embed) + + if self.training: + bins = embed_onehot.sum(dim=1) + self.all_reduce_fn(bins) + + ema_inplace(self.cluster_size, bins, self.decay) + + zero_mask = bins == 0 + bins = bins.masked_fill(zero_mask, 1.0) + + embed_sum = einsum("h n d, h n c -> h c d", flatten, embed_onehot) + self.all_reduce_fn(embed_sum) + + embed_normalized = embed_sum / rearrange(bins, "... -> ... 1") + embed_normalized = l2norm(embed_normalized) + + embed_normalized = flow.where( + rearrange(zero_mask, "... -> ... 1"), embed, embed_normalized + ) + + ema_inplace(self.embed, embed_normalized, self.decay) + self.expire_codes_(x) + + if needs_codebook_dim: + quantize, embed_ind = map(lambda t: rearrange(t, "1 ... -> ..."), (quantize, embed_ind)) + + return quantize, embed_ind + + +# main class + + +class VectorQuantize(nn.Module): + def __init__( + self, + dim, + codebook_size, + codebook_dim=None, + heads=1, + separate_codebook_per_head=False, + decay=0.8, + eps=1e-5, + kmeans_init=False, + kmeans_iters=10, + use_cosine_sim=False, + threshold_ema_dead_code=0, + channel_last=True, + accept_image_fmap=False, + commitment_weight=1.0, + orthogonal_reg_weight=0.0, + orthogonal_reg_active_codes_only=False, + orthogonal_reg_max_codes=None, + sample_codebook_temp=0.0, + sync_codebook=False, + ): + super().__init__() + self.heads = heads + self.separate_codebook_per_head = separate_codebook_per_head + + codebook_dim = default(codebook_dim, dim) + codebook_input_dim = codebook_dim * heads + + requires_projection = codebook_input_dim != dim + self.project_in = ( + nn.Linear(dim, codebook_input_dim) if requires_projection else nn.Identity() + ) + self.project_out = ( + nn.Linear(codebook_input_dim, dim) if requires_projection else nn.Identity() + ) + + self.eps = eps + self.commitment_weight = commitment_weight + + has_codebook_orthogonal_loss = orthogonal_reg_weight > 0 + self.orthogonal_reg_weight = orthogonal_reg_weight + self.orthogonal_reg_active_codes_only = orthogonal_reg_active_codes_only + self.orthogonal_reg_max_codes = orthogonal_reg_max_codes + + codebook_class = EuclideanCodebook if not use_cosine_sim else CosineSimCodebook + + self._codebook = codebook_class( + dim=codebook_dim, + num_codebooks=heads if separate_codebook_per_head else 1, + codebook_size=codebook_size, + kmeans_init=kmeans_init, + kmeans_iters=kmeans_iters, + decay=decay, + eps=eps, + threshold_ema_dead_code=threshold_ema_dead_code, + use_ddp=sync_codebook, + learnable_codebook=has_codebook_orthogonal_loss, + sample_codebook_temp=sample_codebook_temp, + ) + + self.codebook_size = codebook_size + + self.accept_image_fmap = accept_image_fmap + self.channel_last = channel_last + + @property + def codebook(self): + return self._codebook.embed + + def forward(self, x): + _, device, heads, is_multiheaded, _ = ( + x.shape, + x.device, + self.heads, + self.heads > 1, + self.codebook_size, + ) + + need_transpose = not self.channel_last and not self.accept_image_fmap + + if self.accept_image_fmap: + height, width = x.shape[-2:] + x = rearrange(x, "b c h w -> b (h w) c") + + if need_transpose: + x = rearrange(x, "b d n -> b n d") + + x = self.project_in(x) + + if is_multiheaded: + ein_rhs_eq = "h b n d" if self.separate_codebook_per_head else "1 (b h) n d" + x = rearrange(x, f"b n (h d) -> {ein_rhs_eq}", h=heads) + + quantize, embed_ind = self._codebook(x) + + if self.training: + quantize = x + (quantize - x).detach() + + loss = flow.tensor([0.0], device=device, requires_grad=self.training) + + if self.training: + if self.commitment_weight > 0: + commit_loss = F.mse_loss(quantize.detach(), x) + loss = loss + commit_loss * self.commitment_weight + + if self.orthogonal_reg_weight > 0: + codebook = self.codebook + + if self.orthogonal_reg_active_codes_only: + # only calculate orthogonal loss for the activated codes for this batch + unique_code_ids = flow.unique(embed_ind) + codebook = codebook[unique_code_ids] + + num_codes = codebook.shape[0] + if ( + exists(self.orthogonal_reg_max_codes) + and num_codes > self.orthogonal_reg_max_codes + ): + rand_ids = flow.randperm(num_codes, device=device)[ + : self.orthogonal_reg_max_codes + ] + codebook = codebook[rand_ids] + + orthogonal_reg_loss = orthgonal_loss_fn(codebook) + loss = loss + orthogonal_reg_loss * self.orthogonal_reg_weight + + if is_multiheaded: + if self.separate_codebook_per_head: + quantize = rearrange(quantize, "h b n d -> b n (h d)", h=heads) + embed_ind = rearrange(embed_ind, "h b n -> b n h", h=heads) + else: + quantize = rearrange(quantize, "1 (b h) n d -> b n (h d)", h=heads) + embed_ind = rearrange(embed_ind, "1 (b h) n -> b n h", h=heads) + + quantize = self.project_out(quantize) + + if need_transpose: + quantize = rearrange(quantize, "b n d -> b d n") + + if self.accept_image_fmap: + quantize = rearrange(quantize, "b (h w) c -> b c h w", h=height, w=width) + embed_ind = rearrange(embed_ind, "b (h w) ... -> b h w ...", h=height, w=width) + + return quantize, embed_ind, loss diff --git a/projects/DALLE2/dalle2/vqgan_vae.py b/projects/DALLE2/dalle2/vqgan_vae.py new file mode 100644 index 0000000000000000000000000000000000000000..f3dfdeeabfa77456c05aaf8a87e582aa9c6a3a22 --- /dev/null +++ b/projects/DALLE2/dalle2/vqgan_vae.py @@ -0,0 +1,805 @@ +import copy +from functools import partial, wraps +from math import sqrt + +import flowvision +import oneflow as flow +import oneflow.nn.functional as F +from einops import rearrange, repeat +from oneflow import einsum, nn +from oneflow.autograd import grad as flow_grad + +from libai.layers import Linear + +from .einops_exts import Rearrange, rearrange_many +from .vector_quantize_flow import VectorQuantize as VQ + +# constants + +MList = nn.ModuleList + +# helper functions + + +def exists(val): + return val is not None + + +def default(val, d): + return val if exists(val) else d + + +# decorators + + +def eval_decorator(fn): + def inner(model, *args, **kwargs): + was_training = model.training + model.eval() + out = fn(model, *args, **kwargs) + model.train(was_training) + return out + + return inner + + +def remove_vgg(fn): + @wraps(fn) + def inner(self, *args, **kwargs): + has_vgg = hasattr(self, "vgg") + if has_vgg: + vgg = self.vgg + delattr(self, "vgg") + + out = fn(self, *args, **kwargs) + + if has_vgg: + self.vgg = vgg + + return out + + return inner + + +# keyword argument helpers + + +def pick_and_pop(keys, d): + values = list(map(lambda key: d.pop(key), keys)) + return dict(zip(keys, values)) + + +def group_dict_by_key(cond, d): + return_val = [dict(), dict()] + for key in d.keys(): + match = bool(cond(key)) + ind = int(not match) + return_val[ind][key] = d[key] + return (*return_val,) + + +def string_begins_with(prefix, string_input): + return string_input.startswith(prefix) + + +def group_by_key_prefix(prefix, d): + return group_dict_by_key(partial(string_begins_with, prefix), d) + + +def groupby_prefix_and_trim(prefix, d): + kwargs_with_prefix, kwargs = group_dict_by_key(partial(string_begins_with, prefix), d) + kwargs_without_prefix = dict( + map(lambda x: (x[0][len(prefix) :], x[1]), tuple(kwargs_with_prefix.items())) + ) + return kwargs_without_prefix, kwargs + + +# tensor helper functions + + +def log(t, eps=1e-10): + return flow.log(t + eps) + + +def gradient_penalty(images, output, weight=10): + images.shape[0] + gradients = flow_grad( + outputs=output, + inputs=images, + grad_outputs=flow.ones(output.size(), device=images.device), + create_graph=True, + retain_graph=True, + only_inputs=True, + )[0] + + gradients = rearrange(gradients, "b ... -> b (...)") + return weight * ((gradients.norm(2, dim=1) - 1) ** 2).mean() + + +def l2norm(t): + return F.normalize(t, dim=-1) + + +def leaky_relu(p=0.1): + return nn.LeakyReLU(0.1) + + +def stable_softmax(t, dim=-1, alpha=32 ** 2): + t = t / alpha + t = t - flow.amax(t, dim=dim, keepdim=True).detach() + return (t * alpha).softmax(dim=dim) + + +def safe_div(numer, denom, eps=1e-8): + return numer / (denom + eps) + + +# gan losses + + +def hinge_discr_loss(fake, real): + return (F.relu(1 + fake) + F.relu(1 - real)).mean() + + +def hinge_gen_loss(fake): + return -fake.mean() + + +def bce_discr_loss(fake, real): + return (-log(1 - flow.sigmoid(fake)) - log(flow.sigmoid(real))).mean() + + +def bce_gen_loss(fake): + return -log(flow.sigmoid(fake)).mean() + + +def grad_layer_wrt_loss(loss, layer): + return flow_grad( + outputs=loss, inputs=layer, grad_outputs=flow.ones_like(loss), retain_graph=True + )[0].detach() + + +# vqgan vae + + +class LayerNormChan(nn.Module): + def __init__(self, dim, eps=1e-5): + super().__init__() + self.eps = eps + self.gamma = nn.Parameter(flow.ones(1, dim, 1, 1)) + + def forward(self, x): + var = flow.var(x, dim=1, unbiased=False, keepdim=True) + mean = flow.mean(x, dim=1, keepdim=True) + return (x - mean) / (var + self.eps).sqrt() * self.gamma + + +# discriminator + + +class Discriminator(nn.Module): + def __init__(self, dims, channels=3, groups=16, init_kernel_size=5): + super().__init__() + dim_pairs = zip(dims[:-1], dims[1:]) + + self.layers = MList( + [ + nn.Sequential( + nn.Conv2d(channels, dims[0], init_kernel_size, padding=init_kernel_size // 2), + leaky_relu(), + ) + ] + ) + + for dim_in, dim_out in dim_pairs: + self.layers.append( + nn.Sequential( + nn.Conv2d(dim_in, dim_out, 4, stride=2, padding=1), + nn.GroupNorm(groups, dim_out), + leaky_relu(), + ) + ) + + dim = dims[-1] + self.to_logits = nn.Sequential( # return 5 x 5, for PatchGAN-esque training + nn.Conv2d(dim, dim, 1), leaky_relu(), nn.Conv2d(dim, 1, 4) + ) + + def forward(self, x): + for net in self.layers: + x = net(x) + + return self.to_logits(x) + + +# positional encoding + + +class ContinuousPositionBias(nn.Module): + """from https://arxiv.org/abs/2111.09883""" + + def __init__(self, *, dim, heads, layers=2): + super().__init__() + self.net = MList([]) + self.net.append(nn.Sequential(Linear(2, dim), leaky_relu())) + + for _ in range(layers - 1): + self.net.append(nn.Sequential(Linear(dim, dim), leaky_relu())) + + self.net.append(Linear(dim, heads)) + self.register_buffer("rel_pos", None, persistent=False) + + def forward(self, x): + n, device = x.shape[-1], x.device + fmap_size = int(sqrt(n)) + + if not exists(self.rel_pos): + pos = flow.arange(fmap_size, device=device) + grid = flow.stack(flow.meshgrid(pos, pos, indexing="ij")) + grid = rearrange(grid, "c i j -> (i j) c") + rel_pos = rearrange(grid, "i c -> i 1 c") - rearrange(grid, "j c -> 1 j c") + rel_pos = flow.sign(rel_pos) * flow.log(rel_pos.abs() + 1) + self.register_buffer("rel_pos", rel_pos, persistent=False) + + rel_pos = self.rel_pos.float() + + for layer in self.net: + rel_pos = layer(rel_pos) + + bias = rearrange(rel_pos, "i j h -> h i j") + return x + bias + + +# resnet encoder / decoder + + +class ResnetEncDec(nn.Module): + def __init__( + self, + dim, + *, + channels=3, + layers=4, + layer_mults=None, + num_resnet_blocks=1, + resnet_groups=16, + first_conv_kernel_size=5, + use_attn=True, + attn_dim_head=64, + attn_heads=8, + attn_dropout=0.0, + ): + super().__init__() + assert ( + dim % resnet_groups == 0 + ), f"dimension {dim} must be divisible by {resnet_groups} (groups for the groupnorm)" + + self.layers = layers + + self.encoders = MList([]) + self.decoders = MList([]) + + layer_mults = default(layer_mults, list(map(lambda t: 2 ** t, range(layers)))) + assert ( + len(layer_mults) == layers + ), "layer multipliers must be equal to designated number of layers" + + layer_dims = [dim * mult for mult in layer_mults] + dims = (dim, *layer_dims) + + self.encoded_dim = dims[-1] + + dim_pairs = zip(dims[:-1], dims[1:]) + + def append(arr, t): + arr.append(t) + + def prepend(arr, t): + arr.insert(0, t) + + if not isinstance(num_resnet_blocks, tuple): + num_resnet_blocks = (*((0,) * (layers - 1)), num_resnet_blocks) + + if not isinstance(use_attn, tuple): + use_attn = (*((False,) * (layers - 1)), use_attn) + + assert ( + len(num_resnet_blocks) == layers + ), "number of resnet blocks config must be equal to number of layers" + assert len(use_attn) == layers + + for layer_index, (dim_in, dim_out), layer_num_resnet_blocks, layer_use_attn in zip( + range(layers), dim_pairs, num_resnet_blocks, use_attn + ): + append( + self.encoders, + nn.Sequential(nn.Conv2d(dim_in, dim_out, 4, stride=2, padding=1), leaky_relu()), + ) + prepend( + self.decoders, + nn.Sequential(nn.ConvTranspose2d(dim_out, dim_in, 4, 2, 1), leaky_relu()), + ) + + if layer_use_attn: + prepend( + self.decoders, + VQGanAttention( + dim=dim_out, heads=attn_heads, dim_head=attn_dim_head, dropout=attn_dropout + ), + ) + + for _ in range(layer_num_resnet_blocks): + append(self.encoders, ResBlock(dim_out, groups=resnet_groups)) + prepend(self.decoders, GLUResBlock(dim_out, groups=resnet_groups)) + + if layer_use_attn: + append( + self.encoders, + VQGanAttention( + dim=dim_out, heads=attn_heads, dim_head=attn_dim_head, dropout=attn_dropout + ), + ) + + prepend( + self.encoders, + nn.Conv2d(channels, dim, first_conv_kernel_size, padding=first_conv_kernel_size // 2), + ) + append(self.decoders, nn.Conv2d(dim, channels, 1)) + + def get_encoded_fmap_size(self, image_size): + return image_size // (2 ** self.layers) + + @property + def last_dec_layer(self): + return self.decoders[-1].weight + + def encode(self, x): + for enc in self.encoders: + x = enc(x) + return x + + def decode(self, x): + for dec in self.decoders: + x = dec(x) + return x + + +class GLUResBlock(nn.Module): + def __init__(self, chan, groups=16): + super().__init__() + self.net = nn.Sequential( + nn.Conv2d(chan, chan * 2, 3, padding=1), + nn.GLU(dim=1), + nn.GroupNorm(groups, chan), + nn.Conv2d(chan, chan * 2, 3, padding=1), + nn.GLU(dim=1), + nn.GroupNorm(groups, chan), + nn.Conv2d(chan, chan, 1), + ) + + def forward(self, x): + return self.net(x) + x + + +class ResBlock(nn.Module): + def __init__(self, chan, groups=16): + super().__init__() + self.net = nn.Sequential( + nn.Conv2d(chan, chan, 3, padding=1), + nn.GroupNorm(groups, chan), + leaky_relu(), + nn.Conv2d(chan, chan, 3, padding=1), + nn.GroupNorm(groups, chan), + leaky_relu(), + nn.Conv2d(chan, chan, 1), + ) + + def forward(self, x): + return self.net(x) + x + + +# vqgan attention layer + + +class VQGanAttention(nn.Module): + def __init__(self, *, dim, dim_head=64, heads=8, dropout=0.0): + super().__init__() + self.heads = heads + self.scale = dim_head ** -0.5 + inner_dim = heads * dim_head + + self.dropout = nn.Dropout(dropout) + self.pre_norm = LayerNormChan(dim) + + self.cpb = ContinuousPositionBias(dim=dim // 4, heads=heads) + self.to_qkv = nn.Conv2d(dim, inner_dim * 3, 1, bias=False) + self.to_out = nn.Conv2d(inner_dim, dim, 1, bias=False) + + def forward(self, x): + h = self.heads + height, width, residual = *x.shape[-2:], x.clone() + + x = self.pre_norm(x) + + q, k, v = self.to_qkv(x).chunk(3, dim=1) + + q, k, v = map(lambda t: rearrange(t, "b (h c) x y -> b h c (x y)", h=h), (q, k, v)) + + sim = einsum("b h c i, b h c j -> b h i j", q, k) * self.scale + + sim = self.cpb(sim) + + attn = stable_softmax(sim, dim=-1) + attn = self.dropout(attn) + + out = einsum("b h i j, b h c j -> b h c i", attn, v) + out = rearrange(out, "b h c (x y) -> b (h c) x y", x=height, y=width) + out = self.to_out(out) + + return out + residual + + +# ViT encoder / decoder + + +class RearrangeImage(nn.Module): + def forward(self, x): + n = x.shape[1] + w = h = int(sqrt(n)) + return rearrange(x, "b (h w) ... -> b h w ...", h=h, w=w) + + +class Attention(nn.Module): + def __init__(self, dim, *, heads=8, dim_head=32): + super().__init__() + self.norm = nn.LayerNorm(dim) + self.heads = heads + self.scale = dim_head ** -0.5 + inner_dim = dim_head * heads + + self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False) + self.to_out = nn.Linear(inner_dim, dim) + + def forward(self, x): + h = self.heads + + x = self.norm(x) + + q, k, v = self.to_qkv(x).chunk(3, dim=-1) + q, k, v = rearrange_many((q, k, v), "b n (h d) -> b h n d", h=h) + + q = q * self.scale + sim = einsum("b h i d, b h j d -> b h i j", q, k) + + sim = sim - sim.amax(dim=-1, keepdim=True).detach() + attn = sim.softmax(dim=-1) + + out = einsum("b h i j, b h j d -> b h i d", attn, v) + + out = rearrange(out, "b h n d -> b n (h d)") + return self.to_out(out) + + +def FeedForward(dim, mult=4): + return nn.Sequential( + nn.LayerNorm(dim), + nn.Linear(dim, dim * mult, bias=False), + nn.GELU(), + nn.Linear(dim * mult, dim, bias=False), + ) + + +class Transformer(nn.Module): + def __init__(self, dim, *, layers, dim_head=32, heads=8, ff_mult=4): + super().__init__() + self.layers = nn.ModuleList([]) + for _ in range(layers): + self.layers.append( + nn.ModuleList( + [ + Attention(dim=dim, dim_head=dim_head, heads=heads), + FeedForward(dim=dim, mult=ff_mult), + ] + ) + ) + + self.norm = nn.LayerNorm(dim) + + def forward(self, x): + for attn, ff in self.layers: + x = attn(x) + x + x = ff(x) + x + + return self.norm(x) + + +class ViTEncDec(nn.Module): + def __init__(self, dim, channels=3, layers=4, patch_size=8, dim_head=32, heads=8, ff_mult=4): + super().__init__() + self.encoded_dim = dim + self.patch_size = patch_size + + input_dim = channels * (patch_size ** 2) + + self.encoder = nn.Sequential( + Rearrange("b c (h p1) (w p2) -> b (h w) (p1 p2 c)", p1=patch_size, p2=patch_size), + Linear(input_dim, dim), + Transformer(dim=dim, dim_head=dim_head, heads=heads, ff_mult=ff_mult, layers=layers), + RearrangeImage(), + Rearrange("b h w c -> b c h w"), + ) + + self.decoder = nn.Sequential( + Rearrange("b c h w -> b (h w) c"), + Transformer(dim=dim, dim_head=dim_head, heads=heads, ff_mult=ff_mult, layers=layers), + nn.Sequential( + Linear(dim, dim * 4, bias=False), + nn.Tanh(), + Linear(dim * 4, input_dim, bias=False), + ), + RearrangeImage(), + Rearrange("b h w (p1 p2 c) -> b c (h p1) (w p2)", p1=patch_size, p2=patch_size), + ) + + def get_encoded_fmap_size(self, image_size): + return image_size // self.patch_size + + @property + def last_dec_layer(self): + return self.decoder[-3][-1].weight + + def encode(self, x): + return self.encoder(x) + + def decode(self, x): + return self.decoder(x) + + +# main vqgan-vae classes + + +class NullVQGanVAE(nn.Module): + def __init__(self, *, channels): + super().__init__() + self.encoded_dim = channels + self.layers = 0 + + def get_encoded_fmap_size(self, size): + return size + + def copy_for_eval(self): + return self + + def encode(self, x): + return x + + def decode(self, x): + return x + + +class VQGanVAE(nn.Module): + def __init__( + self, + *, + dim, + image_size, + channels=3, + layers=4, + l2_recon_loss=False, + use_hinge_loss=True, + vgg=None, + vq_codebook_dim=256, + vq_codebook_size=512, + vq_decay=0.8, + vq_commitment_weight=1.0, + vq_kmeans_init=True, + vq_use_cosine_sim=True, + use_vgg_and_gan=True, + vae_type="resnet", + discr_layers=4, + **kwargs, + ): + super().__init__() + vq_kwargs, kwargs = groupby_prefix_and_trim("vq_", kwargs) + encdec_kwargs, kwargs = groupby_prefix_and_trim("encdec_", kwargs) + + self.image_size = image_size + self.channels = channels + self.codebook_size = vq_codebook_size + + if vae_type == "resnet": + enc_dec_klass = ResnetEncDec + elif vae_type == "vit": + enc_dec_klass = ViTEncDec + else: + raise ValueError(f"{vae_type} not valid") + + self.enc_dec = enc_dec_klass(dim=dim, channels=channels, layers=layers, **encdec_kwargs) + + self.vq = VQ( + dim=self.enc_dec.encoded_dim, + codebook_dim=vq_codebook_dim, + codebook_size=vq_codebook_size, + decay=vq_decay, + commitment_weight=vq_commitment_weight, + accept_image_fmap=True, + kmeans_init=vq_kmeans_init, + use_cosine_sim=vq_use_cosine_sim, + **vq_kwargs, + ) + + # reconstruction loss + + self.recon_loss_fn = F.mse_loss if l2_recon_loss else F.l1_loss + + # turn off GAN and perceptual loss if grayscale + + self.vgg = None + self.discr = None + self.use_vgg_and_gan = use_vgg_and_gan + + if not use_vgg_and_gan: + return + + # preceptual loss + + if exists(vgg): + self.vgg = vgg + else: + self.vgg = flowvision.models.vgg16(pretrained=True) + self.vgg.classifier = nn.Sequential(*self.vgg.classifier[:-2]) + + # gan related losses + + layer_mults = list(map(lambda t: 2 ** t, range(discr_layers))) + layer_dims = [dim * mult for mult in layer_mults] + dims = (dim, *layer_dims) + + self.discr = Discriminator(dims=dims, channels=channels) + + self.discr_loss = hinge_discr_loss if use_hinge_loss else bce_discr_loss + self.gen_loss = hinge_gen_loss if use_hinge_loss else bce_gen_loss + + @property + def encoded_dim(self): + return self.enc_dec.encoded_dim + + def get_encoded_fmap_size(self, image_size): + return self.enc_dec.get_encoded_fmap_size(image_size) + + def copy_for_eval(self): + device = next(self.parameters()).device + vae_copy = copy.deepcopy(self.cpu()) + + if vae_copy.use_vgg_and_gan: + del vae_copy.discr + del vae_copy.vgg + + vae_copy.eval() + return vae_copy.to(device) + + @remove_vgg + def state_dict(self, *args, **kwargs): + return super().state_dict(*args, **kwargs) + + @remove_vgg + def load_state_dict(self, *args, **kwargs): + return super().load_state_dict(*args, **kwargs) + + @property + def codebook(self): + return self.vq.codebook + + def encode(self, fmap): + fmap = self.enc_dec.encode(fmap) + return fmap + + def decode(self, fmap, return_indices_and_loss=False): + fmap, indices, commit_loss = self.vq(fmap) + + fmap = self.enc_dec.decode(fmap) + + if not return_indices_and_loss: + return fmap + + return fmap, indices, commit_loss + + def forward( + self, + img, + return_loss=False, + return_discr_loss=False, + return_recons=False, + add_gradient_penalty=True, + ): + _, channels, height, width, _ = *img.shape, img.device + assert ( + height == self.image_size and width == self.image_size + ), "height and width of input image must be equal to {self.image_size}" + assert ( + channels == self.channels + ), "number of channels on image or sketch is not equal to the channels set on this VQGanVAE" + + fmap = self.encode(img) + + fmap, indices, commit_loss = self.decode(fmap, return_indices_and_loss=True) + + if not return_loss and not return_discr_loss: + return fmap + + assert ( + return_loss ^ return_discr_loss + ), "you should either return autoencoder loss or discriminator loss, but not both" + + # whether to return discriminator loss + + if return_discr_loss: + assert exists(self.discr), "discriminator must exist to train it" + + fmap.detach_() + img.requires_grad_() + + fmap_discr_logits, img_discr_logits = map(self.discr, (fmap, img)) + + discr_loss = self.discr_loss(fmap_discr_logits, img_discr_logits) + + if add_gradient_penalty: + gp = gradient_penalty(img, img_discr_logits) + loss = discr_loss + gp + + if return_recons: + return loss, fmap + + return loss + + # reconstruction loss + + recon_loss = self.recon_loss_fn(fmap, img) + + # early return if training on grayscale + + if not self.use_vgg_and_gan: + if return_recons: + return recon_loss, fmap + + return recon_loss + + # perceptual loss + + img_vgg_input = img + fmap_vgg_input = fmap + + if img.shape[1] == 1: + # handle grayscale for vgg + img_vgg_input, fmap_vgg_input = map( + lambda t: repeat(t, "b 1 ... -> b c ...", c=3), (img_vgg_input, fmap_vgg_input) + ) + + img_vgg_feats = self.vgg(img_vgg_input) + recon_vgg_feats = self.vgg(fmap_vgg_input) + perceptual_loss = F.mse_loss(img_vgg_feats, recon_vgg_feats) + + # generator loss + + gen_loss = self.gen_loss(self.discr(fmap)) + + # calculate adaptive weight + + last_dec_layer = self.enc_dec.last_dec_layer + + norm_grad_wrt_gen_loss = grad_layer_wrt_loss(gen_loss, last_dec_layer).norm(p=2) + norm_grad_wrt_perceptual_loss = grad_layer_wrt_loss(perceptual_loss, last_dec_layer).norm( + p=2 + ) + + adaptive_weight = safe_div(norm_grad_wrt_perceptual_loss, norm_grad_wrt_gen_loss) + adaptive_weight.clamp_(max=1e4) + + # combine losses + + loss = recon_loss + perceptual_loss + commit_loss + adaptive_weight * gen_loss + + if return_recons: + return loss, fmap + + return loss diff --git a/projects/DALLE2/dalle2_inference.py b/projects/DALLE2/dalle2_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..fd73ad78f997fb9c28e9c28a4357a7a9a1b2a6d6 --- /dev/null +++ b/projects/DALLE2/dalle2_inference.py @@ -0,0 +1,184 @@ +import os +from typing import Dict + +import oneflow as flow +from dalle2.dalle2_loader import Dalle2ModelLoader +from dalle2.model_weights.download_utils import download_dalle2_weights +from dalle2.tokenizer import SimpleTokenizer +from oneflow.framework import balanced_splitter + +import libai.utils.distributed as dist +from libai.inference.basic import BasePipeline + + +class Dalle2Pipeline(BasePipeline): + def __init__( + self, + config_file, + data_parallel=None, + tensor_parallel=None, + pipeline_parallel=None, + pipeline_stage_id=None, + pipeline_num_layers=None, + model_path=None, + mode="libai", + **kwargs, + ): + super().__init__( + config_file, + data_parallel, + tensor_parallel, + pipeline_parallel, + pipeline_stage_id, + model_path, + pipeline_num_layers, + mode, + **kwargs, + ) + + def update_cfg( + self, + data_parallel=1, + tensor_parallel=1, + pipeline_parallel=1, + pipeline_stage_id=None, + pipeline_num_layers=None, + ): + super().update_cfg( + data_parallel, + tensor_parallel, + pipeline_parallel, + pipeline_stage_id, + pipeline_num_layers, + ) + self.cfg.model.prior.clip.name = "./dalle2/model_weights/ViT-L-14.pt" + self.cfg.model.prior_weight_path = "./dalle2/model_weights/prior_aes_finetune.pth" + self.cfg.model.decoder_weight_path = "./dalle2/model_weights/latest.pth" + self.cfg.swinir.swinir_path = ( + "./swinir/weights/003_realSR_BSRGAN_DFOWMFC_s64w8_SwinIR-L_x4_GAN.pth" + ) + + def load_pretrain_weight(self, libai_cfg_model, model_path, mode=None): + if dist.is_main_process(): + download_dalle2_weights(self.cfg) + dist.synchronize() + model_loader = Dalle2ModelLoader(libai_cfg_model, self.cfg, model_path) + return model_loader.load() + + def build_tokenizer(self, cfg): + return SimpleTokenizer() # return instantiate(cfg.tokenizer) + + def _parse_parameters(self, model_path=None, save_images=False, upsample_scale=None, **kwargs): + preprocess_params = {} + forward_params = { + "model_path": model_path, + "num_samples_per_batch": kwargs.get("num_samples_per_batch", 2), + "prior_cond_scale": kwargs.get("prior_cond_scale", 1.0), + "decoder_cond_scale": kwargs.get("decoder_cond_scale", 3.5), + } + postprocess_params = { + "save_images": save_images, + "upsample_scale": upsample_scale, + "swinir_path": "./swinir/weights/003_realSR_BSRGAN_DFOWMFC_s64w8_SwinIR-L_x4_GAN.pth", + } + + return preprocess_params, forward_params, postprocess_params + + def split_data(self, text): + rank = dist.get_rank() + indices = balanced_splitter.BalancedRanges(len(text), dist.get_world_size()) + return text[indices[rank][0] : indices[rank][1]] + + def preprocess(self, input_, **preprocess_parameters: Dict) -> dict: + tokens = self.tokenizer.tokenize(input_).to_global( + placement=flow.placement(type="cuda", ranks=list(range(dist.get_world_size()))), + sbp=flow.sbp.broadcast, + ) + return {"text": input_, "tokens": tokens} + + def forward(self, model_input_dict, **forward_params) -> dict: + tokens = model_input_dict["tokens"] + text_embed, text_encodings, text_mask = self.model.prior.clip.embed_text(tokens) + image_embed = self.model.prior.sample( + tokens, + num_samples_per_batch=forward_params["num_samples_per_batch"], + cond_scale=forward_params["prior_cond_scale"], + ) + + image_embed = self.model.decoder.sample( + image_embed=image_embed, + text_encodings=text_encodings, + text_mask=text_mask, + cond_scale=forward_params["decoder_cond_scale"], + ) + + return {"image_embed": image_embed} + + def postprocess(self, model_output_dict, **postprocess_params: Dict) -> dict: + if not postprocess_params.get("save_images", False): + return model_output_dict + output_path = postprocess_params.get("output_dit", "./outputs") + os.makedirs(output_path, exist_ok=True) + + import flowvision.transforms as T + + to_pil = T.ToPILImage() + images = model_output_dict["image_embed"].to("cpu") + images_64x64 = list(map(to_pil, [images[i] for i in range(images.shape[0])])) + for i, image in enumerate(images_64x64): + image.save(f"{output_path}/{i}.png") + if postprocess_params.get("upsample_scale", False): + from swinir import load_model, upsample4x, upsample16x + + swinir = load_model(postprocess_params.get("swinir_path", "")) + upsample_fun = upsample4x if args.upsample_scale == 4 else upsample16x + images = upsample_fun(images, swinir).to("cpu") + images = list(map(to_pil, [images[i] for i in range(images.shape[0])])) + for i, image in enumerate(images): + image.save(f"{output_path}/{i}_{args.upsample_scale}x.png") + print(f"Images have been saved under {output_path}.") + return model_output_dict + + +def parse_args(): + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--config_file", type=str, default="configs/dalle2_config.py") + parser.add_argument("--data_parallel", type=int, default=1) + parser.add_argument("--tensor_parallel", type=int, default=4) + parser.add_argument("--pipeline_parallel", type=int, default=1) + parser.add_argument( + "--upsample_scale", + type=int, + choices=[4, 16], + default=None, + help="upsample scale, if 4x, output resolution will be 256 x 256.", + ) + parser.add_argument( + "--swinir_path", + type=str, + default="./swinir/weights/003_realSR_BSRGAN_DFOWMFC_s64w8_SwinIR-L_x4_GAN.pth", + ) + parser.add_argument("--output_dir", type=str, default="./outputs") + parser.add_argument("--save_images", action="store_true") + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + model = Dalle2Pipeline( + config_file=args.config_file, + data_parallel=args.data_parallel, + tensor_parallel=args.tensor_parallel, + pipeline_parallel=args.pipeline_parallel, + ) + + texts = [ + "a shiba inu wearing a beret and black turtleneck", + "a teddy bear on a skateboard in times square", + "trump fight with biden in white house", + "Donald trump fight with biden in white house", + ] + + imgs = model(texts, **vars(args)) diff --git a/projects/DALLE2/readme.md b/projects/DALLE2/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..6ce100663238fc8bd1a1c7c2d7e8dd28b5ce00a4 --- /dev/null +++ b/projects/DALLE2/readme.md @@ -0,0 +1,19 @@ +# DALLE2 +This project is adapted from [dalle2_pytorch](https://github.com/lucidrains/DALLE2-pytorch); And dalle2_pytorch version=0.15.4 is used following this [colab](https://colab.research.google.com/github/LAION-AI/dalle2-laion/blob/main/notebooks/dalle2_laion_alpha.ipynb). +This project aims at guiding how to transfer pytorch models to oneflow and use distributed inference for new users with [LiBai](https://github.com/Oneflow-Inc/libai), details could be found [here](../../docs/source/notes/How_to_use_model_parallel_in_LiBai.md). + +## How to run this project +```sh +cd libai/projects/DALLE2 +pip install -r requirements.txt +python3 -m oneflow.distributed.launch \ + --nproc_per_node 4 \ + dalle2_inference.py \ + --save_images \ + --output_dir ./outputs \ + --upsample_scale 4 +``` +`--nprec_per_node 4` means this model will be executed on 4 gpus under the model parallel mode. +The output images will be saved to `--output_dir` by setting `--save_images`. The resolution of the generated images are 64x64 by default, and could be resize to 256x256 with `--upsample_scale 4` (and 1024x1024 with `--upsample_scale 16`) by using [SwinIR](https://github.com/JingyunLiang/SwinIR). + +At the bottom of the dalle2_inference.py, try feeding different text and see what the model will generated. \ No newline at end of file diff --git a/projects/DALLE2/requirements.txt b/projects/DALLE2/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4fdc6743ca3b1ce23e3e0a88c4d4c602815b1f88 --- /dev/null +++ b/projects/DALLE2/requirements.txt @@ -0,0 +1,4 @@ +ftfy +resize_right +einops +kornia diff --git a/projects/DALLE2/swinir/__init__.py b/projects/DALLE2/swinir/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3e9bebd0f5bde009a6cd42af115d0e4fe4675e90 --- /dev/null +++ b/projects/DALLE2/swinir/__init__.py @@ -0,0 +1,2 @@ +from .models import SwinIR +from .upsample import load_model, upsample4x, upsample16x diff --git a/projects/DALLE2/swinir/models.py b/projects/DALLE2/swinir/models.py new file mode 100644 index 0000000000000000000000000000000000000000..27c190d235a11d41103f7380a640a6db4800ff05 --- /dev/null +++ b/projects/DALLE2/swinir/models.py @@ -0,0 +1,1035 @@ +# ----------------------------------------------------------------------------------- +# SwinIR: Image Restoration Using Swin Transformer, https://arxiv.org/abs/2108.10257 +# Originally Written by Ze Liu, Modified by Jingyun Liang. +# ----------------------------------------------------------------------------------- +# code from https://github.com/JingyunLiang/SwinIR/blob/main/models/network_swinir.py + +import math + +import oneflow as flow +import oneflow.nn as nn +import oneflow.nn.functional as F +from oneflow.utils import checkpoint + +from .utils import DropPath, to_2tuple, trunc_normal_ + + +class Mlp(nn.Module): + def __init__( + self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.0 + ): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class WindowAttention(nn.Module): + r"""Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): + If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__( + self, + dim, + window_size, + num_heads, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + ): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim ** -0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + flow.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads) + ) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = flow.arange(self.window_size[0]) + coords_w = flow.arange(self.window_size[1]) + coords = flow.stack(flow.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = flow.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer("relative_position_index", relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=0.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """ + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = ( + self.qkv(x) + .reshape(B_, N, 3, self.num_heads, C // self.num_heads) + .permute(2, 0, 3, 1, 4) + ) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = q @ k.transpose(-2, -1) + + relative_position_bias = self.relative_position_bias_table[ + self.relative_position_index.view(-1) + ].view( + self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1 + ) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1 + ).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + def extra_repr(self) -> str: + return f"dim={self.dim}, window_size={self.window_size}, num_heads={self.num_heads}" + + def flops(self, N): + # calculate flops for 1 window with token length of N + flops = 0 + # qkv = self.qkv(x) + flops += N * self.dim * 3 * self.dim + # attn = (q @ k.transpose(-2, -1)) + flops += self.num_heads * N * (self.dim // self.num_heads) * N + # x = (attn @ v) + flops += self.num_heads * N * N * (self.dim // self.num_heads) + # x = self.proj(x) + flops += N * self.dim * self.dim + return flops + + +class SwinTransformerBlock(nn.Module): + r"""Swin Transformer Block. + + Args: + dim (int): Number of input channels. + input_resolution (tuple[int]): Input resulotion. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__( + self, + dim, + input_resolution, + num_heads, + window_size=7, + shift_size=0, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + ): + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + if min(self.input_resolution) <= self.window_size: + # if window size is larger than input resolution, we don't partition windows + self.shift_size = 0 + self.window_size = min(self.input_resolution) + assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size" + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, + window_size=to_2tuple(self.window_size), + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + ) + + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop + ) + + if self.shift_size > 0: + attn_mask = self.calculate_mask(self.input_resolution) + else: + attn_mask = None + + self.register_buffer("attn_mask", attn_mask) + + def calculate_mask(self, x_size): + # calculate attention mask for SW-MSA + H, W = x_size + img_mask = flow.zeros((1, H, W, 1)) # 1 H W 1 + h_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + w_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition( + img_mask, self.window_size + ) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill( + attn_mask == 0, float(0.0) + ) + + return attn_mask + + def forward(self, x, x_size): + H, W = x_size + B, L, C = x.shape + # assert L == H * W, "input feature has wrong size" + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # cyclic shift + if self.shift_size > 0: + shifted_x = flow.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + else: + shifted_x = x + + # partition windows + x_windows = window_partition( + shifted_x, self.window_size + ) # nW*B, window_size, window_size, C + x_windows = x_windows.view( + -1, self.window_size * self.window_size, C + ) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA (to be compatible for testing on images + # whose shapes are the multiple of window size + if self.input_resolution == x_size: + attn_windows = self.attn( + x_windows, mask=self.attn_mask + ) # nW*B, window_size*window_size, C + else: + attn_windows = self.attn(x_windows, mask=self.calculate_mask(x_size).to(x.device)) + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, H, W) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = flow.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2)) + else: + x = shifted_x + x = x.view(B, H * W, C) + + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x + + def extra_repr(self) -> str: + return ( + f"dim={self.dim}, input_resolution={self.input_resolution}, " + f"num_heads={self.num_heads}, window_size={self.window_size}," + f"shift_size={self.shift_size}, mlp_ratio={self.mlp_ratio}" + ) + + def flops(self): + flops = 0 + H, W = self.input_resolution + # norm1 + flops += self.dim * H * W + # W-MSA/SW-MSA + nW = H * W / self.window_size / self.window_size + flops += nW * self.attn.flops(self.window_size * self.window_size) + # mlp + flops += 2 * H * W * self.dim * self.dim * self.mlp_ratio + # norm2 + flops += self.dim * H * W + return flops + + +class PatchMerging(nn.Module): + r"""Patch Merging Layer. + + Args: + input_resolution (tuple[int]): Resolution of input feature. + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, input_resolution, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.input_resolution = input_resolution + self.dim = dim + self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) + self.norm = norm_layer(4 * dim) + + def forward(self, x): + """ + x: B, H*W, C + """ + H, W = self.input_resolution + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + assert H % 2 == 0 and W % 2 == 0, f"x size ({H}*{W}) are not even." + + x = x.view(B, H, W, C) + + x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C + x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C + x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C + x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C + x = flow.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C + x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C + + x = self.norm(x) + x = self.reduction(x) + + return x + + def extra_repr(self) -> str: + return f"input_resolution={self.input_resolution}, dim={self.dim}" + + def flops(self): + H, W = self.input_resolution + flops = H * W * self.dim + flops += (H // 2) * (W // 2) * 4 * self.dim * 2 * self.dim + return flops + + +class BasicLayer(nn.Module): + """A basic Swin Transformer layer for one stage. + + Args: + dim (int): Number of input channels. + input_resolution (tuple[int]): Input resolution. + depth (int): Number of blocks. + num_heads (int): Number of attention heads. + window_size (int): Local window size. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): + Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__( + self, + dim, + input_resolution, + depth, + num_heads, + window_size, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False, + ): + + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + self.blocks = nn.ModuleList( + [ + SwinTransformerBlock( + dim=dim, + input_resolution=input_resolution, + num_heads=num_heads, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, + ) + for i in range(depth) + ] + ) + + # patch merging layer + if downsample is not None: + self.downsample = downsample(input_resolution, dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x, x_size): + for blk in self.blocks: + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x, x_size) + else: + x = blk(x, x_size) + if self.downsample is not None: + x = self.downsample(x) + return x + + def extra_repr(self) -> str: + return f"dim={self.dim}, input_resolution={self.input_resolution}, depth={self.depth}" + + def flops(self): + flops = 0 + for blk in self.blocks: + flops += blk.flops() + if self.downsample is not None: + flops += self.downsample.flops() + return flops + + +class RSTB(nn.Module): + """Residual Swin Transformer Block (RSTB). + + Args: + dim (int): Number of input channels. + input_resolution (tuple[int]): Input resolution. + depth (int): Number of blocks. + num_heads (int): Number of attention heads. + window_size (int): Local window size. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): + If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): + Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): + Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): + Whether to use checkpointing to save memory. Default: False. + img_size: Input image size. + patch_size: Patch size. + resi_connection: The convolutional block before residual connection. + """ + + def __init__( + self, + dim, + input_resolution, + depth, + num_heads, + window_size, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False, + img_size=224, + patch_size=4, + resi_connection="1conv", + ): + super(RSTB, self).__init__() + + self.dim = dim + self.input_resolution = input_resolution + + self.residual_group = BasicLayer( + dim=dim, + input_resolution=input_resolution, + depth=depth, + num_heads=num_heads, + window_size=window_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path, + norm_layer=norm_layer, + downsample=downsample, + use_checkpoint=use_checkpoint, + ) + + if resi_connection == "1conv": + self.conv = nn.Conv2d(dim, dim, 3, 1, 1) + elif resi_connection == "3conv": + # to save parameters and memory + self.conv = nn.Sequential( + nn.Conv2d(dim, dim // 4, 3, 1, 1), + nn.LeakyReLU(negative_slope=0.2, inplace=True), + nn.Conv2d(dim // 4, dim // 4, 1, 1, 0), + nn.LeakyReLU(negative_slope=0.2, inplace=True), + nn.Conv2d(dim // 4, dim, 3, 1, 1), + ) + + self.patch_embed = PatchEmbed( + img_size=img_size, patch_size=patch_size, in_chans=0, embed_dim=dim, norm_layer=None + ) + + self.patch_unembed = PatchUnEmbed( + img_size=img_size, patch_size=patch_size, in_chans=0, embed_dim=dim, norm_layer=None + ) + + def forward(self, x, x_size): + return ( + self.patch_embed(self.conv(self.patch_unembed(self.residual_group(x, x_size), x_size))) + + x + ) + + def flops(self): + flops = 0 + flops += self.residual_group.flops() + H, W = self.input_resolution + flops += H * W * self.dim * self.dim * 9 + flops += self.patch_embed.flops() + flops += self.patch_unembed.flops() + + return flops + + +class PatchEmbed(nn.Module): + r"""Image to Patch Embedding + + Args: + img_size (int): Image size. Default: 224. + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, img_size=224, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + patches_resolution = [img_size[0] // patch_size[0], img_size[1] // patch_size[1]] + self.img_size = img_size + self.patch_size = patch_size + self.patches_resolution = patches_resolution + self.num_patches = patches_resolution[0] * patches_resolution[1] + + self.in_chans = in_chans + self.embed_dim = embed_dim + + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + x = x.flatten(2).transpose(1, 2) # B Ph*Pw C + if self.norm is not None: + x = self.norm(x) + return x + + def flops(self): + flops = 0 + H, W = self.img_size + if self.norm is not None: + flops += H * W * self.embed_dim + return flops + + +class PatchUnEmbed(nn.Module): + r"""Image to Patch Unembedding + + Args: + img_size (int): Image size. Default: 224. + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, img_size=224, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + patches_resolution = [img_size[0] // patch_size[0], img_size[1] // patch_size[1]] + self.img_size = img_size + self.patch_size = patch_size + self.patches_resolution = patches_resolution + self.num_patches = patches_resolution[0] * patches_resolution[1] + + self.in_chans = in_chans + self.embed_dim = embed_dim + + def forward(self, x, x_size): + B, HW, C = x.shape + x = x.transpose(1, 2).view(B, self.embed_dim, x_size[0], x_size[1]) # B Ph*Pw C + return x + + def flops(self): + flops = 0 + return flops + + +class Upsample(nn.Sequential): + """Upsample module. + + Args: + scale (int): Scale factor. Supported scales: 2^n and 3. + num_feat (int): Channel number of intermediate features. + """ + + def __init__(self, scale, num_feat): + m = [] + if (scale & (scale - 1)) == 0: # scale = 2^n + for _ in range(int(math.log(scale, 2))): + m.append(nn.Conv2d(num_feat, 4 * num_feat, 3, 1, 1)) + m.append(nn.PixelShuffle(2)) + elif scale == 3: + m.append(nn.Conv2d(num_feat, 9 * num_feat, 3, 1, 1)) + m.append(nn.PixelShuffle(3)) + else: + raise ValueError(f"scale {scale} is not supported. " "Supported scales: 2^n and 3.") + super(Upsample, self).__init__(*m) + + +class UpsampleOneStep(nn.Sequential): + """UpsampleOneStep module + (the difference with Upsample is that it always only has 1conv + 1pixelshuffle) + Used in lightweight SR to save parameters. + + Args: + scale (int): Scale factor. Supported scales: 2^n and 3. + num_feat (int): Channel number of intermediate features. + + """ + + def __init__(self, scale, num_feat, num_out_ch, input_resolution=None): + self.num_feat = num_feat + self.input_resolution = input_resolution + m = [] + m.append(nn.Conv2d(num_feat, (scale ** 2) * num_out_ch, 3, 1, 1)) + m.append(nn.PixelShuffle(scale)) + super(UpsampleOneStep, self).__init__(*m) + + def flops(self): + H, W = self.input_resolution + flops = H * W * self.num_feat * 3 * 9 + return flops + + +class SwinIR(nn.Module): + r"""SwinIR + A PyTorch impl of : `SwinIR: Image Restoration Using Swin Transformer`, + based on Swin Transformer. + + Args: + img_size (int | tuple(int)): Input image size. Default 64 + patch_size (int | tuple(int)): Patch size. Default: 1 + in_chans (int): Number of input image channels. Default: 3 + embed_dim (int): Patch embedding dimension. Default: 96 + depths (tuple(int)): Depth of each Swin Transformer layer. + num_heads (tuple(int)): Number of attention heads in different layers. + window_size (int): Window size. Default: 7 + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4 + qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float): + Override default qk scale of head_dim ** -0.5 if set. Default: None + drop_rate (float): Dropout rate. Default: 0 + attn_drop_rate (float): Attention dropout rate. Default: 0 + drop_path_rate (float): Stochastic depth rate. Default: 0.1 + norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. + ape (bool): + If True, add absolute position embedding to the patch embedding. Default: False + patch_norm (bool): If True, add normalization after patch embedding. Default: True + use_checkpoint (bool): + Whether to use checkpointing to save memory. Default: False + upscale: Upscale factor. + 2/3/4/8 for image SR, 1 for denoising and compress artifact reduction + img_range: Image range. 1. or 255. + upsampler: The reconstruction reconstruction module. + 'pixelshuffle'/'pixelshuffledirect'/'nearest+conv'/None + resi_connection: The convolutional block before residual connection. '1conv'/'3conv' + """ + + def __init__( + self, + img_size=64, + patch_size=1, + in_chans=3, + embed_dim=96, + depths=[6, 6, 6, 6], + num_heads=[6, 6, 6, 6], + window_size=7, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.1, + norm_layer=nn.LayerNorm, + ape=False, + patch_norm=True, + use_checkpoint=False, + upscale=2, + img_range=1.0, + upsampler="", + resi_connection="1conv", + **kwargs, + ): + super(SwinIR, self).__init__() + num_in_ch = in_chans + num_out_ch = in_chans + num_feat = 64 + self.img_range = img_range + if in_chans == 3: + rgb_mean = (0.4488, 0.4371, 0.4040) + self.mean = flow.Tensor(rgb_mean).view(1, 3, 1, 1) + else: + self.mean = flow.zeros(1, 1, 1, 1) + self.upscale = upscale + self.upsampler = upsampler + self.window_size = window_size + + # 1, shallow feature extraction + self.conv_first = nn.Conv2d(num_in_ch, embed_dim, 3, 1, 1) + + # 2, deep feature extraction + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.ape = ape + self.patch_norm = patch_norm + self.num_features = embed_dim + self.mlp_ratio = mlp_ratio + + # split image into non-overlapping patches + self.patch_embed = PatchEmbed( + img_size=img_size, + patch_size=patch_size, + in_chans=embed_dim, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None, + ) + num_patches = self.patch_embed.num_patches + patches_resolution = self.patch_embed.patches_resolution + self.patches_resolution = patches_resolution + + # merge non-overlapping patches into image + self.patch_unembed = PatchUnEmbed( + img_size=img_size, + patch_size=patch_size, + in_chans=embed_dim, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None, + ) + + # absolute position embedding + if self.ape: + self.absolute_pos_embed = nn.Parameter(flow.zeros(1, num_patches, embed_dim)) + trunc_normal_(self.absolute_pos_embed, std=0.02) + + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth + dpr = [ + x.item() for x in flow.linspace(0, drop_path_rate, sum(depths)) + ] # stochastic depth decay rule + + # build Residual Swin Transformer blocks (RSTB) + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = RSTB( + dim=embed_dim, + input_resolution=(patches_resolution[0], patches_resolution[1]), + depth=depths[i_layer], + num_heads=num_heads[i_layer], + window_size=window_size, + mlp_ratio=self.mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[ + sum(depths[:i_layer]) : sum(depths[: i_layer + 1]) + ], # no impact on SR results + norm_layer=norm_layer, + downsample=None, + use_checkpoint=use_checkpoint, + img_size=img_size, + patch_size=patch_size, + resi_connection=resi_connection, + ) + self.layers.append(layer) + self.norm = norm_layer(self.num_features) + + # build the last conv layer in deep feature extraction + if resi_connection == "1conv": + self.conv_after_body = nn.Conv2d(embed_dim, embed_dim, 3, 1, 1) + elif resi_connection == "3conv": + # to save parameters and memory + self.conv_after_body = nn.Sequential( + nn.Conv2d(embed_dim, embed_dim // 4, 3, 1, 1), + nn.LeakyReLU(negative_slope=0.2, inplace=True), + nn.Conv2d(embed_dim // 4, embed_dim // 4, 1, 1, 0), + nn.LeakyReLU(negative_slope=0.2, inplace=True), + nn.Conv2d(embed_dim // 4, embed_dim, 3, 1, 1), + ) + + # 3, high quality image reconstruction + if self.upsampler == "pixelshuffle": + # for classical SR + self.conv_before_upsample = nn.Sequential( + nn.Conv2d(embed_dim, num_feat, 3, 1, 1), nn.LeakyReLU(inplace=True) + ) + self.upsample = Upsample(upscale, num_feat) + self.conv_last = nn.Conv2d(num_feat, num_out_ch, 3, 1, 1) + elif self.upsampler == "pixelshuffledirect": + # for lightweight SR (to save parameters) + self.upsample = UpsampleOneStep( + upscale, embed_dim, num_out_ch, (patches_resolution[0], patches_resolution[1]) + ) + elif self.upsampler == "nearest+conv": + # for real-world SR (less artifacts) + self.conv_before_upsample = nn.Sequential( + nn.Conv2d(embed_dim, num_feat, 3, 1, 1), nn.LeakyReLU(inplace=True) + ) + self.conv_up1 = nn.Conv2d(num_feat, num_feat, 3, 1, 1) + if self.upscale == 4: + self.conv_up2 = nn.Conv2d(num_feat, num_feat, 3, 1, 1) + self.conv_hr = nn.Conv2d(num_feat, num_feat, 3, 1, 1) + self.conv_last = nn.Conv2d(num_feat, num_out_ch, 3, 1, 1) + self.lrelu = nn.LeakyReLU(negative_slope=0.2, inplace=True) + else: + # for image denoising and JPEG compression artifact reduction + self.conv_last = nn.Conv2d(embed_dim, num_out_ch, 3, 1, 1) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def no_weight_decay(self): + return {"absolute_pos_embed"} + + def no_weight_decay_keywords(self): + return {"relative_position_bias_table"} + + def check_image_size(self, x): + _, _, h, w = x.size() + mod_pad_h = (self.window_size - h % self.window_size) % self.window_size + mod_pad_w = (self.window_size - w % self.window_size) % self.window_size + x = F.pad(x, (0, mod_pad_w, 0, mod_pad_h), "reflect") + return x + + def forward_features(self, x): + x_size = (x.shape[2], x.shape[3]) + x = self.patch_embed(x) + if self.ape: + x = x + self.absolute_pos_embed + x = self.pos_drop(x) + + for layer in self.layers: + x = layer(x, x_size) + + x = self.norm(x) # B L C + x = self.patch_unembed(x, x_size) + + return x + + def forward(self, x): + H, W = x.shape[2:] + x = self.check_image_size(x) + + self.mean = self.mean.type_as(x).to(x.device) + x = (x - self.mean) * self.img_range + + if self.upsampler == "pixelshuffle": + # for classical SR + x = self.conv_first(x) + x = self.conv_after_body(self.forward_features(x)) + x + x = self.conv_before_upsample(x) + x = self.conv_last(self.upsample(x)) + elif self.upsampler == "pixelshuffledirect": + # for lightweight SR + x = self.conv_first(x) + x = self.conv_after_body(self.forward_features(x)) + x + x = self.upsample(x) + elif self.upsampler == "nearest+conv": + # for real-world SR + x = self.conv_first(x) + x = self.conv_after_body(self.forward_features(x)) + x + x = self.conv_before_upsample(x) + x = self.lrelu( + self.conv_up1(flow.nn.functional.interpolate(x, scale_factor=2, mode="nearest")) + ) + if self.upscale == 4: + x = self.lrelu( + self.conv_up2(flow.nn.functional.interpolate(x, scale_factor=2, mode="nearest")) + ) + x = self.conv_last(self.lrelu(self.conv_hr(x))) + else: + # for image denoising and JPEG compression artifact reduction + x_first = self.conv_first(x) + res = self.conv_after_body(self.forward_features(x_first)) + x_first + x = x + self.conv_last(res) + + x = x / self.img_range + self.mean + + return x[:, :, : H * self.upscale, : W * self.upscale] + + def flops(self): + flops = 0 + H, W = self.patches_resolution + flops += H * W * 3 * self.embed_dim * 9 + flops += self.patch_embed.flops() + for i, layer in enumerate(self.layers): + flops += layer.flops() + flops += H * W * 3 * self.embed_dim * self.embed_dim + flops += self.upsample.flops() + return flops + + +if __name__ == "__main__": + upscale = 4 + window_size = 8 + height = (1024 // upscale // window_size + 1) * window_size + width = (720 // upscale // window_size + 1) * window_size + model = SwinIR( + upscale=2, + img_size=(height, width), + window_size=window_size, + img_range=1.0, + depths=[6, 6, 6, 6], + embed_dim=60, + num_heads=[6, 6, 6, 6], + mlp_ratio=2, + upsampler="pixelshuffledirect", + ) + print(model) + print(height, width, model.flops() / 1e9) + + x = flow.randn((1, 3, height, width)) + x = model(x) + print(x.shape) diff --git a/projects/DALLE2/swinir/upsample.py b/projects/DALLE2/swinir/upsample.py new file mode 100644 index 0000000000000000000000000000000000000000..19ec6f5d822c87693f7a91cdea272dcb370d6a89 --- /dev/null +++ b/projects/DALLE2/swinir/upsample.py @@ -0,0 +1,83 @@ +import os + +import oneflow as flow +import requests + +from .models import SwinIR as net + + +def load_torch_weight(model, model_path): + # load torch weight + import torch + + param_key_g = "params_ema" + pretrained_model = torch.load(model_path, map_location="cpu") + pretrained_model = ( + pretrained_model[param_key_g] + if param_key_g in pretrained_model.keys() + else pretrained_model + ) + new_state_dict = {} + for k, v in pretrained_model.items(): + flow_tensor = flow.tensor(v.numpy()) + new_state_dict[k] = flow_tensor + model.load_state_dict(new_state_dict, strict=True) + return model + + +def load_model(model_path=None): + # set up model + if os.path.exists(model_path): + print(f"loading model from {model_path}") + else: + os.makedirs(os.path.dirname(model_path), exist_ok=True) + url = "https://github.com/JingyunLiang/SwinIR/releases/download/v0.0/{}".format( + os.path.basename(model_path) + ) + r = requests.get(url, allow_redirects=True) + print(f"downloading model {model_path}") + open(model_path, "wb").write(r.content) + model = net( + upscale=4, + in_chans=3, + img_size=64, + window_size=8, + img_range=1.0, + depths=[6, 6, 6, 6, 6, 6, 6, 6, 6], + embed_dim=240, + num_heads=[8, 8, 8, 8, 8, 8, 8, 8, 8], + mlp_ratio=2, + upsampler="nearest+conv", + resi_connection="3conv", + ) + model = load_torch_weight(model, model_path) + return model + + +def upsample4x(img_lq, model): + """upsample img from h*w to (4h) * (4w)""" + device = flow.device("cuda" if flow.cuda.is_available() else "cpu") + + model.eval() + model = model.to(device) + img_lq = img_lq.to(device) + + window_size = 8 + scale = 4 + + # inference + with flow.no_grad(): + # pad input image to be a multiple of window_size + _, _, h_old, w_old = img_lq.size() + h_pad = (h_old // window_size + 1) * window_size - h_old + w_pad = (w_old // window_size + 1) * window_size - w_old + img_lq = flow.cat([img_lq, flow.flip(img_lq, [2])], 2)[:, :, : h_old + h_pad, :] + img_lq = flow.cat([img_lq, flow.flip(img_lq, [3])], 3)[:, :, :, : w_old + w_pad] + output = model(img_lq) + output = output[..., : h_old * scale, : w_old * scale] + output = output.clamp_(0, 1) + return output + + +def upsample16x(imgs, model): + return upsample4x(upsample4x(imgs, model), model) diff --git a/projects/DALLE2/swinir/utils.py b/projects/DALLE2/swinir/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..09b7e23d689ae4c89a833e2bdc3cd1744e693bda --- /dev/null +++ b/projects/DALLE2/swinir/utils.py @@ -0,0 +1,132 @@ +# ----------------------------------------------------------------------------------- +# from +# https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/layers/weight_init.py +# ----------------------------------------------------------------------------------- +import collections.abc +import math +import warnings +from itertools import repeat + +import oneflow as flow +import oneflow.nn as nn + + +def _no_grad_trunc_normal_(tensor, mean, std, a, b): + # Cut & paste from Pytorch official master until it's in a few official releases - RW + # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0 + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn( + "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. " + "The distribution of values may be incorrect.", + stacklevel=2, + ) + + with flow.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + l = norm_cdf((a - mean) / std) + u = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * l - 1, 2 * u - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.0)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor, mean=0.0, std=1.0, a=-2.0, b=2.0): + # type: (flow.Tensor, float, float, float, float) -> flow.Tensor + r"""Fills the input Tensor with values drawn from a truncated + normal distribution. The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + NOTE: this impl is similar to the Pytorch trunc_normal_, the bounds [a, b] are + applied while sampling the normal with mean/std applied, therefore a, b args + should be adjusted to match the range of mean, std args. + Args: + tensor: an n-dimensional `torch.Tensor` + mean: the mean of the normal distribution + std: the standard deviation of the normal distribution + a: the minimum cutoff value + b: the maximum cutoff value + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.trunc_normal_(w) + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b) + + +# ----------------------------------------------------------------------------------- +# from https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/layers/helpers.py +# ----------------------------------------------------------------------------------- + +# From Pytorch internals +def _ntuple(n): + def parse(x): + if isinstance(x, collections.abc.Iterable) and not isinstance(x, str): + return x + return tuple(repeat(x, n)) + + return parse + + +to_1tuple = _ntuple(1) +to_2tuple = _ntuple(2) +to_3tuple = _ntuple(3) +to_4tuple = _ntuple(4) +to_ntuple = _ntuple + +# ----------------------------------------------------------------------------------- +# from https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/layers/drop.py +# ----------------------------------------------------------------------------------- + + +def drop_path(x, drop_prob: float = 0.0, training: bool = False, scale_by_keep: bool = True): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks). + This is the same as the DropConnect impl I created for EfficientNet, etc networks, however, + the original name is misleading as 'Drop Connect' is a different form of dropout in a + separate paper... + See discussion: https://github.com/tensortorch/tpu/issues/494#issuecomment-532968956 ... + I've opted for changing the layer and argument names to 'drop path' rather than mix DropConnect + as a layer name and use 'survival rate' as the argument. + """ + if drop_prob == 0.0 or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0],) + (1,) * (x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = x.new_empty(shape).bernoulli_(keep_prob) + if keep_prob > 0.0 and scale_by_keep: + random_tensor.div_(keep_prob) + return x * random_tensor + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" + + def __init__(self, drop_prob: float = 0.0, scale_by_keep: bool = True): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + self.scale_by_keep = scale_by_keep + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training, self.scale_by_keep) + + def extra_repr(self): + return f"drop_prob={round(self.drop_prob,3):0.3f}" diff --git a/projects/GLM/configs/glm_inference.py b/projects/GLM/configs/glm_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..bed922a339fd04fc8d11de5f44f2f9178b9fd21d --- /dev/null +++ b/projects/GLM/configs/glm_inference.py @@ -0,0 +1,61 @@ +from omegaconf import DictConfig +from libai.config import LazyCall +from projects.GLM.modeling_glm import GLMModel + + +cfg = dict( + num_layers=48, + vocab_size=30592, + hidden_size=4096, + num_attention_heads=64, + max_sequence_length=1024, + embedding_dropout_prob=0.1, + attention_dropout_prob=0.1, + output_dropout_prob=0.1, + layernorm_epsilon=1e-5, + initializer_range=0.02, + use_scaled_init_for_output_weights=True, + bias_gelu_fusion=True, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + amp_enabled=False, + block_position_encoding=True, + attention_scale=1.0, + padding_idx=None, + # Inference + is_encoder_decoder=False, + max_length=512, + min_length=0, + do_sample=False, + early_stopping=False, + num_beams=1, + num_beam_groups=1, + diversity_penalty=0.0, + temperature=1.0, + top_k=50, + top_p=1.0, + typical_p=1.0, + repetition_penalty=1.0, + length_penalty=1.0, + no_repeat_ngram_size=0, + encoder_no_repeat_ngram_size=0, + num_return_sequences=1, + chunk_size_feed_forward=0, + output_scores=False, + forced_bos_token_id=None, + forced_eos_token_id=None, + remove_invalid_values=False, + exponential_decay_length_penalty=None, + use_cache=False, + # Tokenizer + pad_token_id=50000, + eos_token_id=50007, + bos_token_id=None, + sep_token_id=None, + decoder_start_token_id=None, +) + +cfg = DictConfig(cfg) + +glm_model = LazyCall(GLMModel)(cfg=cfg) diff --git a/projects/GLM/layers/attention_layer.py b/projects/GLM/layers/attention_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..2d8581ec1b93b04c86ca77e8095c47dd3eea76a9 --- /dev/null +++ b/projects/GLM/layers/attention_layer.py @@ -0,0 +1,152 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import oneflow as flow +from oneflow import nn + +from libai.layers.linear import Linear + + +class MultiheadAttention(nn.Module): + def __init__( + self, + hidden_size, + num_attention_heads, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + attention_scale=1.0, + *, + layer_idx=0 + ): + super().__init__() + self.hidden_size = hidden_size + self.attention_scale = attention_scale + if output_layer_init_method is None: + output_layer_init_method = init_method + + assert ( + hidden_size % num_attention_heads == 0 + ), "hidden_size must be divisible by num_attention_heads." + + self.num_heads = num_attention_heads + self.head_size = hidden_size // num_attention_heads + + self.attention_dropout_prob = attention_dropout_prob + self.dropout = nn.Dropout(p=attention_dropout_prob) + self.norm_factor = 1.0 / math.sqrt(float(self.head_size)) + self.coeff = None + if apply_query_key_layer_scaling: + self.coeff = layer_idx + 1 + self.norm_factor /= self.coeff + + self.scale_mask_softmax_fusion = scale_mask_softmax_fusion + self.bias_dropout_fusion = bias_dropout_fusion + + if self.bias_dropout_fusion: + self.output_dropout_prob = output_dropout_prob + else: + self.output_dropout = nn.Dropout(p=output_dropout_prob) + + self.query_key_value = Linear( + self.hidden_size, + self.hidden_size * 3, + parallel="col", + init_method=init_method, + layer_idx=layer_idx, + ) + + self.dense = Linear( + self.hidden_size, + self.hidden_size, + parallel="row", + init_method=output_layer_init_method, + skip_bias_add=self.bias_dropout_fusion, + layer_idx=layer_idx, + ) + + def forward( + self, + hidden_states: flow.Tensor, + attention_mask: flow.Tensor = None, + mem=None, + ): + attention_mask = ( + attention_mask.to_global(placement=hidden_states.placement) + if attention_mask is not None + else None + ) + + bsz, tgt_len = hidden_states.size()[:2] + + if mem is not None: + hidden_states = flow.cat((mem, hidden_states), dim=1) + query_key_value = self.query_key_value(hidden_states) + query_key_value = query_key_value.view(bsz, -1, self.num_heads, 3 * self.head_size) + query_key_value = query_key_value.permute(0, 2, 1, 3) + query, key, value = flow.chunk(query_key_value, chunks=3, dim=-1) + if mem is not None: + query = query[:, :, -tgt_len:] + + if self.attention_scale > 1.0: + attention_scores = flow.matmul( + query / math.sqrt(self.attention_scale), + key / math.sqrt(self.head_size * self.attention_scale), + transpose_b=True, + ) + else: + attention_scores = flow.matmul(query, key, transpose_b=True, alpha=self.norm_factor) + + if self.scale_mask_softmax_fusion: + attention_weights = flow._C.fused_scale_mask_softmax_dropout( + attention_scores, + attention_mask, + fill_value=-10000.0, + scale=self.coeff, + p=self.attention_dropout_prob, + )[0] + else: + if self.coeff is not None: + attention_scores *= self.coeff + attention_scores = flow.mul(attention_scores, attention_mask) + attention_scores = attention_scores - 10000.0 * (1 - attention_mask) + attention_weights = flow.softmax(attention_scores, dim=-1) + attention_weights = self.dropout(attention_weights) + + context = flow.matmul(attention_weights, value) + context = context.transpose(1, 2) + output = self.dense(context.flatten(2)) + + if self.bias_dropout_fusion: + output, bias = output + output = flow._C.fused_bias_add_dropout( + output, bias, p=self.output_dropout_prob, axis=output.ndim - 1 + ) + else: + output = self.output_dropout(output) + + return output + + def extra_repr(self) -> str: + return "hidden_size={}, num_heads={}".format( + self.hidden_size, + self.num_heads, + ) diff --git a/projects/GLM/layers/embedding_layer.py b/projects/GLM/layers/embedding_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..446bee542f2c95e1475d4238ff427571af75f6a5 --- /dev/null +++ b/projects/GLM/layers/embedding_layer.py @@ -0,0 +1,79 @@ +import oneflow as flow +from oneflow import nn + +import libai.utils.distributed as dist +from libai.layers import Embedding, VocabEmbedding +from libai.models.utils import init_method_normal + + +class GLMEmbedding(nn.Module): + def __init__( + self, + vocab_size, + hidden_size, + max_seq_length, + padding_idx=None, + init_method=init_method_normal(0.02, 0), + embedding_dropout_prob=0.0, + amp_enabled=False, + block_position_encoding=False, + ): + super().__init__() + self.block_position_encoding = block_position_encoding + + self.word_embeddings = VocabEmbedding( + vocab_size, + hidden_size, + padding_idx=padding_idx, + init_method=init_method, + amp_enabled=amp_enabled, + ) + + if block_position_encoding: + self.position_embeddings = Embedding( + max_seq_length + 1, hidden_size, init_method=init_method, amp_enabled=amp_enabled + ) + self.block_position_embeddings = Embedding( + max_seq_length + 1, hidden_size, init_method=init_method, amp_enabled=amp_enabled + ) + self.embedding_dropout = nn.Dropout(embedding_dropout_prob) + + self.position_ids = flow.arange( + max_seq_length, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ).unsqueeze(0) + + self.block_position_ids = flow.zeros( + (1, max_seq_length), + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + + def forward(self, input_ids, position_ids=None): + bsz, seq_len = input_ids.size() + + if self.block_position_encoding and position_ids is not None: + position_ids, block_position_ids = position_ids[:, 0], position_ids[:, 1] + + if position_ids is None: + position_ids = self.position_ids[:, :seq_len] + position_ids = position_ids.expand_as(input_ids).to_global(sbp=input_ids.sbp) + block_position_ids = self.block_position_ids[:, :seq_len] + block_position_ids = block_position_ids.expand_as(input_ids).to_global( + sbp=input_ids.sbp + ) + + word_embeddings = self.word_embeddings(input_ids) + + position_embeddings = self.position_embeddings(position_ids) + input_embeddings = word_embeddings + position_embeddings + + if self.block_position_encoding: + block_position_embeddings = self.block_position_embeddings(block_position_ids) + input_embeddings = input_embeddings + block_position_embeddings + + input_embeddings = self.embedding_dropout(input_embeddings) + return input_embeddings diff --git a/projects/GLM/layers/position_embedding.py b/projects/GLM/layers/position_embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..90e47f7220e97de45441f97a4576258cd915ee8e --- /dev/null +++ b/projects/GLM/layers/position_embedding.py @@ -0,0 +1,64 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import oneflow as flow +from oneflow import nn + +import libai.utils.distributed as dist + + +class SinePositionalEmbedding(nn.Module): + def __init__(self, num_embeddings, embedding_dim): + super().__init__() + + self.embedding_dim = embedding_dim + self.num_embeddings = num_embeddings + + position_embedding = flow.zeros( + num_embeddings, + embedding_dim, + dtype=flow.float32, + placement=dist.get_layer_placement(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + position = flow._C.global_arange( + start=0, + end=num_embeddings, + placement=dist.get_layer_placement(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + dtype=flow.float32, + ).unsqueeze(1) + position_range = flow._C.global_arange( + start=0, + end=embedding_dim, + step=2, + placement=dist.get_layer_placement(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + dtype=flow.float32, + ) + div_term = flow.exp(position_range * (-math.log(10000.0) / embedding_dim)) + position_embedding[:, : embedding_dim // 2] = flow.sin(position * div_term) + position_embedding[:, embedding_dim // 2 :] = flow.cos(position * div_term) + self.register_buffer("position_embedding", position_embedding) + + def forward(self, position_ids): + position_embeds = flow._C.gather(self.position_embedding, position_ids, axis=0) + return position_embeds + + def extra_repr(self) -> str: + s = "num_embeddings={num_embeddings}, embedding_dim={embedding_dim}" + return s.format(**self.__dict__) diff --git a/projects/GLM/layers/transformer_layer.py b/projects/GLM/layers/transformer_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..ed7ba2a920e01e09a24882c7c7ecec6ad8011888 --- /dev/null +++ b/projects/GLM/layers/transformer_layer.py @@ -0,0 +1,131 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow.nn as nn + +from libai.layers.layer_norm import LayerNorm +from libai.layers.mlp import MLP +from libai.utils import distributed as dist +from projects.GLM.layers.attention_layer import MultiheadAttention + + +class TransformerLayer(nn.Module): + def __init__( + self, + hidden_size, + num_attention_heads, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + layernorm_epsilon=1e-5, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + attention_scale=1.0, + *, + layer_idx=0 + ): + super().__init__() + self.hidden_size = hidden_size + self.num_attention_heads = num_attention_heads + self.attention_dropout_prob = attention_dropout_prob + self.output_dropout_prob = output_dropout_prob + self.layernorm_epsilon = layernorm_epsilon + self.attention_scale = attention_scale + + self.layer_idx = layer_idx + + self.bias_gelu_fusion = bias_gelu_fusion + self.bias_dropout_fusion = bias_dropout_fusion + self.scale_mask_softmax_fusion = scale_mask_softmax_fusion + self.apply_query_key_layer_scaling = apply_query_key_layer_scaling + + self.init_method = init_method + if output_layer_init_method is None: + output_layer_init_method = init_method + self.output_layer_init_method = output_layer_init_method + + self.input_layernorm = LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon, layer_idx=self.layer_idx + ) + + self.attention = self.build_attention() + self.post_attention_layernorm = LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon, layer_idx=self.layer_idx + ) + + self.mlp = MLP( + self.hidden_size, + 4 * self.hidden_size, + self.output_dropout_prob, + self.init_method, + output_layer_init_method=self.output_layer_init_method, + bias_gelu_fusion=self.bias_gelu_fusion, + bias_dropout_fusion=self.bias_dropout_fusion, + layer_idx=self.layer_idx, + ) + + def forward( + self, + hidden_states, + attention_mask, + mem=None, + ): + hidden_states = hidden_states.to_global(placement=dist.get_layer_placement(self.layer_idx)) + attention_mask = ( + attention_mask.to_global(placement=dist.get_layer_placement(self.layer_idx)) + if attention_mask is not None + else None + ) + mem = ( + mem.to_global(placement=dist.get_layer_placement(self.layer_idx)) + if mem is not None + else None + ) + + layernorm_output = self.input_layernorm(hidden_states) + mem = self.input_layernorm(mem) if mem is not None else None + attention_output = self.attention( + layernorm_output, + attention_mask=attention_mask, + mem=mem, + ) + + hidden_states = hidden_states + attention_output + + layernorm_output = self.post_attention_layernorm(hidden_states) + + mlp_output = self.mlp(layernorm_output) + + output = hidden_states + mlp_output + + return output + + def build_attention(self): + return MultiheadAttention( + self.hidden_size, + self.num_attention_heads, + attention_dropout_prob=self.attention_dropout_prob, + output_dropout_prob=self.output_dropout_prob, + init_method=self.init_method, + output_layer_init_method=self.output_layer_init_method, + bias_dropout_fusion=self.bias_dropout_fusion, + scale_mask_softmax_fusion=self.scale_mask_softmax_fusion, + apply_query_key_layer_scaling=self.apply_query_key_layer_scaling, + attention_scale=self.attention_scale, + layer_idx=self.layer_idx, + ) diff --git a/projects/GLM/modeling_glm.py b/projects/GLM/modeling_glm.py new file mode 100644 index 0000000000000000000000000000000000000000..5e62690b9c9bd40e8035c47066107d183b4a8622 --- /dev/null +++ b/projects/GLM/modeling_glm.py @@ -0,0 +1,471 @@ +import oneflow as flow +import oneflow.nn.functional as F +from oneflow import nn + +import libai.utils.distributed as dist +from libai.config import configurable +from libai.inference.generator.generation_utils import Generator +from libai.layers import LayerNorm, LMLogits, ParallelCrossEntropyLoss +from libai.models.utils import init_method_normal, scaled_init_method_normal +from projects.GLM.layers.embedding_layer import GLMEmbedding +from projects.GLM.layers.transformer_layer import TransformerLayer + + +class Transformer(nn.Module): + def __init__( + self, + num_layers, + hidden_size, + num_attention_heads, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + layernorm_epsilon=1.0e-5, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + attention_scale=1.0, + ): + super().__init__() + self.num_layers = num_layers + + def build_layer(layer_number): + return TransformerLayer( + hidden_size, + num_attention_heads, + attention_dropout_prob=attention_dropout_prob, + output_dropout_prob=output_dropout_prob, + layernorm_epsilon=layernorm_epsilon, + init_method=init_method, + output_layer_init_method=output_layer_init_method, + bias_gelu_fusion=bias_gelu_fusion, + bias_dropout_fusion=bias_dropout_fusion, + scale_mask_softmax_fusion=scale_mask_softmax_fusion, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + attention_scale=attention_scale, + layer_idx=layer_number, + ) + + self.layers = nn.ModuleList([build_layer(i) for i in range(self.num_layers)]) + self.final_layernorm = LayerNorm(hidden_size, eps=layernorm_epsilon, layer_idx=-1) + + def forward(self, hidden_states, attention_mask, memory_states=None): + mem_layers = [hidden_states.detach()] + + for i, layer in enumerate(self.layers): + mem_i = memory_states[i] if memory_states is not None else None + hidden_states = layer(hidden_states, attention_mask, mem=mem_i) + mem_layers.append(hidden_states.detach()) + + output = self.final_layernorm(hidden_states) + + return output, mem_layers + + +class GLMModel(nn.Module): + @configurable + def __init__( + self, + num_layers, + vocab_size, + hidden_size, + num_attention_heads, + max_sequence_length=1024, + embedding_dropout_prob=0.0, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + layernorm_epsilon=1e-5, + initializer_range=0.02, + use_scaled_init_for_output_weights=True, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + amp_enabled=False, + block_position_encoding=False, + attention_scale=1.0, + padding_idx=None, + ): + super().__init__() + init_method = init_method_normal(sigma=initializer_range, mean=0) + if use_scaled_init_for_output_weights: + output_layer_init_method = scaled_init_method_normal(initializer_range, num_layers) + else: + output_layer_init_method = init_method + + self.embeddings = GLMEmbedding( + vocab_size, + hidden_size, + max_sequence_length, + padding_idx=padding_idx, + init_method=init_method, + embedding_dropout_prob=embedding_dropout_prob, + amp_enabled=amp_enabled, + block_position_encoding=block_position_encoding, + ) + + self.transformer = Transformer( + num_layers, + hidden_size, + num_attention_heads, + attention_dropout_prob=attention_dropout_prob, + output_dropout_prob=output_dropout_prob, + layernorm_epsilon=layernorm_epsilon, + init_method=init_method, + output_layer_init_method=output_layer_init_method, + bias_gelu_fusion=bias_gelu_fusion, + bias_dropout_fusion=bias_dropout_fusion, + scale_mask_softmax_fusion=scale_mask_softmax_fusion, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + attention_scale=attention_scale, + ) + + self.lm_head = LMLogits(vocab_size, bias=False) + + @classmethod + def from_config(cls, cfg): + return { + "num_layers": cfg.num_layers, + "vocab_size": cfg.vocab_size, + "hidden_size": cfg.hidden_size, + "num_attention_heads": cfg.num_attention_heads, + "max_sequence_length": cfg.max_sequence_length, + "embedding_dropout_prob": cfg.embedding_dropout_prob, + "attention_dropout_prob": cfg.attention_dropout_prob, + "output_dropout_prob": cfg.output_dropout_prob, + "layernorm_epsilon": cfg.layernorm_epsilon, + "initializer_range": cfg.initializer_range, + "use_scaled_init_for_output_weights": cfg.use_scaled_init_for_output_weights, + "bias_gelu_fusion": cfg.bias_gelu_fusion, + "bias_dropout_fusion": cfg.bias_dropout_fusion, + "scale_mask_softmax_fusion": cfg.scale_mask_softmax_fusion, + "apply_query_key_layer_scaling": cfg.apply_query_key_layer_scaling, + "amp_enabled": cfg.amp_enabled, + "block_position_encoding": cfg.block_position_encoding, + "attention_scale": cfg.attention_scale, + "padding_idx": cfg.padding_idx, + } + + def forward( + self, + input_ids, + position_ids=None, + attention_mask=None, + memory_states=None, + output_predict=True, + ): + input_ids = input_ids.to_global(placement=dist.get_layer_placement(0)) + position_ids = ( + position_ids.to_global(placement=dist.get_layer_placement(0)) + if position_ids is not None + else None + ) + attention_mask = ( + attention_mask.to_global(placement=dist.get_layer_placement(0)) + if attention_mask is not None + else None + ) + + batch_size, query_length = input_ids.size() + memory_length = memory_states[0].size(1) if memory_states is not None else 0 + is_scalar = flow.numel(attention_mask) == 1 + is_sep = is_scalar or flow.numel(attention_mask) == batch_size + + if is_sep: + sep = attention_mask.item() if is_scalar else attention_mask + attention_mask = self.build_mask_matrix( + batch_size, query_length, sep, memory_length=memory_length, is_scalar=is_scalar + ) + else: + if attention_mask.dim() == 2: + attention_mask = attention_mask.unsqueeze(1).unsqueeze(1) + attention_mask = attention_mask[:, :, :, -query_length - memory_length :] + + input_embeds = self.embeddings(input_ids, position_ids) + + logits, mem_layers = self.transformer( + input_embeds, attention_mask=attention_mask, memory_states=memory_states + ) + mem_layers = self.update_mems(mem_layers, memory_states) + + if output_predict: + logits = self.lm_head(logits, self.embeddings.word_embeddings.weight) + + return (logits, mem_layers) + + @staticmethod + def set_activation_checkpoint(model): + for module_block in model.modules(): + # Old API in OneFlow 0.8 + if hasattr(module_block, "origin"): + if isinstance(module_block.origin, TransformerLayer): + module_block.config.activation_checkpointing = True + else: + if isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(nn.graph.GraphModule).activation_checkpointing = True + + def build_mask_matrix(self, batch_size, seq_length, sep, memory_length=0, is_scalar=False): + m = flow.tril( + flow.ones((1, seq_length, seq_length)), + ) + if is_scalar: + m[0, :, : int(sep)] = 1 + else: + m = m.expand(batch_size, -1, -1) + ids = flow.arange(seq_length, device=sep.device, dtype=sep.dtype).view(1, -1) + mask = ids < sep.view(-1, 1) + m = m.masked_fill(mask.unsqueeze(1).expand_as(m), 1) + if memory_length > 0: + m = m.expand(batch_size, -1, -1) + m = flow.cat((flow.ones((batch_size, seq_length, memory_length)), m), dim=2) + m = m.unsqueeze(1) + m = m.to_global( + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + return m + + def update_mems(self, hiddens, mems): + memory_length = mems[0].size(1) if mems is not None else 0 + query_length = hiddens[0].size(1) + new_memory_length = memory_length + query_length + + new_mems = [] + for i in range(len(hiddens)): + if new_memory_length <= query_length: + new_mems.append(hiddens[i][:, -new_memory_length:]) + else: + new_mems.append( + flow.cat((mems[i][:, -new_memory_length + query_length :], hiddens[i]), dim=1) + ) + return new_mems + + +class GLMLoss(nn.Module): + def __init__(self): + super().__init__() + self.loss_func = ParallelCrossEntropyLoss() + + def forward(self, logits, labels): + lm_loss = self.loss_func(logits, labels) + lm_loss = lm_loss.mean() + return {"lm_loss": lm_loss} + + +class GLMForMultipleChoice(nn.Module): + def __init__(self, cfg): + super().__init__() + self.glm = GLMModel(cfg) + self.loss_func = GLMLoss() + + def forward( + self, + input_ids=None, + position_ids=None, + attention_mask=None, + choice_ids=None, + choice_indices=None, + labels=None, + mems=None, + **kwargs, + ): + lm_logits, mem_layers = self.glm( + input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + memory_states=mems, + **kwargs, + ) + outputs = F.log_softmax(lm_logits, dim=-1) + log_probs = [] + for output, choices, choice_index in zip(outputs, choice_ids, choice_indices): + log_probs_single = [] + for choice, choice_target_id in zip(choices, choice_index): + tmp = output[choice_target_id, choice] + log_probs_single.append(tmp.sum()) + log_probs.append(flow.stack(log_probs_single)) + log_probs = flow.stack(log_probs) + loss = None + if labels is not None: + loss = self.loss_func(log_probs, labels) + return {"loss": loss, "logits": log_probs, "lm_logits": lm_logits, "mems": mem_layers} + + +class GLMForConditionalGeneration(nn.Module, Generator): + @configurable + def __init__( + self, + num_layers, + vocab_size, + hidden_size, + num_attention_heads, + max_sequence_length=1024, + embedding_dropout_prob=0.0, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + layernorm_epsilon=1e-5, + initializer_range=0.02, + use_scaled_init_for_output_weights=True, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + amp_enabled=False, + block_position_encoding=False, + attention_scale=1.0, + padding_idx=None, + cfg=None, + ): + super().__init__() + self.cfg = cfg + self.glm = GLMModel( + num_layers=num_layers, + vocab_size=vocab_size, + hidden_size=hidden_size, + num_attention_heads=num_attention_heads, + max_sequence_length=max_sequence_length, + embedding_dropout_prob=embedding_dropout_prob, + attention_dropout_prob=attention_dropout_prob, + output_dropout_prob=output_dropout_prob, + layernorm_epsilon=layernorm_epsilon, + initializer_range=initializer_range, + use_scaled_init_for_output_weights=use_scaled_init_for_output_weights, + bias_gelu_fusion=bias_gelu_fusion, + bias_dropout_fusion=bias_dropout_fusion, + scale_mask_softmax_fusion=scale_mask_softmax_fusion, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + amp_enabled=amp_enabled, + block_position_encoding=block_position_encoding, + attention_scale=attention_scale, + padding_idx=padding_idx, + cfg=cfg, + ) + self.loss_func = GLMLoss() + + @classmethod + def from_config(cls, cfg): + return { + "num_layers": cfg.num_layers, + "vocab_size": cfg.vocab_size, + "hidden_size": cfg.hidden_size, + "num_attention_heads": cfg.num_attention_heads, + "max_sequence_length": cfg.max_sequence_length, + "embedding_dropout_prob": cfg.embedding_dropout_prob, + "attention_dropout_prob": cfg.attention_dropout_prob, + "output_dropout_prob": cfg.output_dropout_prob, + "layernorm_epsilon": cfg.layernorm_epsilon, + "initializer_range": cfg.initializer_range, + "use_scaled_init_for_output_weights": cfg.use_scaled_init_for_output_weights, + "bias_gelu_fusion": cfg.bias_gelu_fusion, + "bias_dropout_fusion": cfg.bias_dropout_fusion, + "scale_mask_softmax_fusion": cfg.scale_mask_softmax_fusion, + "apply_query_key_layer_scaling": cfg.apply_query_key_layer_scaling, + "amp_enabled": cfg.amp_enabled, + "block_position_encoding": cfg.block_position_encoding, + "attention_scale": cfg.attention_scale, + "padding_idx": cfg.padding_idx, + "cfg": cfg, + } + + def forward( + self, + input_ids=None, + position_ids=None, + attention_mask=None, + labels=None, + memory_states=None, + **kwargs, + ): + lm_logits, mems = self.glm( + input_ids, position_ids, attention_mask, memory_states=memory_states, **kwargs + ) + loss = None + if labels is not None: + loss = self.loss_func(lm_logits, labels) + return {"loss": loss, "logits": lm_logits, "mems": mems} + + def _reorder_cache(self, past, beam_idx): + if past is None: + return past + reordered_decoder_past = () + for layer_past_states in past: + beam_idx = beam_idx.to_global(placement=layer_past_states.placement) + reordered_decoder_past = reordered_decoder_past + ( + layer_past_states.index_select(0, beam_idx), + ) + return reordered_decoder_past + + def prepare_inputs_for_generation( + self, + input_ids, + past=None, + position_ids=None, + generation_attention_mask=None, + **kwargs, + ): + attention_mask = generation_attention_mask + # only last token for inputs_ids if past is defined in kwargs + seq_length = input_ids.shape[1] + if past: + if position_ids is not None: + position_ids = position_ids[:, :, seq_length - 1].unsqueeze(-1) + if attention_mask is not None: + attention_mask = attention_mask[:, :, seq_length - 1, :seq_length].unsqueeze(-2) + input_ids = input_ids[:, -1].unsqueeze(-1) + else: + if position_ids is not None: + position_ids = position_ids[:, :, :seq_length] + if attention_mask is not None: + attention_mask = attention_mask[:, :, :seq_length, :seq_length] + return { + "input_ids": input_ids, + "position_ids": position_ids, + "attention_mask": attention_mask, + "memory_states": past, + } + + @staticmethod + def set_pipeline_stage_id(model: nn.Module): + dist_utils = dist.get_dist_util() + + if hasattr(model.glm.transformer.final_layernorm, "config"): + # Old API in OneFlow 0.8 + for module_block in model.modules(): + if isinstance(module_block.origin, GLMEmbedding): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, TransformerLayer): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.origin, (LMLogits, GLMLoss)): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + model.glm.transformer.final_layernorm.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + else: + for module_block in model.modules(): + if isinstance(module_block.to(nn.Module), GLMEmbedding): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.to(nn.Module), (LMLogits, GLMLoss)): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + model.glm.transformer.final_layernorm.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) diff --git a/projects/GLM/readme.md b/projects/GLM/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..d0eca0d76d6005f48275167b45f6b0f447e7e2e4 --- /dev/null +++ b/projects/GLM/readme.md @@ -0,0 +1,214 @@ +# GLM + +2017 年, Google 提出了 Transformer 架构, 随后 BERT 、GPT、T5等预训练模型不断涌现, 并在各项任务中都不断刷新 SOTA 纪录。去年, 清华提出了 GLM 模型(https://github.com/THUDM/GLM), 不同于上述预训练模型架构，它采用了一种自回归的空白填充方法, 在 NLP 领域三种主要的任务（自然语言理解、无条件生成、有条件生成）上都取得了不错的结果。 + +在LiBai中主要实现了GLM推理部分的工作，训练相关内容可以参考： + +- [GLM国产大模型训练加速：性能最高提升3倍，显存节省1/3，低成本上手](https://mp.weixin.qq.com/s/dkTGXuJV38KuLb4_LmM20Q) +- https://github.com/Oneflow-Inc/one-glm + + +## GLM-Inference +当模型规模过于庞大，单个 GPU 设备无法容纳大规模模型参数时，便捷好用的分布式训练和推理需求就相继出现，业内也随之推出相应的工具。 + +基于 OneFlow 构建的 LiBai 模型库让分布式上手难度降到最低，用户不需要关注模型如何分配在不同的显卡设备，只需要修改几个配置数据就可以设置不同的分布式策略。当然，加速性能更是出众。 + +用 LiBai 搭建的 GLM 可以便捷地实现model parallel + pipeline parallel推理, 很好地解决单卡放不下大规模模型的问题。 + +那么，用户如何利用大规模模型训练与推理仓库 LiBai 来构建 GLM 的分布式推理部分？下面用一个小例子解释一下。 + +### 分布式推理具有天然优势 + +要知道，模型的参数其实就是许多 tensor，也就是以矩阵的形式出现，大模型的参数也就是大矩阵，并行策略就是把大矩阵分为多个小矩阵，并分配到不同的显卡或不同的设备上，基础的 LinearLayer 在LiBai中的实现代码如下： + +```python +class Linear1D(nn.Module): + def __init__(self, in_features, out_features, parallel="data", layer_idx=0, ...): + super().__init__() + + if parallel == "col": + weight_sbp = dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.split(0)]) + elif parallel == "row": + weight_sbp = dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.split(1)]) + elif parallel == "data": + weight_sbp = dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]) + else: + raise KeyError(f"{parallel} is not supported! Only support ('data', 'row' and 'col')") + + self.weight = flow.nn.Parameter( + flow.empty( + (out_features, in_features), + dtype=flow.float32, + placement=dist.get_layer_placement(layer_idx), # for pipeline parallelism placement + sbp=weight_sbp, + ) + ) + init_method(self.weight) + ... + + def forward(self, x): + ... +``` + +在这里，用户可选择去如何切分 Linear 层的矩阵，如何切分数据矩阵，而OneFlow 中的 SBP 控制竖着切、横着切以及其他拆分矩阵的方案（模型并行、数据并行），以及通过设置 Placement 来控制这个 LinearLayer 是放在第几张显卡上（流水并行）。 + +所以，根据 LiBai 中各种 layer 的设计原理以及基于 OneFlow 中 tensor 自带的 SBP 和 Placement 属性的天然优势，使得用户搭建的模型能够很简单地就实现数据并行、模型并行以及流水并行操作。 + +### GLM 推理的 Demo 演示 + +这里为用户展示 LiBai 中 GLM 便捷的4卡`model parallel+pipeline parallel`推理 Demo，模型可在 HuggingFace 上获取：https://huggingface.co/models?filter=glm + + +#### glm-10b的文件结构 + +```python +$ tree data +path/to/glm-10b +├── added_tokens.json +├── vocab.json +├── merges.txt +├── config.json +└── pytorch_model.bin +``` + +#### 推理 + +运行以下代码： +```bash +# 运行前修改 glm_inference.py 中 `pad_token_id=0, eos_token_id=50258, bos_token_id=50000` +python3 -m oneflow.distributed.launch --nproc_per_node 4 demo.py +``` + +```python +# model parallel + pipeline parallel demo + +import oneflow as flow +from projects.GLM.tokenizer.glm_tokenizer import GLMGPT2Tokenizer +from libai.utils import distributed as dist +from projects.GLM.configs.glm_inference import cfg +from projects.GLM.modeling_glm import GLMForConditionalGeneration +from projects.GLM.utils.glm_loader import GLMLoaderHuggerFace +from omegaconf import DictConfig + +# 只需简单配置并行方案 +parallel_config = DictConfig( + dict( + data_parallel_size=1, + tensor_parallel_size=2, + pipeline_parallel_size=2, + pipeline_num_layers=2 * 24 + ) +) +dist.setup_dist_util(parallel_config) + +tokenizer = GLMGPT2Tokenizer.from_pretrained("/path/to/glm-10b") +input_ids = tokenizer.encode( + [ + "Ng is an adjunct professor at [MASK] (formerly associate professor and Director of its Stanford AI Lab or SAIL ). Also a pioneer in online education, Ng co-founded Coursera and deeplearning.ai." + ], + return_tensors="of", +) +inputs = {"input_ids": input_ids, "attention_mask": flow.ones(input_ids.size())} +inputs = tokenizer.build_inputs_for_generation(inputs, max_gen_length=512) + +sbp = dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]) +placement = dist.get_layer_placement(0) + +loader = GLMLoaderHuggerFace(GLMForConditionalGeneration, cfg, "/path/to/glm-10b") +model = loader.load() + +outputs = model.generate( + inputs=inputs['input_ids'].to_global(sbp=sbp, placement=placement), + position_ids=inputs['position_ids'].to_global(sbp=sbp, placement=placement), + generation_attention_mask=inputs['generation_attention_mask'].to_global(sbp=sbp, placement=placement), + max_length=512 +) +res = tokenizer.decode(outputs[0]) +if dist.is_main_process(): + print(res) + +>>> [CLS] Ng is an adjunct professor at [MASK] (formerly associate professor and Director of its Stanford AI Lab or SAIL ). Also a pioneer in online education, Ng co-founded Coursera and deeplearning.ai.<|endoftext|> <|startofpiece|> Stanford University and a co-founder of <|endofpiece|> + +``` + +#### glm-10b-chinese的文件结构 + +```python +$ tree data +path/to/glm-10b-chinese +├── added_tokens.json +├── cog-pretrain.model +├── config.json +└── pytorch_model.bin +``` + +#### 推理 + +运行以下代码： +```bash +# 运行前修改 glm_inference.py 中 `pad_token_id=50000, eos_token_id=50007, bos_token_id=None` +python3 -m oneflow.distributed.launch --nproc_per_node 4 demo.py +``` + +```python +# model parallel + pipeline parallel demo + +import oneflow as flow +from projects.GLM.tokenizer.glm_tokenizer import GLMChineseTokenzier +from libai.utils import distributed as dist +from projects.GLM.configs.glm_inference import cfg +from projects.GLM.modeling_glm import GLMForConditionalGeneration +from projects.GLM.utils.glm_loader import GLMLoaderHuggerFace +from omegaconf import DictConfig + +# 只需简单配置并行方案 +parallel_config = DictConfig( + dict( + data_parallel_size=1, + tensor_parallel_size=2, + pipeline_parallel_size=2, + pipeline_num_layers=2 * 24 + ) +) +dist.setup_dist_util(parallel_config) + +tokenizer = GLMChineseTokenzier.from_pretrained("/path/to/glm-10b-chinese") +input_ids = tokenizer.encode( + [ + "凯旋门位于意大利米兰市古城堡旁。1807年为纪念[MASK]而建，门高25米，顶上矗立两武士青铜古兵车铸像。" + ], + return_tensors="of", +) +inputs = {"input_ids": input_ids, "attention_mask": flow.ones(input_ids.size())} +inputs = tokenizer.build_inputs_for_generation(inputs, max_gen_length=512) + +sbp = dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]) +placement = dist.get_layer_placement(0) + +loader = GLMLoaderHuggerFace( + GLMForConditionalGeneration, + cfg, + "/path/to/glm-10b-chinese", + embedding_dropout_prob=0, + attention_dropout_prob=0, + output_dropout_prob=0, +) +model = loader.load() + +outputs = model.generate( + inputs=inputs['input_ids'].to_global(sbp=sbp, placement=placement), + position_ids=inputs['position_ids'].to_global(sbp=sbp, placement=placement), + generation_attention_mask=inputs['generation_attention_mask'].to_global(sbp=sbp, placement=placement), + max_length=512 +) + +res = tokenizer.decode(outputs[0]) +if dist.is_main_process(): + print(res) + +>>> [CLS] 凯旋门位于意大利米兰市古城堡旁。1807年为纪念 [MASK] 而建,门高25米,顶上矗立两武士青铜古兵车铸像。 <|endoftext|> <|startofpiece|> 拿破仑军队攻克米兰城 <|endofpiece|> +``` + +#### 使用 One-GLM 训练的模型进行推理 + +LiBai对于OneFlow的模型加载同样方便，如果你希望使用one-glm训练后的模型进行推理，只需简单的将上述demo中的 GLMLoaderHuggerFace 替换为 GLMLoaderLiBai。 \ No newline at end of file diff --git a/projects/GLM/tokenizer/glm_tokenizer.py b/projects/GLM/tokenizer/glm_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..f777ca28a7ef9a411bc46c6f936b8a42afa4d77a --- /dev/null +++ b/projects/GLM/tokenizer/glm_tokenizer.py @@ -0,0 +1,403 @@ +import logging +import os +from shutil import copyfile +from typing import List, Optional, Tuple + +import oneflow as flow +import sentencepiece as spm + +from libai.tokenizer import BertTokenizer, GPT2Tokenizer, PreTrainedTokenizer, RobertaTokenizer + +logger = logging.getLogger(__name__) + + +class GLMTokenizerMixin(PreTrainedTokenizer): + @property + def sop_token(self) -> Optional[str]: + return "<|startofpiece|>" + + @property + def sop_token_id(self) -> Optional[int]: + """ + `Optional[int]`: Id of the start token in the vocabulary, used when training a model with + autoregressive blank filling. + """ + return self.convert_tokens_to_ids(self.sop_token) + + @property + def eop_token(self) -> Optional[str]: + return "<|endofpiece|>" + + @property + def eop_token_id(self) -> Optional[int]: + """ + `Optional[int]`: Id of the end token in the vocabulary, used when training a model with + autoregressive blank filling. + """ + return self.convert_tokens_to_ids(self.eop_token) + + @property + def gmask_token_id(self) -> int: + return self.convert_tokens_to_ids("[gMASK]") + + @property + def smask_token_id(self) -> int: + return self.convert_tokens_to_ids("[sMASK]") + + @property + def mask_token_ids(self): + return [self.mask_token_id, self.smask_token_id, self.gmask_token_id] + + def _build_input_for_multiple_choice(self, context, choices): + context_id = context["input_ids"] + if flow.is_tensor(context_id): + context_id = context_id.tolist() + + division = len(context_id) + mask_position = context_id.index(self.mask_token_id) + + token = flow.tensor(context_id, dtype=flow.long) + attention_mask = [context["attention_mask"].expand(division, -1)] + position_id = flow.arange(division, dtype=flow.long) + block_position_id = flow.zeros(division, dtype=flow.long) + + choice_ids, choice_indices = [], [] + + for choice_str in choices: + res = self.encode(choice_str) + choice = flow.tensor(res, dtype=flow.long) + choice_ids.append(choice) + choice_indices.append( + flow.arange(len(token), len(token) + len(choice), dtype=flow.long) + ) + attention_mask.append(flow.tril(flow.ones((len(choice), len(choice)), dtype=flow.long))) + + token = flow.cat( + (token, flow.tensor([self.sop_token_id], dtype=flow.long), choice[:-1]) + ) + position_id = flow.cat( + (position_id, flow.tensor([mask_position] * len(choice), dtype=flow.long)) + ) + block_position_id = flow.cat( + (block_position_id, flow.arange(1, 1 + len(choice), dtype=flow.long)) + ) + + attention_mask = flow.block_diag(*attention_mask) + attention_mask[division:, :division] = context["attention_mask"].unsqueeze(0) + + return { + "input_ids": token, + "position_ids": flow.stack((position_id, block_position_id)), + "attention_mask": attention_mask, + "choice_ids": choice_ids, + "choice_indices": choice_indices, + } + + def _pad_batch(self, tokens, position_ids, attention_mask, max_seq_length): + pad_length = max_seq_length - len(tokens) + attention_mask = flow.nn.functional.pad( + attention_mask, + (0, pad_length, 0, pad_length), + mode="constant", + value=0, + ) + tokens = flow.cat((tokens, flow.zeros(pad_length, dtype=flow.long))) + position_ids = flow.cat( + (position_ids, position_ids[..., -1:].expand(-1, pad_length)), dim=-1 + ) + return tokens, position_ids, attention_mask + + def _collate(self, samples): + TILE = 1 + length_to_pad = ( + (max(map(lambda spl: len(spl["input_ids"]), samples)) + TILE - 1) // TILE * TILE + ) + + token_batch, position_id_batch, attention_mask_batch = [], [], [] + choices_batch, choice_target_ids_batch = [], [] + + for sample in samples: + token, position_id, attention_mask = self._pad_batch( + sample["input_ids"], sample["position_ids"], sample["attention_mask"], length_to_pad + ) + token_batch.append(token) + position_id_batch.append(position_id) + attention_mask_batch.append(attention_mask) + choices_batch.append(sample["choice_ids"]) + choice_target_ids_batch.append(sample["choice_indices"]) + return { + "input_ids": flow.stack(token_batch), + "position_ids": flow.stack(position_id_batch), + "attention_mask": flow.stack(attention_mask_batch).unsqueeze(1), + "choice_ids": choices_batch, + "choice_indices": choice_target_ids_batch, + } + + def build_inputs_for_multiple_choice(self, model_input, choices, max_length=None): + samples = [ + {key: value[i] for key, value in model_input.items()} + for i in range(len(model_input["input_ids"])) + ] + samples = [ + self._build_input_for_multiple_choice(sample, choice) + for sample, choice in zip(samples, choices) + ] + inputs = self._collate(samples) + return inputs + + def build_inputs_for_generation( + self, model_input, max_gen_length=512, targets=None, padding=False + ): + mask_ids = self.mask_token_ids + input_ids = model_input["input_ids"] + batch_size, seq_length = input_ids.shape[:2] + position_id, block_position_id = list(range(seq_length)), [0 for _ in range(seq_length)] + position_ids, block_position_ids = [], [] + labels = None + if targets is not None: + is_batched = isinstance(targets, (list, tuple)) + targets = self.encode(targets) + if not is_batched: + targets = [targets] + assert len(targets) == len(input_ids) + targets = [(target + [self.eop_token_id])[:max_gen_length] for target in targets] + if not padding: + max_gen_length = max(map(len, targets)) + targets = [[self.sop_token_id] + target for target in targets] + labels = [target[1:] for target in targets] + targets = [ + target + [self.pad_token_id] * (max_gen_length + 1 - len(target)) + for target in targets + ] + labels = [label + [-100] * (max_gen_length - len(label)) for label in labels] + targets = flow.tensor(targets, dtype=input_ids.dtype) + labels = flow.tensor(labels, dtype=input_ids.dtype) + labels = flow.cat((input_ids.new_full((batch_size, seq_length), -100), labels), dim=1) + for i in range(batch_size): + mask_positions = [] + for mask_id in mask_ids: + mask_positions += (input_ids[i] == mask_id).nonzero(as_tuple=True)[0].tolist() + if not mask_positions: + raise ValueError("Cannot find mask token in the input") + mask_positions.sort() + mask_pos = mask_positions[0] + position_ids.append(position_id + [mask_pos] * max_gen_length) + block_position_ids.append(block_position_id + list(range(1, max_gen_length + 1))) + position_ids = flow.tensor(position_ids, dtype=input_ids.dtype) + block_position_ids = flow.tensor(block_position_ids, dtype=input_ids.dtype) + position_ids = flow.stack((position_ids, block_position_ids), dim=1) + attention_mask = model_input["attention_mask"] + attention_mask = attention_mask.unsqueeze(1).expand(-1, seq_length + max_gen_length, -1) + generation_attention_mask = ( + flow.cat( + [ + attention_mask.new_zeros((seq_length, max_gen_length)), + flow.tril(attention_mask.new_ones((max_gen_length, max_gen_length))), + ], + dim=0, + ) + .unsqueeze(0) + .expand(batch_size, -1, -1) + ) + attention_mask = flow.cat((attention_mask, generation_attention_mask), dim=2) + attention_mask = attention_mask.unsqueeze(1) + if targets is None: + input_ids = flow.cat( + (input_ids, input_ids.new_full((batch_size, 1), self.sop_token_id)), dim=-1 + ) + else: + input_ids = flow.cat((input_ids, targets[:, :-1]), dim=1) + batch = {"input_ids": input_ids, "position_ids": position_ids} + if labels is None: + batch["generation_attention_mask"] = attention_mask + else: + batch["attention_mask"] = attention_mask + batch["labels"] = labels + return batch + + +class GLMRobertaTokenizer(RobertaTokenizer, GLMTokenizerMixin): + model_input_names = ["input_ids", "position_ids", "attention_mask"] + truncation_side: str = "left" + + @property + def gmask_token_id(self) -> int: + raise NotImplementedError("The model doesn't support gMASK") + + @property + def smask_token_id(self) -> int: + raise NotImplementedError("The model doesn't support sMASK") + + @property + def mask_token_ids(self): + return [self.mask_token_id] + + +class GLMChineseTokenzier(GLMTokenizerMixin): + vocab_files_names = {"vocab_file": "cog-pretrain.model"} + truncation_side: str = "left" + + def __init__( + self, + vocab_file, + eos_token="<|endoftext|>", + unk_token="[UNK]", + pad_token="<|endoftext|>", + additional_special_tokens=["<|startofpiece|>", "<|endofpiece|>", "[gMASK]", "[sMASK]"], + add_bos_token=False, + **kwargs, + ): + super().__init__( + eos_token=eos_token, + unk_token=unk_token, + pad_token=pad_token, + additional_special_tokens=additional_special_tokens, + **kwargs, + ) + self.add_bos_token = add_bos_token + self.vocab_file = vocab_file + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(vocab_file) + self._eos_token = "<|endoftext|>" + self._unk_token = "[UNK]" + self._pad_token = "<|endoftext|>" + self._cls_token = "[CLS]" + self._mask_token = "[MASK]" + + @property + def vocab_size(self): + return len(self.sp_model) + + def get_vocab(self): + vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} + vocab.update(self.added_tokens_encoder) + return vocab + + def _tokenize(self, text, **kwargs): + return self.sp_model.encode(text, out_type=str) + + def _convert_token_to_id(self, token): + """Converts a token (str) in an id using the vocab.""" + return self.sp_model.PieceToId(token) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + return self.sp_model.IdToPiece(index) + + def convert_tokens_to_string(self, tokens): + return self.sp_model.decode(tokens) + + def save_vocabulary( + self, save_directory: str, filename_prefix: Optional[str] = None + ) -> Tuple[str]: + if not os.path.isdir(save_directory): + logger.error(f"Vocabulary path ({save_directory}) should be a directory") + return + out_vocab_file = os.path.join( + save_directory, + (filename_prefix + "-" if filename_prefix else "") + + self.vocab_files_names["vocab_file"], + ) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile( + self.vocab_file + ): + copyfile(self.vocab_file, out_vocab_file) + elif not os.path.isfile(self.vocab_file): + with open(out_vocab_file, "wb") as fi: + content_spiece_model = self.sp_model.serialized_model_proto() + fi.write(content_spiece_model) + + return (out_vocab_file,) + + def build_inputs_with_special_tokens( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks + by concatenating and + adding special tokens. A BERT sequence has the following format: + - single sequence: ``[CLS] X [SEP]`` + - pair of sequences: ``[CLS] A [SEP] B [SEP]`` + Args: + token_ids_0 (:obj:`List[int]`): + List of IDs to which the special tokens will be added. + token_ids_1 (:obj:`List[int]`, `optional`): + Optional second list of IDs for sequence pairs. + Returns: + :obj:`List[int]`: List of `input IDs <../glossary.html#input-ids>`__ with the + appropriate special tokens. + """ + assert token_ids_1 is None + cls = [self.cls_token_id] + eos = [self.eos_token_id] + return cls + token_ids_0 + eos + + +class GLMGPT2Tokenizer(GPT2Tokenizer, GLMTokenizerMixin): + model_input_names = ["input_ids", "position_ids", "attention_mask"] + truncation_side: str = "left" + + def __init__( + self, + vocab_file, + merges_file, + errors="replace", + unk_token="<|endoftext|>", + bos_token="<|endoftext|>", + eos_token="<|endoftext|>", + add_bos_token=False, + **kwargs, + ): + super().__init__( + vocab_file, + merges_file, + errors, + unk_token, + bos_token, + eos_token, + add_bos_token, + **kwargs, + ) + self.cls_token = "[CLS]" + self.mask_token = "[MASK]" + + def build_inputs_with_special_tokens( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks + by concatenating and + adding special tokens. A BERT sequence has the following format: + - single sequence: ``[CLS] X [SEP]`` + - pair of sequences: ``[CLS] A [SEP] B [SEP]`` + Args: + token_ids_0 (:obj:`List[int]`): + List of IDs to which the special tokens will be added. + token_ids_1 (:obj:`List[int]`, `optional`): + Optional second list of IDs for sequence pairs. + Returns: + :obj:`List[int]`: List of `input IDs <../glossary.html#input-ids>`__ with the + appropriate special tokens. + """ + assert token_ids_1 is None + cls = [self.cls_token_id] + eos = [self.eos_token_id] + return cls + token_ids_0 + eos + + +class GLMBertTokenizer(BertTokenizer, GLMTokenizerMixin): + model_input_names = ["input_ids", "position_ids", "attention_mask"] + truncation_side: str = "left" + + @property + def gmask_token_id(self) -> int: + raise NotImplementedError("The model doesn't support gMASK") + + @property + def smask_token_id(self) -> int: + raise NotImplementedError("The model doesn't support sMASK") + + @property + def mask_token_ids(self): + return [self.mask_token_id] diff --git a/projects/GLM/utils/glm_loader.py b/projects/GLM/utils/glm_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..ed2c0cfd7386f26b0ca6873b0df53387ffb2c617 --- /dev/null +++ b/projects/GLM/utils/glm_loader.py @@ -0,0 +1,158 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from libai.models.utils import ModelLoaderHuggerFace, ModelLoaderLiBai + + +class GLMLoaderHuggerFace(ModelLoaderHuggerFace): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + + """NOTE: base_model_prefix_1 is GLM's prefix in Transformers. + base_model_prefix_2 is GLM's prefix in LiBai.""" + self.base_model_prefix_1 = "glm" + self.base_model_prefix_2 = "glm" + + def _convert_state_dict(self, flow_state_dict, cfg): + """Convert state_dict's keys to match model. + + Args: + flow_state_dict (OrderedDict): model state dict. + cfg (dict): model's default config dict in LiBai. + + Returns: + OrderedDict: flow state dict. + """ + # The converted checkpoint. + oneflow_state_dict = flow_state_dict.copy() + old_keys = list(oneflow_state_dict.keys()) + + # Get configs + num_heads = cfg.get("num_attention_heads") + hidden_size = cfg.get("hidden_size") + head_size = int(hidden_size / num_heads) + + # prefix + has_prefix = any(s.startswith(self.base_model_prefix_1) for s in oneflow_state_dict) + prefix1 = self.base_model_prefix_1 + "." if has_prefix else "" + prefix2 = "glm." if has_prefix else "" + + # Convert Embedding layers. + new_key = prefix2 + "embeddings.word_embeddings.weight" + old_keys.remove(prefix1 + "word_embeddings.weight") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(prefix1 + "word_embeddings.weight") + + if cfg.get("block_position_encoding", False) is True: + new_key = prefix2 + "embeddings.position_embeddings.weight" + old_keys.remove(prefix1 + "transformer.position_embeddings.weight") + oneflow_state_dict[new_key] = oneflow_state_dict.pop( + prefix1 + "transformer.position_embeddings.weight" + ) + + new_key = prefix2 + "embeddings.block_position_embeddings.weight" + old_keys.remove(prefix1 + "transformer.block_position_embeddings.weight") + oneflow_state_dict[new_key] = oneflow_state_dict.pop( + prefix1 + "transformer.block_position_embeddings.weight" + ) + + # Convert other layers. + for key in old_keys: + if "query_key_value" in key: + qkv = oneflow_state_dict.pop(key) + qkv = self._fix_qkv_ordering(qkv, head_size, num_heads) + oneflow_state_dict[prefix2 + key] = qkv + else: + oneflow_state_dict[prefix2 + key] = oneflow_state_dict.pop(key) + + return oneflow_state_dict + + def _load_config_from_json(self, config_file): + """load config from `config.json`, and update default config. + + Args: + config_file (str): Path of config file. + """ + with open(config_file, mode="r", encoding="utf-8") as f: + cfg_dict = json.load(f) + # update libai_cfg by config.json + for k, v in cfg_dict.items(): + self._update_cfg(k, v) + + # update libai_cfg by kwargs + for k, v in self.kwargs.items(): + self._update_cfg(k, v) + + self._update_cfg_log() + + +class GLMLoaderLiBai(ModelLoaderLiBai): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + self.base_model_prefix_2 = "glm" + + def _convert_state_dict(self, flow_state_dict, cfg): + """Convert state_dict's keys to match model. + + Args: + flow_state_dict (OrderedDict): model state dict. + cfg (dict): model's default config dict in LiBai. + + Returns: + OrderedDict: flow state dict. + """ + # The converted checkpoint. + oneflow_state_dict = flow_state_dict.copy() + old_keys = list(oneflow_state_dict.keys()) + + # Get configs + num_heads = cfg.get("num_attention_heads") + hidden_size = cfg.get("hidden_size") + head_size = int(hidden_size / num_heads) + + # prefix + has_prefix = any(s.startswith(self.base_model_prefix_1) for s in oneflow_state_dict) + prefix1 = self.base_model_prefix_1 + "." if has_prefix else "" + prefix2 = "glm." if has_prefix else "" + + # Convert Embedding layers. + new_key = prefix2 + "embeddings.word_embeddings.weight" + old_keys.remove(prefix1 + "word_embeddings.weight") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(prefix1 + "word_embeddings.weight") + + if cfg.get("block_position_encoding", False) is True: + new_key = prefix2 + "embeddings.position_embeddings.weight" + old_keys.remove(prefix1 + "transformer.position_embeddings.weight") + oneflow_state_dict[new_key] = oneflow_state_dict.pop( + prefix1 + "transformer.position_embeddings.weight" + ) + + new_key = prefix2 + "embeddings.block_position_embeddings.weight" + old_keys.remove(prefix1 + "transformer.block_position_embeddings.weight") + oneflow_state_dict[new_key] = oneflow_state_dict.pop( + prefix1 + "transformer.block_position_embeddings.weight" + ) + + # Convert other layers. + for key in old_keys: + if "query_key_value" in key: + qkv = oneflow_state_dict.pop(key) + qkv = self._fix_qkv_ordering(qkv, head_size, num_heads) + oneflow_state_dict[prefix2 + key] = qkv + else: + oneflow_state_dict[prefix2 + key] = oneflow_state_dict.pop(key) + + return oneflow_state_dict diff --git a/projects/MAE/README.md b/projects/MAE/README.md new file mode 100644 index 0000000000000000000000000000000000000000..13f90cb3fc146eb1b634912c595180c55da3235b --- /dev/null +++ b/projects/MAE/README.md @@ -0,0 +1,110 @@ +## MAE in LiBai +**Masked Autoencoders Are Scalable Vision Learners** + +Kaiming He, Xinlei Chen, Saining Xie, Yanghao Li, Piotr Dollár, Ross Girshick + +[[`arXiv`](https://arxiv.org/abs/2111.06377)] [[`BibTeX`](#Citation)] + +

+ +

+ +This is the OneFlow re-implementation of MAE based on [LiBai](https://libai.readthedocs.io/). + +## Catelog +- [x] MAE pretraining code +- [x] MAE finetune code + +## Supported parallel mode and task +Based on [libai.layers](https://libai.readthedocs.io/en/latest/modules/libai.layers.html), MAE model is automatically configured with the following parallelism mode. + + + + + + + + + + + + + + + + + + + + + + +
Model Data ParallelTensor ParallelPipeline Parallel
MAE pretrain ✔--
MAE finetune ✔✔✔
+ + +## Usage +### Installation +Please see [LiBai Installation](https://libai.readthedocs.io/en/latest/tutorials/get_started/Installation.html) to install LiBai + +### Prepare the Data +Please see [Prepare the Data](https://libai.readthedocs.io/en/latest/tutorials/get_started/quick_run.html#prepare-the-data). + + +### Pretraining +Pretraining MAE on 8 GPUs using data parallelism. +```bash +cd /path/to/libai +bash tools/train.sh projects/MAE/train_net.py projects/MAE/configs/mae_pretraining.py 8 +``` + +### Finetuning +1. Setup the weights for finetuning in [mae_finetune.py](./configs/mae_finetune.py) as follows: + +```python +# mae_funetune.py +finetune.enable = True # only load weight if enable is True +finetune.weight_style = "oneflow" # Set "oneflow" for loading oneflow checkpoints +finetune.path = "/path/to/checkpoint" # the checkpoint directory +``` +If you feel confused about the checkpoint format here, please refer to [Load and Save a Checkpoint in LiBai](https://libai.readthedocs.io/en/latest/tutorials/basics/Load_and_Save_Checkpoint.html) for more details. + +1. Finetune MAE on 8 GPUs using data parallelism. +```bash +cd /path/to/libai +bash tools/train.sh projects/MAE/train_net.py projects/MAE/configs/mae_finetune.py 8 +``` +**Notes:** if you want to finetune MAE models using different parallel strategies, please refer to the [Distributed Configuration Tutorial](https://libai.readthedocs.io/en/latest/tutorials/basics/Distributed_Configuration.html) + + +### Evaluation +Evaluate MAE model under LiBai on 8 GPUs: +```bash +cd /path/to/libai +bash tools/train.sh projects/MAE/train_net.py projects/MAE/configs/mae_finetune.py 8 --eval-only +``` + + +## Advanced Usage +### Finetune MAE with pytorch pretrained checkpoint +You can download pytorch pretrained weight from [MAE official repo](https://github.com/facebookresearch/mae#fine-tuning-with-pre-trained-checkpoints) and finetune them in LiBai by updating the [mae_finetune.py](./configs/mae_finetune.py) as follows: +```python +finetune.enable = True # only load weight if enable is True +finetune.weight_style = "pytorch" # Set "pytorch" for loading torch checkpoints +finetune.path = "/path/to/mae_finetuned_vit_base.pth" +``` +Run finetuning on 8 GPUs: +```bash +cd /path/to/libai +bash tools/train.sh projects/MAE/train_net.py projects/MAE/configs/mae_finetune.py 8 +``` + + +## Citation +```BibTeX +@article{he2021masked, + title={Masked autoencoders are scalable vision learners}, + author={He, Kaiming and Chen, Xinlei and Xie, Saining and Li, Yanghao and Doll{\'a}r, Piotr and Girshick, Ross}, + journal={arXiv preprint arXiv:2111.06377}, + year={2021} +} +``` \ No newline at end of file diff --git a/projects/MAE/configs/mae_finetune.py b/projects/MAE/configs/mae_finetune.py new file mode 100644 index 0000000000000000000000000000000000000000..2dd595ed69d7f96b29ba1d8c59737e57a9c3a538 --- /dev/null +++ b/projects/MAE/configs/mae_finetune.py @@ -0,0 +1,131 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from omegaconf import OmegaConf +from flowvision.data import Mixup + +# from flowvision.loss.cross_entropy import SoftTargetCrossEntropy +from libai.config import LazyCall, get_config +from modeling.cross_entropy import SoftTargetCrossEntropy +from configs.models.vit_base_patch16 import model +from utils.scheduler import ( + warmup_layerscale_cosine_lr_scheduler, + warmup_cosine_lr_scheduler, +) +from utils.lr_decay import param_groups_lrd + + +# Get train, optim and graph configs +train = get_config("common/train.py").train +optim = get_config("common/optim.py").optim +graph = get_config("common/models/graph.py").graph +dataloader = get_config("common/data/imagenet.py").dataloader + + +# number devices +n_gpus = 8 + +# Graph training +graph.enabled = True + +# Refine model cfg for vit training on imagenet +model.num_classes = 1000 +model.loss_func = LazyCall(SoftTargetCrossEntropy)() + +# Path to the weight for fine-tune +finetune = OmegaConf.create() +finetune.enable = True # only load weight if enable is True +finetune.weight_style = ( + "oneflow" # Set "oneflow" for loading oneflow weights, set "pytorch" for loading torch weights +) +finetune.path = "/path/to/pretrained_mae_weight" + + +# Refine data path to imagenet +dataloader.train.dataset[0].root = "/path/to/imagenet" +dataloader.test[0].dataset.root = "/path/to/imagenet" + +# Add Mixup Func +dataloader.train.mixup_func = LazyCall(Mixup)( + mixup_alpha=0.8, + cutmix_alpha=1.0, + prob=1.0, + switch_prob=0.5, + mode="batch", + label_smoothing=0.1, + num_classes=model.num_classes, +) + + +# Refine training settings for MAE finetune +train.train_micro_batch_size = 32 +train.num_accumulation_steps = 4 +train.test_micro_batch_size = 32 +effective_batch_size = train.train_micro_batch_size * train.num_accumulation_steps * n_gpus + +train.train_epoch = 100 +train.warmup_ratio = 5 / 100 +train.log_period = 20 +train.evaluation.eval_after_n_epoch = 1 +train.checkpointer.save_model_after_n_epoch = 1 + +# Set layer decay for MAE fine-tune +train.layer_decay = 0.65 + +# AMP +train.amp.enabled = True + + +# Base learning in MAE is set to 1.5e-4 +# The actually learning rate should be computed by linear scaling rule as follows: +# lr = base_lr * batch_size / 256 +# In LiBai, you should refine the actually learning rate due to your on settings +# Here we use 8 GPUs, 128 batch_size per GPU for training, batch_size equals to 1024 +base_lr = 5e-4 +actual_lr = base_lr * effective_batch_size / 256 + +# Refine optim settings +optim.params._target_ = param_groups_lrd +optim.params.weight_decay = 0.05 +optim.params.layer_decay = 0.65 +optim.lr = actual_lr + +del optim.params.clip_grad_max_norm +del optim.params.clip_grad_norm_type +del optim.params.weight_decay_norm +del optim.params.weight_decay_bias +del optim.weight_decay + +# Refine scheduler +if graph.enabled: + train.scheduler = LazyCall(warmup_cosine_lr_scheduler)( + warmup_factor=0.0, + min_lr=1e-6, + ) +else: + train.scheduler = LazyCall(warmup_layerscale_cosine_lr_scheduler)( + warmup_factor=0.0, + min_lr=1e-6, + ) + + +# Distributed Settings +train.dist.pipeline_num_layers = model.depth +train.dist.data_parallel_size = n_gpus +train.dist.tensor_parallel_size = 1 +train.dist.pipeline_parallel_size = 1 + + +eval_only = False diff --git a/projects/MAE/configs/mae_pretraining.py b/projects/MAE/configs/mae_pretraining.py new file mode 100644 index 0000000000000000000000000000000000000000..52d2d597aa7f39d0cabb9a82ea5cfef95b2dfdde --- /dev/null +++ b/projects/MAE/configs/mae_pretraining.py @@ -0,0 +1,102 @@ +from flowvision.transforms import transforms, InterpolationMode +from flowvision.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD + +from libai.config import LazyCall, get_config +from configs.models.mae_vit_base_patch16 import model +from data.pretraining_imagenet import PretrainingImageNetDataset +from utils.lr_decay import param_groups_weight_decay +from utils.scheduler import warmup_cosine_lr_scheduler + + +train = get_config("common/train.py").train +optim = get_config("common/optim.py").optim +graph = get_config("common/models/graph.py").graph +dataloader = get_config("common/data/imagenet.py").dataloader + + +# MAE Graph training for faster speed +graph.enabled = True + +# Refine data path to imagenet +dataloader.train.dataset[0].root = "/path/to/imagenet" +dataloader.train.dataset[0]._target_ = PretrainingImageNetDataset + +# No test data for pretraining +del dataloader.test + +# Refine data transform to MAE's default settings +transform_train = LazyCall(transforms.Compose)( + transforms=[ + LazyCall(transforms.RandomResizedCrop)( + size=(224, 224), + scale=(0.2, 1.0), + interpolation=InterpolationMode.BICUBIC, + ), + LazyCall(transforms.RandomHorizontalFlip)(), + LazyCall(transforms.ToTensor)(), + LazyCall(transforms.Normalize)( + mean=IMAGENET_DEFAULT_MEAN, + std=IMAGENET_DEFAULT_STD, + ), + ] +) +dataloader.train.dataset[0].transform = transform_train + + +# number devices +n_gpus = 8 + +# Refine training settings for MAE +train.train_micro_batch_size = 64 +train.num_accumulation_steps = 8 +effective_batch_size = train.train_micro_batch_size * train.num_accumulation_steps * n_gpus + +train.train_epoch = 800 +train.warmup_ratio = 40 / 800 +train.log_period = 20 +train.checkpointer.save_model_after_n_epoch = 20 + +# enable activation checkpointing +# train.activation_checkpoint.enabled = True + +# set rdma enabled when num nodes > 1 +# train.rdma_enabled = False + + +# Base learning in MAE is set to 1.5e-4 +# The actually learning rate should be computed by linear scaling rule as follows: +# lr = base_lr * batch_size / 256 +# In LiBai, you should refine the actually learning rate due to your on settings +# Here we use 8 GPUs, 128 batch_size per GPU for training, batch_size equals to 1024 +base_lr = 1.5e-4 +actual_lr = base_lr * effective_batch_size / 256 + +# Refine optim settings +optim.params._target_ = param_groups_weight_decay +optim.params.weight_decay = 0.05 +optim.lr = actual_lr +optim.betas = (0.9, 0.95) + +del optim.params.clip_grad_max_norm +del optim.params.clip_grad_norm_type +del optim.params.weight_decay_norm +del optim.params.weight_decay_bias +del optim.weight_decay + +# Refine scheduler +# Default scheduler in LiBai training config is WarmupCosineLR +train.scheduler = LazyCall(warmup_cosine_lr_scheduler)( + warmup_factor=0.0, + min_lr=0.0, +) + + +# AMP +train.amp.enabled = True + + +# Distributed Settings +train.dist.data_parallel_size = n_gpus +train.dist.tensor_parallel_size = 1 +train.dist.pipeline_parallel_size = 1 +# train.dist.pipeline_num_layers = model.depth diff --git a/projects/MAE/configs/models/mae_vit_base_patch16.py b/projects/MAE/configs/models/mae_vit_base_patch16.py new file mode 100644 index 0000000000000000000000000000000000000000..4cb7ce2a1c89a5af10e9bc15e95ec1dea945c9a3 --- /dev/null +++ b/projects/MAE/configs/models/mae_vit_base_patch16.py @@ -0,0 +1,23 @@ +from functools import partial + +from libai.config import LazyCall +from libai.layers import LayerNorm + +from modeling.mae import MaskedAutoencoderViT + + +model = LazyCall(MaskedAutoencoderViT)( + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + depth=12, + num_heads=12, + decoder_embed_dim=512, + decoder_depth=8, + decoder_num_heads=16, + mlp_ratio=4, + norm_layer=partial(LayerNorm, eps=1e-6), + norm_pix_loss=True, + mask_ratio=0.75, +) diff --git a/projects/MAE/configs/models/mae_vit_huge_patch14.py b/projects/MAE/configs/models/mae_vit_huge_patch14.py new file mode 100644 index 0000000000000000000000000000000000000000..6e8c2c652733d0db540f058bc16379c232d85074 --- /dev/null +++ b/projects/MAE/configs/models/mae_vit_huge_patch14.py @@ -0,0 +1,7 @@ +from .mae_vit_base_patch16 import model + + +model.patch_size = 14 +model.embed_dim = 1280 +model.depth = 32 +model.num_heads = 16 diff --git a/projects/MAE/configs/models/mae_vit_large_patch16.py b/projects/MAE/configs/models/mae_vit_large_patch16.py new file mode 100644 index 0000000000000000000000000000000000000000..da4ed9bf7399c10496df9fc86b37b6b00a06a103 --- /dev/null +++ b/projects/MAE/configs/models/mae_vit_large_patch16.py @@ -0,0 +1,6 @@ +from .mae_vit_base_patch16 import model + + +model.embed_dim = 1024 +model.depth = 24 +model.num_heads = 16 diff --git a/projects/MAE/configs/models/vit_base_patch16.py b/projects/MAE/configs/models/vit_base_patch16.py new file mode 100644 index 0000000000000000000000000000000000000000..6fba67fec1c1d6b2b3aa4b246f46e505a13d474a --- /dev/null +++ b/projects/MAE/configs/models/vit_base_patch16.py @@ -0,0 +1,16 @@ +from libai.config import LazyCall + +from modeling.vit import VisionTransformer + + +model = LazyCall(VisionTransformer)( + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4, + drop_path_rate=0.1, + global_pool=True, +) diff --git a/projects/MAE/configs/models/vit_huge_patch14.py b/projects/MAE/configs/models/vit_huge_patch14.py new file mode 100644 index 0000000000000000000000000000000000000000..b07f72f379ee3919ce5fa55496d966af678cf453 --- /dev/null +++ b/projects/MAE/configs/models/vit_huge_patch14.py @@ -0,0 +1,8 @@ +from .vit_base_patch16 import model + + +model.patch_size = 14 +model.embed_dim = 1280 +model.depth = 32 +model.num_heads = 16 +model.drop_path_rate = 0.2 diff --git a/projects/MAE/configs/models/vit_large_patch16.py b/projects/MAE/configs/models/vit_large_patch16.py new file mode 100644 index 0000000000000000000000000000000000000000..9d22776676733990b4b1d53426b4c7ccd8c991a5 --- /dev/null +++ b/projects/MAE/configs/models/vit_large_patch16.py @@ -0,0 +1,6 @@ +from .vit_base_patch16 import model + + +model.embed_dim = 1024 +model.depth = 24 +model.num_heads = 16 diff --git a/projects/MAE/data/pretraining_imagenet.py b/projects/MAE/data/pretraining_imagenet.py new file mode 100644 index 0000000000000000000000000000000000000000..829368c5a9ad2b7cad5ea0c85bc032cbebde6bfe --- /dev/null +++ b/projects/MAE/data/pretraining_imagenet.py @@ -0,0 +1,29 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from libai.data.datasets.imagenet import ImageNetDataset +from libai.data.structures import Instance + + +class PretrainingImageNetDataset(ImageNetDataset): + """ImageNet Dataset in LiBai for Pretraining + Return: + images: ImageNet train set images + """ + + def __getitem__(self, index: int): + data_sample = super().__getitem__(index) + return Instance(images=data_sample.get("images")) diff --git a/projects/MAE/modeling/cross_entropy.py b/projects/MAE/modeling/cross_entropy.py new file mode 100644 index 0000000000000000000000000000000000000000..772c70862b112ed03d484324f916853152bd0774 --- /dev/null +++ b/projects/MAE/modeling/cross_entropy.py @@ -0,0 +1,33 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +import oneflow.nn as nn + + +class SoftTargetCrossEntropy(nn.Module): + def __init__(self): + super(SoftTargetCrossEntropy, self).__init__() + + def forward(self, x: flow.Tensor, target: flow.Tensor) -> flow.Tensor: + pred = flow.log_softmax(x, dim=-1) + loss = -target * pred + # sum and mean should be calculated with float32 + # amp_white_identity ensure -target * pred using float16 + # amp_black_identity ensure sum and mean using float32 + loss = flow._C.amp_white_identity(loss) + loss = flow._C.amp_black_identity(loss) + loss = flow.sum(loss, dim=-1) + return loss.mean() diff --git a/projects/MAE/modeling/mae.py b/projects/MAE/modeling/mae.py new file mode 100644 index 0000000000000000000000000000000000000000..c0ac7ca9993c465b7b5738d69750f1b0ebc7fc83 --- /dev/null +++ b/projects/MAE/modeling/mae.py @@ -0,0 +1,386 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# -------------------------------------------------------- +# MAE Model +# References: +# mae: https://github.com/facebookresearch/mae/blob/main/models_mae.py +# -------------------------------------------------------- + + +import oneflow as flow +import oneflow.nn as nn + +import libai.utils.distributed as dist +from libai.config import configurable +from libai.layers import LayerNorm, Linear, PatchEmbedding, TransformerLayer + +from .pos_embed import get_2d_sincos_pos_embed + + +class MaskedAutoencoderViT(nn.Module): + """Masked Autoencoder with VisionTransformer backbone""" + + @configurable + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=1024, + depth=24, + num_heads=16, + decoder_embed_dim=512, + decoder_depth=8, + decoder_num_heads=16, + mlp_ratio=4.0, + norm_layer=LayerNorm, + norm_pix_loss=False, + mask_ratio=0.75, + ): + super().__init__() + + self.mask_ratio = mask_ratio + # -------------------------------------------------------------------------- + # MAE encoder specifics + self.patch_embed = PatchEmbedding(img_size, patch_size, in_chans, embed_dim) + num_patches = self.patch_embed.num_patches + self.cls_token = nn.Parameter( + flow.zeros( + 1, + 1, + embed_dim, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + ) + self.pos_embed = nn.Parameter( + flow.zeros( + 1, + num_patches + 1, + embed_dim, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + ) + self.blocks = nn.ModuleList( + [ + TransformerLayer( + hidden_size=embed_dim, + ffn_hidden_size=int(embed_dim * mlp_ratio), + num_attention_heads=num_heads, + layer_idx=i, + ) + for i in range(depth) + ] + ) + # TODO: set norm layer placement stage id + self.norm = norm_layer(embed_dim, layer_idx=depth) + # -------------------------------------------------------------------------- + + # -------------------------------------------------------------------------- + # MAE decoder specifics + self.decoder_embed = Linear(embed_dim, decoder_embed_dim, bias=True, layer_idx=depth) + + self.mask_token = nn.Parameter( + flow.zeros( + 1, + 1, + decoder_embed_dim, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(depth), + ) + ) + + self.decoder_pos_embed = nn.Parameter( + flow.zeros( + 1, + num_patches + 1, + decoder_embed_dim, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(depth), + ) + ) + + self.decoder_blocks = nn.ModuleList( + [ + TransformerLayer( + hidden_size=decoder_embed_dim, + ffn_hidden_size=int(decoder_embed_dim * mlp_ratio), + num_attention_heads=decoder_num_heads, + layer_idx=(i + depth), + ) + for i in range(decoder_depth) + ] + ) + + self.decoder_norm = norm_layer(decoder_embed_dim, layer_idx=-1) + self.decoder_pred = Linear( + decoder_embed_dim, patch_size ** 2 * in_chans, bias=True, layer_idx=-1 + ) # decoder to patch + # -------------------------------------------------------------------------- + + self.norm_pix_loss = norm_pix_loss + + self.initialize_weights() + + def initialize_weights(self): + # initialization + # initialize (and freeze) pos_embed by sin-cos embedding + pos_embed = get_2d_sincos_pos_embed( + self.pos_embed.shape[-1], int(self.patch_embed.num_patches ** 0.5), cls_token=True + ) + self.pos_embed.data.copy_( + flow.from_numpy(pos_embed) + .float() + .unsqueeze(0) + .to_global( + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=self.pos_embed.placement, + ) + ) + + decoder_pos_embed = get_2d_sincos_pos_embed( + self.decoder_pos_embed.shape[-1], + int(self.patch_embed.num_patches ** 0.5), + cls_token=True, + ) + self.decoder_pos_embed.data.copy_( + flow.from_numpy(decoder_pos_embed) + .float() + .unsqueeze(0) + .to_global( + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=self.decoder_pos_embed.placement, + ) + ) + + # initialize patch_embed like nn.Linear (instead of nn.Conv2d) + w = self.patch_embed.proj.weight.data + flow.nn.init.xavier_uniform_(w.view([w.shape[0], -1])) + + # timm's trunc_normal_(std=.02) is effectively normal_(std=0.02) as cutoff is too big (2.) + flow.nn.init.normal_(self.cls_token, std=0.02) + flow.nn.init.normal_(self.mask_token, std=0.02) + + # initialize nn.Linear and nn.LayerNorm + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, Linear): + # we use xavier_uniform following official JAX ViT: + flow.nn.init.xavier_uniform_(m.weight) + if isinstance(m, Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @classmethod + def from_config(cls, cfg): + return { + "img_size": cfg.img_size, + "patch_size": cfg.patch_size, + "in_chans": cfg.in_chans, + "embed_dim": cfg.embed_dim, + "depth": cfg.depth, + "num_heads": cfg.num_heads, + "decoder_embed_dim": cfg.decoder_embed_dim, + "decoder_depth": cfg.decoder_depth, + "decoder_num_heads": cfg.decoder_num_heads, + "mlp_ratio": cfg.mlp_ratio, + "norm_layer": cfg.norm_layer, + "norm_pix_loss": cfg.norm_pix_loss, + "mask_ratio": cfg.mask_ratio, + } + + def patchify(self, imgs): + """ + imgs: (N, 3, H, W) + x: (N, L, patch_size**2 *3) + """ + p = self.patch_embed.patch_size[0] + assert imgs.shape[2] == imgs.shape[3] and imgs.shape[2] % p == 0 + + h = w = imgs.shape[2] // p + x = imgs.reshape(imgs.shape[0], 3, h, p, w, p) + # TODO: replace permute with flow.einsum + # (n c h p w q) -> (n h w p q c) + x = x.permute(0, 2, 4, 3, 5, 1) + x = x.reshape(imgs.shape[0], h * w, p ** 2 * 3) + return x + + def unpatchify(self, x): + """ + x: (N, L, patch_size**2 *3) + imgs: (N, 3, H, W) + """ + p = self.patch_embed.patch_size[0] + h = w = int(x.shape[1] ** 0.5) + assert h * w == x.shape[1] + + x = x.reshape(x.shape[0], h, w, p, p, 3) + # TODO: replace permute with flow.einsum + # (n h w p q c) -> (n c h p w q) + x = x.permute(0, 5, 1, 3, 2, 4) + imgs = x.reshape(x.shape[0], 3, h * p, h * p) + return imgs + + def random_masking(self, x, mask_ratio): + """ + Perform per-sample random masking by per-sample shuffling. + Per-sample shuffling is done by argsort random noise. + x: [N, L, D], sequence + """ + N, L, D = x.shape + len_keep = int(L * (1 - mask_ratio)) + + noise = flow.rand(N, L, sbp=x.sbp, placement=x.placement) # noise in [0, 1] + + # sort noise for each sample + ids_shuffle = flow.argsort(noise, dim=1) # ascend: small is keep, large is remove + ids_restore = flow.argsort(ids_shuffle, dim=1) + + # keep the first subset + ids_keep = ids_shuffle[:, :len_keep] + x_masked = flow.gather(x, dim=1, index=ids_keep.unsqueeze(-1).repeat(1, 1, D)) + + # generate the binary mask: 0 is keep, 1 is remove + mask = flow.ones([N, L], sbp=x.sbp, placement=x.placement) + mask[:, :len_keep] = 0 + + # unshuffle to get binary mask + mask = flow.gather(mask, dim=1, index=ids_restore) + + return x_masked, mask, ids_restore + + def forward_encoder(self, x, mask_ratio): + # embed patches + x = self.patch_embed(x) + + # add pos embed w/o cls token + x = x + self.pos_embed[:, 1:, :] + + # masking: length -> length * mask_ratio + x, mask, ids_restore = self.random_masking(x, mask_ratio) + + # append cls token + cls_token = self.cls_token + self.pos_embed[:, :1, :] + # Directly expanding cls_token (with shape=(1, 1, D) and sbp=B) + # will produce a huge tensor with shape [B*N, 1, D] + # (while B = local batch size, N = total num devices), + # however we only need an expanded cls_token with shape [B, 1, D], + # meanwhile local to global tensor is not avaible in graph mode for now, + # we have to use a two stage expanding way to expand cls_token as below. + world_size = flow.env.get_world_size() + # repeat to (N, 1, D), sbp = B + cls_token = cls_token.expand(world_size, -1, -1) + # to_global(sbp=S(0)), local shape = (1, 1, D) + cls_token = cls_token.to_global(sbp=x.sbp) + # second expand from (N, 1, D) to (B*N, 1, D) + # (global shape, sbp=S(0)), local shape=(B, 1, D), + # by this way we wouldn't produce a (B*N, 1, D) tensor in local view. + cls_tokens = cls_token.repeat(x.shape[0] // world_size, 1, 1) + x = flow.cat((cls_tokens, x), dim=1) + + # apply Transformer blocks + for blk in self.blocks: + x = blk(x) + x = self.norm(x) + + return x, mask, ids_restore + + def forward_decoder(self, x, ids_restore): + # embed tokens + x = self.decoder_embed(x) + + # append mask tokens to sequence + # mask_tokens = self.mask_token.repeat(x.shape[0], ids_restore.shape[1] + 1 - x.shape[1], 1) + # The line above will produce a huge mask_tokens with shape [B*N, L, D] + # (while B = local batch size, N = total num devices), + # actually we only need a mask_tokens with shape [B, L, D] in local view, + # meanwhile local to global tensor is not avaible in graph mode for now, + # we have to use a two stage repeat way as below. + world_size = flow.env.get_world_size() + # repeat to (N, 1, D), sbp = B + mask_token = self.mask_token.repeat(world_size, 1, 1) + # to_global(sbp=S(0)), local shape = (1, 1, D) + mask_token = mask_token.to_global(sbp=x.sbp) + # second repeat from (N, 1, D) to (B*N, L, D) + # (global shape, sbp=S(0)), local shape = (B, L, D), + # and the originally huge mask_tokens with shape (B*N, L, D) + # wouldn't be produced in local view. + mask_tokens = mask_token.repeat( + x.shape[0] // world_size, ids_restore.shape[1] + 1 - x.shape[1], 1 + ) + + x_ = flow.cat([x[:, 1:, :], mask_tokens], dim=1) # no cls token + x_ = flow.gather( + x_, dim=1, index=ids_restore.unsqueeze(-1).repeat(1, 1, x.shape[2]) + ) # unshuffle + x = flow.cat([x[:, :1, :], x_], dim=1) # append cls token + + # add pos embed + x = x + self.decoder_pos_embed + + # apply Transformer blocks + for blk in self.decoder_blocks: + x = blk(x) + x = self.decoder_norm(x) + + # predictor projection + x = self.decoder_pred(x) + + # remove cls token + x = x[:, 1:, :] + + return x + + def forward_loss(self, imgs, pred, mask): + """ + imgs: [N, 3, H, W] + pred: [N, L, p*p*3] + mask: [N, L], 0 is keep, 1 is remove, + """ + target = self.patchify(imgs) + if self.norm_pix_loss: + mean = target.mean(dim=-1, keepdim=True) + var = target.var(dim=-1, keepdim=True) + target = (target - mean) / (var + 1.0e-6) ** 0.5 + + loss = (pred - target) ** 2 + # We want the prev loss to be calculated with float16, + # and mean/sum below to be calculated with float32. + # this amp_white_identity will affect preceding ops to be float16 + loss = flow._C.amp_white_identity(loss) + # this amp_black_identity will affect succeeding ops to be float32 + loss = flow._C.amp_black_identity(loss) + loss = loss.mean(dim=-1) # [N, L], mean loss per patch + loss = (loss * mask).sum() / mask.sum() # mean loss on removed patches + return loss + + def forward(self, images): + latent, mask, ids_restore = self.forward_encoder(images, self.mask_ratio) + pred = self.forward_decoder(latent, ids_restore) # [N, L, p*p*3] + loss = self.forward_loss(images, pred, mask) + if self.training: + return {"losses": loss} + else: + return { + "losses": loss, + "pred": pred, + "mask": mask, + } diff --git a/projects/MAE/modeling/pos_embed.py b/projects/MAE/modeling/pos_embed.py new file mode 100644 index 0000000000000000000000000000000000000000..3db82b4c07170fbdb7f4ea68b19c63f9606eae6d --- /dev/null +++ b/projects/MAE/modeling/pos_embed.py @@ -0,0 +1,77 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np + +# -------------------------------------------------------- +# 2D sine-cosine position embedding +# References: +# MoCo v3: https://github.com/facebookresearch/moco-v3 +# -------------------------------------------------------- + + +def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False): + """ + Arguments: + embed_dim: hidden_size of the input tokens + grid_size: int of the grid height and width + cls_token: with cls_token or not + + Return: + pos_embed: [grid_size*grid_size, embed_dim] + or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token) + """ + grid_h = np.arange(grid_size, dtype=np.float32) + grid_w = np.arange(grid_size, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) # here w goes first + grid = np.stack(grid, axis=0) + + grid = grid.reshape([2, 1, grid_size, grid_size]) + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) + if cls_token: + pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0) + return pos_embed + + +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): + assert embed_dim % 2 == 0 + + # use half of dimensions to encode grid_h + emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2) + emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2) + + emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) + return emb + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000 ** omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb diff --git a/projects/MAE/modeling/vit.py b/projects/MAE/modeling/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..61e161934a3c3bc519e4b980d9ebfc512c612e5f --- /dev/null +++ b/projects/MAE/modeling/vit.py @@ -0,0 +1,86 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# -------------------------------------------------------- +# ViT Model +# References: +# mae: https://github.com/facebookresearch/mae/blob/main/models_vit.py +# -------------------------------------------------------- + +import oneflow as flow + +import libai.models.vision_transformer + + +class VisionTransformer(libai.models.vision_transformer.VisionTransformer): + """Vision Transformer for MAE + LiBai impl of: `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` + - https://arxiv.org/abs/2010.11929 + """ + + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4.0, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.0, + global_pool=False, + num_classes=1000, + loss_func=None, + ): + super(VisionTransformer, self).__init__( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + depth=depth, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + drop_rate=drop_rate, + attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rate, + num_classes=num_classes, + loss_func=loss_func, + ) + self.global_pool = global_pool + + def no_weight_decay(self): + return {"pos_embed", "cls_token"} + + def forward_head(self, x): + if self.global_pool: + x = x[:, 1:, :] # global pool without cls token + # we want mean to be calculated with float32 + # the amp_white_identity pair make the calculation before and after mean using float16 + # the amp_black_identity pair make mean using float32 + x = flow._C.amp_white_identity(x) + x = flow._C.amp_black_identity(x) + x = x.mean(dim=1) + x = flow._C.amp_black_identity(x) + x = flow._C.amp_white_identity(x) + outcome = self.norm(x) + outcome = self.head(outcome) + else: + x = self.norm(x) + outcome = x[:, 0] + outcome = self.head(outcome) + return outcome diff --git a/projects/MAE/train_net.py b/projects/MAE/train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..69729cb472dd2f0cbf6cfc5dc10b76c55a82ab6f --- /dev/null +++ b/projects/MAE/train_net.py @@ -0,0 +1,93 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import random +import sys + +import numpy as np +import oneflow as flow +from utils.weight_convert import load_torch_checkpoint + +from libai.config import LazyConfig, default_argument_parser, try_get_key +from libai.engine import DefaultTrainer, default_setup +from libai.utils.checkpoint import Checkpointer + +sys.path.append(".") +logger = logging.getLogger("libai.mae." + __name__) + + +class Trainer(DefaultTrainer): + @classmethod + def build_model(cls, cfg): + model = super().build_model(cfg) + if try_get_key(cfg, "finetune") is not None: + if cfg.finetune.enable is True: + logger.info("Loading pretrained weight for finetuning") + assert cfg.finetune.weight_style in ["oneflow", "pytorch"] + if cfg.finetune.weight_style == "oneflow": + Checkpointer(model).load(cfg.finetune.path) + elif cfg.finetune.weight_style == "pytorch": + model = load_torch_checkpoint(model, cfg, path=cfg.finetune.path, strict=False) + else: + raise NotImplementedError( + "Only support loading oneflow & pytorch pretrained weight now." + ) + return model + + +def main(args): + cfg = LazyConfig.load(args.config_file) + cfg = LazyConfig.apply_overrides(cfg, args.opts) + default_setup(cfg, args) + + if args.fast_dev_run: + cfg.train.train_epoch = 0 + cfg.train.checkpointer.period = 5 + cfg.train.train_iter = 10 + cfg.train.evaluation.eval_period = 10 + cfg.train.log_period = 1 + + if args.eval_only: + cfg.eval_only = True + tokenizer = None + if try_get_key(cfg, "tokenization.setup", default=False): + tokenizer = Trainer.build_tokenizer(cfg) + model = Trainer.build_model(cfg) + Checkpointer(model, save_dir=cfg.train.output_dir).resume_or_load( + cfg.train.load_weight, resume=args.resume + ) + if try_get_key(cfg, "train.graph.enabled", default=False): + model = Trainer.build_graph(cfg, model, is_train=False) + test_loader = Trainer.build_test_loader(cfg, tokenizer) + if len(test_loader) == 0: + logger.info("No dataset in dataloader.test, please set dataset for dataloader.test") + _ = Trainer.test(cfg, test_loader, model) + return + + # manual different seed for each rank + seed_for_rank = cfg.train.seed + flow.env.get_rank() + flow.manual_seed(seed_for_rank) + flow.cuda.manual_seed(seed_for_rank) + np.random.seed(seed_for_rank) + random.seed(seed_for_rank) + + trainer = Trainer(cfg) + return trainer.train() + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + main(args) diff --git a/projects/MAE/utils/lr_decay.py b/projects/MAE/utils/lr_decay.py new file mode 100644 index 0000000000000000000000000000000000000000..95ab2c9d0681b0b42a88b33de6b5859199d250a9 --- /dev/null +++ b/projects/MAE/utils/lr_decay.py @@ -0,0 +1,108 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# -------------------------------------------------------- +# References: +# mae: https://github.com/facebookresearch/mae/blob/main/util/lr_decay.py +# -------------------------------------------------------- +import logging + +logger = logging.getLogger("libai.mae." + __name__) + + +def param_groups_lrd(model, weight_decay=0.05, layer_decay=0.75): + """ + Parameter groups for layer-wise lr decay + Modified from BEiT: https://github.com/microsoft/unilm/blob/master/beit/optim_factory.py#L58 + """ + param_group_names = {} + param_groups = {} + no_weight_decay_list = model.no_weight_decay() + num_layers = len(model.blocks) + 1 + layer_scales = list(layer_decay ** (num_layers - i) for i in range(num_layers + 1)) + + for name, param in model.named_parameters(): + if not param.requires_grad: + continue + + if param.ndim == 1 or name in no_weight_decay_list: + g_decay = "no_decay" + this_decay = 0.0 + else: + g_decay = "decay" + this_decay = weight_decay + + layer_idx = get_layer_idx_for_vit(name, num_layers) + group_name = "layer_%d_%s" % (layer_idx, g_decay) + + # logger.info( + # f"{name}, shape={param.shape}, {g_decay}={this_decay}" + # f", layer_scale={layer_scales[layer_idx]}" + # ) + + if group_name not in param_group_names: + this_scale = layer_scales[layer_idx] + + param_group_names[group_name] = { + "lr_scale": this_scale, + "weight_decay": this_decay, + "params": [], + } + param_groups[group_name] = { + "lr_scale": this_scale, + "weight_decay": this_decay, + "params": [], + } + param_group_names[group_name]["params"].append(name) + param_groups[group_name]["params"].append(param) + + return list(param_groups.values()) + + +def get_layer_idx_for_vit(name, num_layers): + """ + Assign a parameter with its layer id + Following BEiT: https://github.com/microsoft/unilm/blob/master/beit/optim_factory.py#L33 + """ + if name in ["cls_token", "pos_embed"]: + return 0 + elif name.startswith("patch_embed"): + return 0 + elif name.startswith("blocks"): + return int(name.split(".")[1]) + 1 + else: + return num_layers + + +# Refer to: add_weight_decay in +# https://github.com/rwightman/pytorch-image-models/blob/v0.3.3/timm/optim/optim_factory.py +def param_groups_weight_decay(model, weight_decay=1e-5, skip_list=()): + decay_params = [] + no_decay_params = [] + + for name, param in model.named_parameters(): + if not param.requires_grad: + continue # frozen weights + + if len(param.shape) == 1 or name.endswith(".bias") or name in skip_list: + no_decay_params.append(param) + else: + decay_params.append(param) + + return [ + {"params": no_decay_params, "weight_decay": 0.0}, + {"params": decay_params, "weight_decay": weight_decay}, + ] diff --git a/projects/MAE/utils/scheduler.py b/projects/MAE/utils/scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..af2ffdc2a0314cefc8f2fcd7456c36d6a78e7cf7 --- /dev/null +++ b/projects/MAE/utils/scheduler.py @@ -0,0 +1,108 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import math + +import oneflow as flow +from oneflow.optim.lr_scheduler import _LRScheduler + +logger = logging.getLogger(__name__) + + +class LayerScaleWarmupCosineDecayLR(_LRScheduler): + def __init__( + self, + optimizer: flow.optim.Optimizer, + steps: int, + warmup_steps: int, + warmup_factor: float, + min_lr: float = 0.0, + last_step: int = -1, + verbose: bool = False, + ): + self.total_steps = steps + self.decay_steps = steps - warmup_steps + self.warmup_steps = warmup_steps + self.warmup_factor = warmup_factor + self.min_lr = min_lr + super().__init__(optimizer, last_step, verbose) + + def get_lr(self, base_lr, step): + if step < self.warmup_steps: + progress = step / self.warmup_steps + lr = base_lr * progress + elif step < self.total_steps: + progress = (step - self.warmup_steps) / self.decay_steps + lr = self.min_lr + (base_lr - self.min_lr) * 0.5 * (1.0 + math.cos(math.pi * progress)) + else: + lr = self.min_lr + + return lr + + def update_lrs(self, lrs): + self._last_lr = [] + for i, (group, lr) in enumerate(zip(self.optimizer.param_groups, lrs)): + if "lr_scale" in group: + group["lr"] = lr * group["lr_scale"] + else: + group["lr"] = lr + + self._last_lr.append(lr) + if self.verbose: + self.print_lr(i, lr) + + +def warmup_layerscale_cosine_lr_scheduler( + optimizer: flow.optim.Optimizer, + max_iter: int, + warmup_iter: int, + warmup_factor: float, + min_lr: float = 0.0, +): + return LayerScaleWarmupCosineDecayLR( + optimizer, + steps=max_iter, + warmup_steps=warmup_iter, + warmup_factor=warmup_factor, + min_lr=min_lr, + ) + + +def warmup_cosine_lr_scheduler( + optimizer: flow.optim.Optimizer, + max_iter: int, + warmup_iter: int, + warmup_factor: float = 0.0, + warmup_method: str = "linear", + min_lr: float = 0.0, +): + cosine_lr = flow.optim.lr_scheduler.CosineAnnealingLR( + optimizer, T_max=max_iter - warmup_iter, eta_min=min_lr + ) + if warmup_iter == 0: + logger.warning("warmup iters equals to zero, return CosineLR") + return cosine_lr + + if warmup_iter > max_iter: + logger.warning("warmup iters is larger than the total training iters") + + warmup_cosine_lr = flow.optim.lr_scheduler.WarmupLR( + cosine_lr, + warmup_factor=warmup_factor, + warmup_iters=warmup_iter, + warmup_method=warmup_method, + ) + return warmup_cosine_lr diff --git a/projects/MAE/utils/weight_convert.py b/projects/MAE/utils/weight_convert.py new file mode 100644 index 0000000000000000000000000000000000000000..f8be544b2dbe9853cf26b1fa7cb5032048bdd6c3 --- /dev/null +++ b/projects/MAE/utils/weight_convert.py @@ -0,0 +1,123 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import oneflow as flow +import torch +from flowvision.layers.weight_init import trunc_normal_ + +logger = logging.getLogger("libai.mae." + __name__) + + +def convert_qkv_weight(cfg, value): + """ + Convert qkv.weight to be compatible with LiBai transformer layer + + Args: + cfg: config file + value: qkv.weight in the loaded checkpoint + """ + num_heads = cfg.model.num_heads + hidden_size = cfg.model.embed_dim + head_size = int(hidden_size / num_heads) + qkv_weight = ( + value.view([3, num_heads, head_size, hidden_size]) + .permute(1, 0, 2, 3) + .contiguous() + .view(hidden_size * 3, hidden_size) + ) + return qkv_weight + + +def convert_qkv_bias(cfg, value): + """ + Convert qkv.bias to be compatible with LiBai transformer layer + + Args: + cfg: config file + value: qkv.bias in the loaded checkpoint + """ + num_heads = cfg.model.num_heads + hidden_size = cfg.model.embed_dim + head_size = int(hidden_size / num_heads) + qkv_bias = ( + value.view(3, num_heads, head_size).permute(1, 0, 2).contiguous().view(hidden_size * 3) + ) + return qkv_bias + + +def filter_keys(key, value, cfg): + """ + Filtering the state_dict keys and values to match LiBai's MAE model + """ + if key.startswith("decoder_"): + value = None + elif "norm1" in key: + key = key.replace("norm1", "input_layernorm") + elif "attn.qkv" in key: + key = key.replace("attn.qkv", "self_attention.query_key_value") + if "weight" in key: + value = convert_qkv_weight(cfg, value) + if "bias" in key: + value = convert_qkv_bias(cfg, value) + elif "attn.proj" in key: + key = key.replace("attn.proj", "self_attention.dense") + elif "norm2" in key: + key = key.replace("norm2", "post_attention_layernorm") + elif "mlp.fc1" in key: + key = key.replace("mlp.fc1", "mlp.dense_h_to_4h") + elif "mlp.fc2" in key: + key = key.replace("mlp.fc2", "mlp.dense_4h_to_h") + elif "fc_norm" in key: + key = key.replace("fc_norm", "norm") + elif key == "norm.weight" or key == "norm.bias": + value = None + + return key, value + + +def log_param(key, value): + logger.info(f"{key}, shape={value.shape}") + + +def load_torch_checkpoint(model, cfg, path="./mae_finetuned_vit_base.pth", strict=False): + """ + Load checkpoint from the given torch weights. + Torch weight can be downloaded from the original repo: + https://github.com/facebookresearch/mae + """ + torch_dict = torch.load(path, map_location="cpu")["model"] + parameters = torch_dict + new_parameters = dict() + for key, value in parameters.items(): + # log_param(key, value) + if "num_batches_tracked" not in key: + # to global tensor + key, val = filter_keys(key, value, cfg) + if val is None: + continue + val = val.detach().cpu().numpy() + val = flow.tensor(val).to_global( + sbp=flow.sbp.broadcast, placement=flow.placement("cuda", ranks=[0]) + ) + new_parameters[key] = val + + msg = model.load_state_dict(new_parameters, strict=strict) + logger.info(msg) + if not cfg.eval_only: + trunc_normal_(model.head.weight, std=2e-5) + logger.info("Successfully load torch mae checkpoint.") + return model diff --git a/projects/MOCOV3/README.md b/projects/MOCOV3/README.md new file mode 100644 index 0000000000000000000000000000000000000000..93c7aba60c1afe234cd8032ee5a2c6529714e88a --- /dev/null +++ b/projects/MOCOV3/README.md @@ -0,0 +1,111 @@ +## MOCOv3 in LiBai +**An Empirical Study of Training Self-Supervised Vision Transformers** + +Xinlei Chen, Saining Xie, Kaiming He + +[[`arXiv`](https://arxiv.org/abs/2104.02057)] [[`BibTeX`](#Citation)] + +

+ +

+ +This is the OneFlow re-implementation of MOCOv3 based on [LiBai](https://libai.readthedocs.io/). + +## Catelog +- [x] MOCOv3 pretraining code +- [x] MOCOv3 linear prob code + +## Supported parallel mode and task +Based on [libai.layers](https://libai.readthedocs.io/en/latest/modules/libai.layers.html), MOCOv3 model is automatically configured with the following parallelism mode. + + + + + + + + + + + + + + + + + + + + + + +
Model Data ParallelTensor ParallelPipeline Parallel
MOCOv3 pretrain ✔--
MOCOv3 linear prob ✔✔✔
+ + +## Usage +### Installation +Please see [LiBai Installation](https://libai.readthedocs.io/en/latest/tutorials/get_started/Installation.html) to install LiBai + +### Prepare the Data +Please see [Prepare the Data](https://libai.readthedocs.io/en/latest/tutorials/get_started/quick_run.html#prepare-the-data). + + +### Pretraining +Pretraining MOCOv3 on 8 GPUs using data parallelism. +```bash +cd /path/to/libai +bash tools/train.sh projects/MOCOV3/pretrain_net.py projects/MOCOV3/configs/moco_pretrain.py 8 +``` + +### Linear Prob +1. Setup the weights for linear prob in [moco_linear_prob.py](./configs/moco_linear_prob.py) as follows: + +```python +# moco_linear_prob.py +# Path to the weight for linear prob +model.linear_prob = "path/to/pretrained_weight" +model.weight_style = "oneflow" +``` +If you feel confused about the checkpoint format here, please refer to [Load and Save a Checkpoint in LiBai](https://libai.readthedocs.io/en/latest/tutorials/basics/Load_and_Save_Checkpoint.html) for more details. + +2. The MOCOv3 linear prob on 8 GPUs using data parallelism. +```bash +cd /path/to/libai +bash tools/train.sh tools/train_net.py projects/MOCOV3/configs/moco_linear_prob.py 8 +``` +**Notes:** if you want to run the MOCOv3 linear prob models using different parallel strategies, please refer to the [Distributed Configuration Tutorial](https://libai.readthedocs.io/en/latest/tutorials/basics/Distributed_Configuration.html) + + +### Evaluation +Evaluate MOCOv3 model under LiBai on 8 GPUs: +```bash +cd /path/to/libai +bash tools/train.sh tools/train_net.py projects/MOCOV3/configs/moco_linear_prob.py 8 --eval-only train.load_weight="path/to/pretrained_weight" +``` + + +## Advanced Usage +### The MOCOv3 linear prob with pytorch pretrained checkpoint +You can download pytorch pretrained weight from [MOCOv3 official repo](https://github.com/facebookresearch/moco-v3/blob/main/CONFIG.md) and run linear prob in LiBai by updating the [moco_linear_prob.py](./configs/moco_linear_prob.py) as follows: +```python +# Path to the weight for linear prob +model.linear_prob = "/path/to/vit-s-300ep.pth.tar" +model.weight_style = "pytorch" +``` +Run linear prob on 8 GPUs: +```bash +cd /path/to/libai +bash tools/train.sh tools/train_net.py projects/MOCOV3/configs/moco_linear_prob.py 8 +``` + + +## Citation +```BibTeX +@inproceedings{chen2021empirical, + title={An empirical study of training self-supervised vision transformers}, + author={Chen, Xinlei and Xie, Saining and He, Kaiming}, + booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision}, + pages={9640--9649}, + year={2021} +} +``` \ No newline at end of file diff --git a/projects/MOCOV3/configs/moco_linear_prob.py b/projects/MOCOV3/configs/moco_linear_prob.py new file mode 100644 index 0000000000000000000000000000000000000000..84e03e9d80fad317b964e0685f2dcad4d6b377cf --- /dev/null +++ b/projects/MOCOV3/configs/moco_linear_prob.py @@ -0,0 +1,56 @@ +from oneflow.optim import SGD +from flowvision.transforms import transforms + +from libai.config import get_config, LazyCall +from .models.vit_small_patch16 import model +from ..transform.linear_prob_transform import train_aug + + +dataloader = get_config("common/data/imagenet.py").dataloader +train = get_config("common/train.py").train +graph = get_config("common/models/graph.py").graph +optim = get_config("common/optim.py").optim + +# Path to the weight for fine-tune +model.linear_prob = "path/to/pretrained_weight" +model.weight_style = "oneflow" + +# Refine data path to imagenet +dataloader.train.dataset[0].root = "/path/to/imagenet/" +dataloader.test[0].dataset.root = "/path/to/imagenet/" + +# Add augmentation Func +dataloader.train.dataset[0].transform = LazyCall(transforms.Compose)(transforms=train_aug) + +# Refine train cfg for moco v3 model +train.train_micro_batch_size = 128 +train.test_micro_batch_size = 32 +train.train_epoch = 90 +train.log_period = 1 +train.evaluation.eval_period = 1000 + +optim._target_ = SGD +optim.params.clip_grad_max_norm = None +optim.params.clip_grad_norm_type = None +optim.params.weight_decay_norm = None +optim.params.weight_decay_bias = None + +del optim.betas +del optim.eps +del optim.do_bias_correction + +# Refine optimizer cfg for moco v3 model +# Reference: +# https://github.com/facebookresearch/moco-v3/blob/main/CONFIG.md +# https://github.com/facebookresearch/moco-v3/blob/main/main_lincls.py +base_lr = 3.0 +actual_lr = base_lr * (train.train_micro_batch_size * 8 / 256) +optim.lr = actual_lr +optim.weight_decay = 0.0 +optim.momentum = 0.9 + +# Scheduler +train.scheduler.warmup_iter = 0 +train.scheduler.alpha = 0 + +graph.enabled = False diff --git a/projects/MOCOV3/configs/moco_pretrain.py b/projects/MOCOV3/configs/moco_pretrain.py new file mode 100644 index 0000000000000000000000000000000000000000..a1861811a6e367b7ef21cd81a733ddd604043b15 --- /dev/null +++ b/projects/MOCOV3/configs/moco_pretrain.py @@ -0,0 +1,49 @@ +from flowvision import transforms + +from libai.config import get_config, LazyCall + +from .models.moco_vit_small_patch16 import model +from transform.pretrain_transform import TwoCropsTransform, augmentation1, augmentation2 + + +dataloader = get_config("common/data/imagenet.py").dataloader +train = get_config("common/train.py").train +graph = get_config("common/models/graph.py").graph +optim = get_config("common/optim.py").optim + +# Refine data path to imagenet +dataloader.train.dataset[0].root = "/path/to/imagenet/" +dataloader.test[0].dataset.root = "/path/to/imagenet/" + +# Add augmentation Func +dataloader.train.dataset[0].transform = LazyCall(TwoCropsTransform)( + base_transform1=LazyCall(transforms.Compose)(transforms=augmentation1), + base_transform2=LazyCall(transforms.Compose)(transforms=augmentation2), +) + +# the momentum of MOCOV3 +model.m = 0.99 + +# the temperature coefficient of MOCOV3 +model.T = 0.2 + +# Refine train cfg for moco v3 model +train.train_micro_batch_size = 32 +train.test_micro_batch_size = 32 +train.train_epoch = 300 +train.warmup_ratio = 40 / 300 +train.eval_period = 5 +train.log_period = 1 + +# Refine optimizer cfg for moco v3 model +base_lr = 1.5e-4 +actual_lr = base_lr * (train.train_micro_batch_size * 8 / 256) +optim.lr = actual_lr +optim.weight_decay = 0.1 + +# Scheduler +train.scheduler.warmup_factor = 0.001 +train.scheduler.alpha = 1.5e-4 +train.scheduler.warmup_method = "linear" + +graph.enabled = False diff --git a/projects/MOCOV3/configs/models/moco_vit_base_patch16.py b/projects/MOCOV3/configs/models/moco_vit_base_patch16.py new file mode 100644 index 0000000000000000000000000000000000000000..621564746b18a8b043ea865037a8b09acf906b3d --- /dev/null +++ b/projects/MOCOV3/configs/models/moco_vit_base_patch16.py @@ -0,0 +1,40 @@ +from libai.config import LazyCall +from modeling.moco import MoCo_ViT +from modeling.vit import VisionTransformer + + +base_encoder = LazyCall(VisionTransformer)( + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4, + drop_path_rate=0.1, + global_pool=False, + stop_grad_conv1=True, +) + +momentum_encoder = LazyCall(VisionTransformer)( + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4, + drop_path_rate=0.1, + global_pool=False, + stop_grad_conv1=True, +) + + +model = LazyCall(MoCo_ViT)( + base_encoder=base_encoder, + momentum_encoder=momentum_encoder, + dim=256, + mlp_dim=4096, + T=0.2, + m=0.99, +) diff --git a/projects/MOCOV3/configs/models/moco_vit_small_patch16.py b/projects/MOCOV3/configs/models/moco_vit_small_patch16.py new file mode 100644 index 0000000000000000000000000000000000000000..4162be502dcfa033367e44c15d1912cc36dc1607 --- /dev/null +++ b/projects/MOCOV3/configs/models/moco_vit_small_patch16.py @@ -0,0 +1,40 @@ +from libai.config import LazyCall +from modeling.moco import MoCo_ViT +from modeling.vit import VisionTransformer + + +base_encoder = LazyCall(VisionTransformer)( + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=384, + depth=12, + num_heads=12, + mlp_ratio=4, + drop_path_rate=0.0, + global_pool=False, + stop_grad_conv1=True, +) + +momentum_encoder = LazyCall(VisionTransformer)( + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=384, + depth=12, + num_heads=12, + mlp_ratio=4, + drop_path_rate=0.0, + global_pool=False, + stop_grad_conv1=True, +) + + +model = LazyCall(MoCo_ViT)( + base_encoder=base_encoder, + momentum_encoder=momentum_encoder, + dim=256, + mlp_dim=4096, + T=0.2, + m=0.99, +) diff --git a/projects/MOCOV3/configs/models/vit_base_patch16.py b/projects/MOCOV3/configs/models/vit_base_patch16.py new file mode 100644 index 0000000000000000000000000000000000000000..1352dd690edd5f45476078cf8a324f3cda8b1e7a --- /dev/null +++ b/projects/MOCOV3/configs/models/vit_base_patch16.py @@ -0,0 +1,19 @@ +import sys + +sys.path.append("projects/MOCOV3") + +from libai.config import LazyCall # noqa: E402 +from modeling.vit import VisionTransformer # noqa: E402 + + +model = LazyCall(VisionTransformer)( + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4, + drop_path_rate=0.1, + global_pool=False, +) diff --git a/projects/MOCOV3/configs/models/vit_small_patch16.py b/projects/MOCOV3/configs/models/vit_small_patch16.py new file mode 100644 index 0000000000000000000000000000000000000000..a0bd5352098762457ddc03b6ceb9879ae7c02447 --- /dev/null +++ b/projects/MOCOV3/configs/models/vit_small_patch16.py @@ -0,0 +1,7 @@ +from .vit_base_patch16 import model + + +model.embed_dim = 384 +model.depth = 12 +model.num_heads = 12 +model.drop_path_rate = 0.0 diff --git a/projects/MOCOV3/modeling/moco.py b/projects/MOCOV3/modeling/moco.py new file mode 100644 index 0000000000000000000000000000000000000000..7d644c1b2f9998a89b317f7b44eb77896ed27e7e --- /dev/null +++ b/projects/MOCOV3/modeling/moco.py @@ -0,0 +1,154 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# -------------------------------------------------------- +# MoCo v3 Model +# References: +# moco-v3: https://github.com/facebookresearch/moco-v3/blob/main/moco/builder.py +# -------------------------------------------------------- + + +import math + +import oneflow as flow +import oneflow.nn as nn + +from libai.layers import Linear +from libai.utils.distributed import get_world_size + + +class MoCo(nn.Module): + """ + Build a MoCo model with a base encoder, a momentum encoder, and two MLPs + https://arxiv.org/abs/1911.05722 + """ + + def __init__( + self, base_encoder, momentum_encoder, dim=256, mlp_dim=4096, T=1.0, m=0.99, max_iter=300 + ): + """ + dim: feature dimension (default: 256) + mlp_dim: hidden dimension in MLPs (default: 4096) + T: softmax temperature (default: 1.0) + """ + super(MoCo, self).__init__() + + self.T = T + self.m = m + # build encoders + self.base_encoder = base_encoder + self.momentum_encoder = momentum_encoder + self.base_encoder.num_classes = dim + self.momentum_encoder.num_classes = dim + self.max_iter = max_iter + + self._build_projector_and_predictor_mlps(dim, mlp_dim) + + for param_b, param_m in zip( + self.base_encoder.parameters(), self.momentum_encoder.parameters() + ): + param_m.data.copy_(param_b.data) # initialize + param_m.requires_grad = False # not update by gradient + + def _build_mlp(self, num_layers, input_dim, mlp_dim, output_dim, last_bn=True): + mlp = [] + for l in range(num_layers): + dim1 = input_dim if l == 0 else mlp_dim + dim2 = output_dim if l == num_layers - 1 else mlp_dim + + mlp.append(Linear(dim1, dim2, bias=False)) # libai + if l < num_layers - 1: + mlp.append(nn.BatchNorm1d(dim2)) + mlp.append(nn.ReLU(inplace=True)) + elif last_bn: + # follow SimCLR's design: + # https://github.com/google-research/simclr/blob/master/model_util.py#L157 + # for simplicity, we further removed gamma in BN + + # TODO: affine should be False (bug here) + mlp.append(nn.BatchNorm1d(dim2, affine=True)) + + return nn.Sequential(*mlp) + + def _build_projector_and_predictor_mlps(self, dim, mlp_dim): + pass + + @flow.no_grad() + def _update_momentum_encoder(self, m): + """Momentum update of the momentum encoder""" + for param_b, param_m in zip( + self.base_encoder.parameters(), self.momentum_encoder.parameters() + ): + param_m.data = param_m.data * m + param_b.data * (1.0 - m) + + def contrastive_loss(self, q, k): + + # normalize + q = nn.functional.normalize(q, dim=1) + k = nn.functional.normalize(k, dim=1) + + # gather all targets + # k = concat_all_gather(k).to_global(sbp=q.sbp, placement=q.placement) + k = k.to_global(sbp=flow.sbp.broadcast) + + # Einstein sum is more intuitive + logits = flow.einsum("nc,mc->nm", q, k) / self.T + N = logits.shape[0] // get_world_size() + labels = (flow.arange(N, dtype=flow.long) + N * flow.env.get_rank()).to_global( + sbp=flow.sbp.split(0), placement=logits.placement + ) + + return nn.CrossEntropyLoss()(logits, labels) * (2 * self.T) + + def adjust_moco_momentum(self, cu_iter, m): + """Adjust moco momentum based on current epoch""" + m = 1.0 - 0.5 * (1.0 + math.cos(math.pi * cu_iter / self.max_iter)) * (1.0 - m) + return m + + def forward(self, images, labels=None, cu_iter=0, m=0.99): + + if self.training: + [x1, x2] = flow.chunk(images, 2, dim=1) + # compute features + q1 = self.predictor(self.base_encoder(x1)["prediction_scores"]) + q2 = self.predictor(self.base_encoder(x2)["prediction_scores"]) + + m = self.adjust_moco_momentum(cu_iter, m) # update the moco_momentum + + with flow.no_grad(): # no gradient + self._update_momentum_encoder(m) # update the momentum encoder + + # compute momentum features as targets + k1 = self.momentum_encoder(x1)["prediction_scores"] + k2 = self.momentum_encoder(x2)["prediction_scores"] + + return ( + {"losses": self.contrastive_loss(q1, k2) + self.contrastive_loss(q2, k1)}, + {"m": m}, + ) + else: + return self.base_encoder(images) + + +class MoCo_ViT(MoCo): + def _build_projector_and_predictor_mlps(self, dim, mlp_dim): + hidden_dim = self.base_encoder.head.weight.shape[1] + # projectors + self.base_encoder.head = self._build_mlp(3, hidden_dim, mlp_dim, dim) + self.momentum_encoder.head = self._build_mlp(3, hidden_dim, mlp_dim, dim) + + # predictor + self.predictor = self._build_mlp(2, dim, mlp_dim, dim) diff --git a/projects/MOCOV3/modeling/vit.py b/projects/MOCOV3/modeling/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..f8a5dc186eed12427bff9e90b8419f8bbfe06932 --- /dev/null +++ b/projects/MOCOV3/modeling/vit.py @@ -0,0 +1,157 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# -------------------------------------------------------- +# ViT Model +# References: +# moco-v3: https://github.com/facebookresearch/moco-v3/blob/main/vits.py +# -------------------------------------------------------- + + +import math +from functools import reduce +from operator import mul + +import oneflow as flow +import oneflow.nn as nn +from flowvision.layers.weight_init import trunc_normal_ +from utils.load_checkpoint import load_checkpoint + +from libai.layers import Linear, PatchEmbedding +from libai.models import vision_transformer + + +class VisionTransformer(vision_transformer.VisionTransformer): + """Vision Transformer for MOCO + LiBai impl of: `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` + - https://arxiv.org/abs/2010.11929 + """ + + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4.0, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.0, + global_pool=False, + num_classes=1000, + loss_func=None, + linear_prob=None, + weight_style="pytorch", + stop_grad_conv1=False, + ): + super(VisionTransformer, self).__init__( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + depth=depth, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + drop_rate=drop_rate, + attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rate, + num_classes=num_classes, + loss_func=loss_func, + ) + self.global_pool = global_pool + + # weight init + if linear_prob: + load_checkpoint(self, linear_prob, weight_style, num_heads, embed_dim) + self.head.weight.data.normal_(mean=0.0, std=0.01) + self.head.bias.data.zeros_() + else: + trunc_normal_(self.pos_embed, std=0.02) + trunc_normal_(self.cls_token, std=0.02) + self.apply(self._init_weights) + + self.stop_grad_conv1 = stop_grad_conv1 + self.embed_dim = embed_dim + self.initialization() + + def initialization(self): + + # Use fixed 2D sin-cos position embedding + self.build_2d_sincos_position_embedding() + + # weight initialization + for name, m in self.named_modules(): + if isinstance(m, Linear): + if "query_key_value" in name: + val = math.sqrt(6.0 / float(m.weight.shape[0] // 3 + m.weight.shape[1])) + nn.init.uniform_(m.weight, -val, val) + else: + nn.init.xavier_uniform_(m.weight) + + nn.init.zeros_(m.bias) + nn.init.normal_(self.cls_token, std=1e-6) + + if isinstance(self.patch_embed, PatchEmbedding): + # xavier_uniform initialization + val = math.sqrt( + 6.0 / float(3 * reduce(mul, self.patch_embed.patch_size, 1) + self.embed_dim) + ) + nn.init.uniform_(self.patch_embed.proj.weight, -val, val) + nn.init.zeros_(self.patch_embed.proj.bias) + + if self.stop_grad_conv1: + self.patch_embed.proj.weight.requires_grad = False + self.patch_embed.proj.bias.requires_grad = False + + def build_2d_sincos_position_embedding(self, temperature=10000.0): + sbp = self.pos_embed.sbp + placement = self.pos_embed.placement + + h, w = self.patch_embed.grid_size + grid_w = flow.arange(w, dtype=flow.float32).to_global(sbp=sbp, placement=placement) + grid_h = flow.arange(h, dtype=flow.float32).to_global(sbp=sbp, placement=placement) + grid_w, grid_h = flow.meshgrid(grid_w, grid_h) + assert ( + self.embed_dim % 4 == 0 + ), "Embed dimension must be divisible by 4 for 2D sin-cos position embedding" + pos_dim = self.embed_dim // 4 + omega = (flow.arange(pos_dim, dtype=flow.float32) / pos_dim).to_global( + sbp=sbp, placement=placement + ) + omega = 1.0 / flow.tensor(temperature).to_global(sbp=sbp, placement=placement) ** omega + out_w = flow.einsum("m,d->md", grid_w.flatten(), omega) + out_h = flow.einsum("m,d->md", grid_h.flatten(), omega) + pos_emb = flow.cat( + [flow.sin(out_w), flow.cos(out_w), flow.sin(out_h), flow.cos(out_h)], dim=1 + )[None, :, :] + pe_token = flow.zeros([1, 1, self.embed_dim], dtype=flow.float32).to_global( + sbp=sbp, placement=placement + ) + self.pos_embed = nn.Parameter(flow.cat([pe_token, pos_emb], dim=1)) + self.pos_embed.requires_grad = False + + def forward_head(self, x): + if self.global_pool: + x = x[:, 1:, :].mean(dim=1) # global pool without cls token + outcome = self.norm(x) + outcome = self.head(outcome) + else: + x = self.norm(x) + outcome = x[:, 0] + outcome = self.head(outcome) + return outcome diff --git a/projects/MOCOV3/pretrain_net.py b/projects/MOCOV3/pretrain_net.py new file mode 100644 index 0000000000000000000000000000000000000000..1ab77e770c0bb32cec6a2b67a96f43d8b22f7cb2 --- /dev/null +++ b/projects/MOCOV3/pretrain_net.py @@ -0,0 +1,79 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging +import sys + +from trainer.moco_trainer import MoCoEagerTrainer + +from libai.config import LazyConfig, default_argument_parser, try_get_key +from libai.engine import DefaultTrainer, default_setup +from libai.utils.checkpoint import Checkpointer + +sys.path.append(".") +logger = logging.getLogger(__name__) + + +class MoCoPretrainingTrainer(DefaultTrainer): + def __init__(self, cfg): + + super().__init__(cfg) + + self.model.max_iter = cfg.train.train_iter + + self._trainer = MoCoEagerTrainer( + self.model, self.train_loader, self.optimizer, cfg.train.num_accumulation_steps + ) + + +def main(args): + cfg = LazyConfig.load(args.config_file) + cfg = LazyConfig.apply_overrides(cfg, args.opts) + + if try_get_key(cfg, "graph.enabled") is True: + raise NotImplementedError( + "LiBai MOCO only support eager global mode now, please set cfg.graph.enabled=False" + ) + + default_setup(cfg, args) + + if args.fast_dev_run: + cfg.train.train_epoch = 0 + cfg.train.train_iter = 20 + cfg.train.eval_period = 10 + cfg.train.log_period = 1 + + if args.eval_only: + tokenizer = None + if try_get_key(cfg, "tokenization.setup", default=False): + tokenizer = MoCoPretrainingTrainer.build_tokenizer(cfg) + model = MoCoPretrainingTrainer.build_model(cfg) + Checkpointer(model, save_dir=cfg.train.output_dir).resume_or_load( + cfg.train.load_weight, resume=args.resume + ) + if try_get_key(cfg, "train.graph.enabled", default=False): + model = MoCoPretrainingTrainer.build_graph(cfg, model, is_train=False) + test_loader = MoCoPretrainingTrainer.build_test_loader(cfg, tokenizer) + _ = MoCoPretrainingTrainer.test(cfg, test_loader, model) + return + + trainer = MoCoPretrainingTrainer(cfg) + return trainer.train() + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + main(args) diff --git a/projects/MOCOV3/trainer/moco_trainer.py b/projects/MOCOV3/trainer/moco_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..06fd4ef46b44a93a2ce46c56831bdc23521a0541 --- /dev/null +++ b/projects/MOCOV3/trainer/moco_trainer.py @@ -0,0 +1,43 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import time +from typing import Callable + +from libai.engine.trainer import EagerTrainer + + +class MoCoEagerTrainer(EagerTrainer): + def run_step(self, get_batch: Callable): + + assert self.model.training, "[SimpleTrainer] model was changed to eval mode!" + start = time.perf_counter() + + # If you want to do something with the data, you can wrap the dataloader. + data = next(self._data_loader_iter) + data = get_batch(data, getattr(self.data_loader, "mixup_func", None)) + data_time = time.perf_counter() - start + + # update the moco_momentum per step + loss_dict, m_dict = self.model(**data, cu_iter=self.iter, m=self.model.m) + self.model.m = m_dict["m"] + losses = sum(loss_dict.values()) / self.grad_acc_steps + losses.backward() + self.write_metrics(loss_dict, data_time) + + if (self.iter + 1) % self.grad_acc_steps == 0: + self.optimizer.step() + self.optimizer.zero_grad() diff --git a/projects/MOCOV3/transform/linear_prob_transform.py b/projects/MOCOV3/transform/linear_prob_transform.py new file mode 100644 index 0000000000000000000000000000000000000000..5e7c2d96c39c9b44219b283acfe642a8ac4e3b9a --- /dev/null +++ b/projects/MOCOV3/transform/linear_prob_transform.py @@ -0,0 +1,27 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from flowvision import transforms +from flowvision.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD + +from libai.config import LazyCall + +train_aug = [ + LazyCall(transforms.RandomResizedCrop)(size=224), + LazyCall(transforms.RandomHorizontalFlip)(), + LazyCall(transforms.ToTensor)(), + LazyCall(transforms.Normalize)(mean=IMAGENET_DEFAULT_MEAN, std=IMAGENET_DEFAULT_STD), +] diff --git a/projects/MOCOV3/transform/pretrain_transform.py b/projects/MOCOV3/transform/pretrain_transform.py new file mode 100644 index 0000000000000000000000000000000000000000..e5cfd6659c4d3965273b605b3a38b32494696527 --- /dev/null +++ b/projects/MOCOV3/transform/pretrain_transform.py @@ -0,0 +1,95 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import random + +import oneflow as flow +from flowvision import transforms +from flowvision.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD +from PIL import ImageFilter, ImageOps + +from libai.config import LazyCall + + +class GaussianBlur(object): + """Gaussian blur augmentation from SimCLR: https://arxiv.org/abs/2002.05709""" + + def __init__(self, sigma=[0.1, 2.0]): + self.sigma = sigma + + def __call__(self, x): + sigma = random.uniform(self.sigma[0], self.sigma[1]) + x = x.filter(ImageFilter.GaussianBlur(radius=sigma)) + return x + + +class Solarize(object): + """Solarize augmentation from BYOL: https://arxiv.org/abs/2006.07733""" + + def __call__(self, x): + return ImageOps.solarize(x) + + +# follow BYOL's augmentation recipe: https://arxiv.org/abs/2006.07733 +augmentation1 = [ + LazyCall(transforms.RandomResizedCrop)(size=224, scale=(0.2, 1.0)), + LazyCall(transforms.RandomApply)( + transforms=[ + LazyCall(transforms.ColorJitter)( + brightness=0.4, contrast=0.4, saturation=0.2, hue=0.1 + ) # not strengthened + ], + p=0.8, + ), + # TODO: Add RandomGrayscale + # LazyCall(transforms.RandomGrayscale)(p=0.2), + LazyCall(transforms.RandomApply)(transforms=[LazyCall(GaussianBlur)(sigma=[0.1, 2.0])], p=1.0), + LazyCall(transforms.RandomHorizontalFlip)(), + LazyCall(transforms.ToTensor)(), + LazyCall(transforms.Normalize)(mean=IMAGENET_DEFAULT_MEAN, std=IMAGENET_DEFAULT_STD), +] + +augmentation2 = [ + LazyCall(transforms.RandomResizedCrop)(size=224, scale=(0.2, 1.0)), + LazyCall(transforms.RandomApply)( + transforms=[ + LazyCall(transforms.ColorJitter)( + brightness=0.4, contrast=0.4, saturation=0.2, hue=0.1 + ) # not strengthened + ], + p=0.8, + ), + # TODO: Add RandomGrayscale + # LazyCall(transforms.RandomGrayscale)(p=0.2), + LazyCall(transforms.RandomApply)(transforms=[LazyCall(GaussianBlur)(sigma=[0.1, 2.0])], p=1.0), + LazyCall(transforms.RandomApply)(transforms=[LazyCall(Solarize)()], p=0.2), + LazyCall(transforms.RandomHorizontalFlip)(), + LazyCall(transforms.ToTensor)(), + LazyCall(transforms.Normalize)(mean=IMAGENET_DEFAULT_MEAN, std=IMAGENET_DEFAULT_STD), +] + + +class TwoCropsTransform: + """Take two random crops of one image""" + + def __init__(self, base_transform1, base_transform2): + self.base_transform1 = base_transform1 + self.base_transform2 = base_transform2 + + def __call__(self, x): + im1 = self.base_transform1(x) + im2 = self.base_transform2(x) + return flow.cat((im1, im2), dim=0) diff --git a/projects/MOCOV3/utils/load_checkpoint.py b/projects/MOCOV3/utils/load_checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..426e0e5d86633b836c24fc0e3232e2b86f3c5ce0 --- /dev/null +++ b/projects/MOCOV3/utils/load_checkpoint.py @@ -0,0 +1,72 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging + +from utils.weight_convert import load_torch_checkpoint_linear_prob + +from libai.utils.checkpoint import ( + Checkpointer, + get_missing_parameters_message, + get_unexpected_parameters_message, +) + +logger = logging.getLogger("libai." + __name__) + + +def load_checkpoint(model, path, weight_style, num_heads, embed_dim): + linear_keyword = "head" + for name, param in model.named_parameters(): + if name not in ["%s.weight" % linear_keyword, "%s.bias" % linear_keyword]: + param.requires_grad = False + assert weight_style in ["pytorch", "oneflow"] + if weight_style == "pytorch": + params = load_torch_checkpoint_linear_prob(num_heads, embed_dim, path=path) + else: + params = Checkpointer(model).load(path) + + model_state_dict = model.state_dict() + + # check the incorrect shape and unexpected keys + incorrect_shapes = [] + unexpected_keys = [] + for k in list(params.keys()): + if k in model_state_dict: + shape_model = tuple(model_state_dict[k].shape) + shape_ckp = tuple(params[k].shape) + if shape_model != shape_ckp: + incorrect_shapes.append((k, shape_ckp, shape_model)) + params.pop(k) + model_state_dict.pop(k) + else: + unexpected_keys.append(k) + + missing_keys = list(model_state_dict.keys()) + + for k, shape_checkpoint, shape_model in incorrect_shapes: + logger.warning( + "Skip loading parameter '{}' to the model due to incompatible " + "shapes: {} in the checkpoint but {} in the " + "model! You might want to double check if this is expected.".format( + k, shape_checkpoint, shape_model + ) + ) + if missing_keys: + logger.info(get_missing_parameters_message(missing_keys)) + if unexpected_keys: + logger.info(get_unexpected_parameters_message(unexpected_keys)) + + model.load_state_dict(params, strict=False) diff --git a/projects/MOCOV3/utils/weight_convert.py b/projects/MOCOV3/utils/weight_convert.py new file mode 100644 index 0000000000000000000000000000000000000000..0839ab2c142cbfc025dc6c02edabfc7565047a57 --- /dev/null +++ b/projects/MOCOV3/utils/weight_convert.py @@ -0,0 +1,107 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging + +import oneflow as flow +import torch + +logger = logging.getLogger(__name__) + + +def convert_qkv_weight(value, num_heads, hidden_size): + """ + convert qkv.weight to be compatible with LiBai transformer layer + + Args: + cfg: config file + value: qkv.weight in the loaded checkpoint + """ + + head_size = int(hidden_size / num_heads) + qkv_weight = ( + value.view(3, num_heads, head_size, hidden_size) + .permute(1, 0, 2, 3) + .contiguous() + .view(hidden_size * 3, hidden_size) + ) + + return qkv_weight + + +def convert_qkv_bias(value, num_heads, hidden_size): + """ + convert qkv.bias to be compatible with LiBai transformer layer + + Args: + cfg: config file + value: qkv.bias in the loaded checkpoint + """ + + head_size = int(hidden_size / num_heads) + qkv_bias = ( + value.view(3, num_heads, head_size).permute(1, 0, 2).contiguous().view(hidden_size * 3) + ) + + return qkv_bias + + +def filter_keys(key, value, num_heads, hidden_size): + """Filtering the state_dict keys and values to match LiBai's MOCOV3 model""" + if "norm1" in key: + key = key.replace("norm1", "input_layernorm") + elif "attn.qkv" in key: + key = key.replace("attn.qkv", "self_attention.query_key_value") + if "weight" in key: + value = convert_qkv_weight(value, num_heads, hidden_size) + if "bias" in key: + value = convert_qkv_bias(value, num_heads, hidden_size) + elif "attn.proj" in key: + key = key.replace("attn.proj", "self_attention.dense") + elif "norm2" in key: + key = key.replace("norm2", "post_attention_layernorm") + elif "mlp.fc1" in key: + key = key.replace("mlp.fc1", "mlp.dense_h_to_4h") + elif "mlp.fc2" in key: + key = key.replace("mlp.fc2", "mlp.dense_4h_to_h") + elif "fc_norm" in key: + key = key.replace("fc_norm", "norm") + + return key, value + + +def load_torch_checkpoint_linear_prob( + num_heads, hidden_size, path="projects/MOCOV3/output/vit-b-300ep.pth.tar", linear_keyword="head" +): + """Load checkpoint from the given torch weights. + Torch weight from: xxx + """ + torch_dict = torch.load(path, map_location="cpu")["state_dict"] + parameters = torch_dict + new_parameters = dict() + for key, value in parameters.items(): + if "num_batches_tracked" not in key: + if key.startswith("module.base_encoder") and not key.startswith( + "module.base_encoder.%s" % linear_keyword + ): + # to global tensor + key, val = filter_keys(key, value, num_heads, hidden_size) + val = val.detach().cpu().numpy() + val = flow.tensor(val).to_global( + sbp=flow.sbp.broadcast, placement=flow.placement("cuda", {0: range(1)}) + ) + new_parameters[key[len("module.base_encoder.") :]] = val + return new_parameters diff --git a/projects/MT5/configs/mt5_base.py b/projects/MT5/configs/mt5_base.py new file mode 100644 index 0000000000000000000000000000000000000000..61640702918072044cc701b271febc4a443fed95 --- /dev/null +++ b/projects/MT5/configs/mt5_base.py @@ -0,0 +1,30 @@ +from omegaconf import DictConfig +from libai.config import LazyCall +from projects.MT5.mt5_model import MT5Model, MT5ForPreTraining + + +cfg = dict( + vocab_size=250112, + hidden_size=768, + hidden_layers=12, + num_attention_heads=12, + head_size=64, + intermediate_size=2048, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + embedding_dropout_prob=0.1, + relative_attention_num_buckets=32, + initializer_range=1.0, + layernorm_eps=1e-06, + amp_enabled=False, + model_type="mt5", + eos_token_id=1, + padding_idx=0, + is_encoder_decoder=True, + tie_word_embeddings=True, +) + +cfg = DictConfig(cfg) + +mt5_model = LazyCall(MT5Model)(cfg=cfg) +pretrain_model = LazyCall(MT5ForPreTraining)(cfg=cfg) diff --git a/projects/MT5/configs/mt5_large.py b/projects/MT5/configs/mt5_large.py new file mode 100644 index 0000000000000000000000000000000000000000..f67f106bdd5ebb38458af916a7fc2a631c045186 --- /dev/null +++ b/projects/MT5/configs/mt5_large.py @@ -0,0 +1,30 @@ +from omegaconf import DictConfig +from libai.config import LazyCall +from projects.MT5.mt5_model import MT5Model, MT5ForPreTraining + + +cfg = dict( + vocab_size=250112, + hidden_size=1024, + hidden_layers=24, + num_attention_heads=16, + head_size=64, + intermediate_size=2816, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + embedding_dropout_prob=0.1, + relative_attention_num_buckets=32, + initializer_range=1.0, + layernorm_eps=1e-06, + amp_enabled=False, + model_type="mt5", + eos_token_id=1, + padding_idx=0, + is_encoder_decoder=True, + tie_word_embeddings=False, +) + +cfg = DictConfig(cfg) + +mt5_model = LazyCall(MT5Model)(cfg=cfg) +pretrain_model = LazyCall(MT5ForPreTraining)(cfg=cfg) diff --git a/projects/MT5/configs/mt5_pretrain.py b/projects/MT5/configs/mt5_pretrain.py new file mode 100644 index 0000000000000000000000000000000000000000..7b4939081b0ecf9ef1bfb08218df0758e1a22582 --- /dev/null +++ b/projects/MT5/configs/mt5_pretrain.py @@ -0,0 +1,71 @@ +from libai.config import LazyCall +from libai.evaluation import PPLEvaluator +from libai.scheduler import WarmupExponentialLR +from configs.common.train import train +from configs.common.data.t5_dataset import dataloader, tokenization +from configs.common.models.graph import graph +from configs.common.optim import optim +from projects.MT5.configs.mt5_base import pretrain_model as model + + +vocab_file = "./data_test/bert_data/bert-base-chinese-vocab.txt" +data_prefix = "./data_test/bert_data/loss_compara_content_sentence" + +tokenization.tokenizer.vocab_file = vocab_file +dataloader.train.dataset[0].data_prefix = data_prefix +dataloader.train.dataset[0].indexed_dataset.data_prefix = data_prefix + +# model config +model.cfg.hidden_size = 768 +model.cfg.hidden_layers = 12 +model.cfg.num_attention_heads = 12 +model.cfg.head_size = 64 +model.cfg.intermediate_size = 2048 +model.cfg.model_type = "mt5" +model.cfg.hidden_dropout_prob = 0.0 +model.cfg.attention_probs_dropout_prob = 0.0 +model.cfg.embedding_dropout_prob = 0.0 +model.cfg.vocab_size = 30522 +model.cfg.padding_idx = 0 +model.cfg.tie_word_embeddings = False +model.cfg.is_encoder_decoder = False +model.cfg.amp_enabled = True +model.cfg.initializer_range = 0.02 +model.cfg.pretrained_model_path = None + +train.update( + dict( + output_dir="projects/MT5/output/mt5_output", + train_micro_batch_size=4, + train_epoch=1, + train_iter=24000, + log_period=10, + amp=dict(enabled=True), + warmup_ratio=1 / 24, + # checkpointer=dict(period=10, max_to_keep=20), + input_placement_device="cpu", + dist=dict( + data_parallel_size=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + pipeline_num_layers=2 * model.cfg.hidden_layers, + ), + scheduler=LazyCall(WarmupExponentialLR)( + warmup_factor=0.001, + gamma=1.0, + warmup_method="linear", + warmup_iter=0.0, + ), + evaluation=dict( + evaluator=LazyCall(PPLEvaluator)(), + enabled=True, + eval_iter=1e5, + eval_period=5000, + ), + ) +) + +train.zero_optimization.enabled = True +train.zero_optimization.stage = 2 +train.activation_checkpoint.enabled = False +train.num_accumulation_steps = 8 diff --git a/projects/MT5/configs/mt5_small.py b/projects/MT5/configs/mt5_small.py new file mode 100644 index 0000000000000000000000000000000000000000..4ea93f94347f9bb2adbe8f061c6fc3b290818cd3 --- /dev/null +++ b/projects/MT5/configs/mt5_small.py @@ -0,0 +1,30 @@ +from omegaconf import DictConfig +from libai.config import LazyCall +from projects.MT5.mt5_model import MT5Model, MT5ForPreTraining + + +cfg = dict( + vocab_size=250112, + hidden_size=512, + hidden_layers=8, + num_attention_heads=6, + head_size=64, + intermediate_size=1024, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + embedding_dropout_prob=0.1, + relative_attention_num_buckets=32, + initializer_range=1.0, + layernorm_eps=1e-06, + amp_enabled=False, + model_type="mt5", + eos_token_id=1, + padding_idx=0, + is_encoder_decoder=True, + tie_word_embeddings=False, +) + +cfg = DictConfig(cfg) + +mt5_model = LazyCall(MT5Model)(cfg=cfg) +pretrain_model = LazyCall(MT5ForPreTraining)(cfg=cfg) diff --git a/projects/MT5/configs/t5_inference.py b/projects/MT5/configs/t5_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..528aca972338792da89c7e57f8d1535eeb4aaf75 --- /dev/null +++ b/projects/MT5/configs/t5_inference.py @@ -0,0 +1,46 @@ +from .mt5_base import cfg +from libai.config import LazyCall +from libai.tokenizer import T5Tokenizer +from projects.MT5.mt5_model import MT5Model, MT5ForPreTraining +from configs.common.train import train +from configs.common.data.t5_dataset import tokenization + +cfg.update( + model_type="t5", + is_encoder_decoder=True, + max_length=20, + min_length=0, + do_sample=False, + early_stopping=False, + num_beams=1, + num_beam_groups=1, + diversity_penalty=0.0, + temperature=1.0, + top_k=50, + top_p=1.0, + typical_p=1.0, + repetition_penalty=1.0, + length_penalty=1.0, + no_repeat_ngram_size=0, + encoder_no_repeat_ngram_size=0, + num_return_sequences=1, + chunk_size_feed_forward=0, + output_scores=False, + forced_bos_token_id=None, + forced_eos_token_id=None, + remove_invalid_values=False, + exponential_decay_length_penalty=None, + use_cache=True, + # Tokenizer + pad_token_id=0, + eos_token_id=1, + bos_token_id=None, + sep_token_id=None, + decoder_start_token_id=0, +) + +model = LazyCall(MT5Model)(cfg=cfg) +tokenization.tokenizer = LazyCall(T5Tokenizer)( + vocab_file="/path/to/spiece.model", + add_bos_token=True, +) diff --git a/projects/MT5/layers/attention_layer.py b/projects/MT5/layers/attention_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..166e792bfc27d1a2081b47f85b2f258c9f9d45de --- /dev/null +++ b/projects/MT5/layers/attention_layer.py @@ -0,0 +1,345 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import Tuple + +import oneflow as flow +from oneflow import nn + +from libai.layers.linear import Linear +from libai.utils import distributed as dist +from projects.MT5.layers.embed_layer import Embedding + + +class MultiheadAttention(nn.Module): + """Multi-head attention layer, support self attention and cross attention. + + Args: + hidden_size: size of hidden state. + num_attention_heads: number of attention heads. + is_cross_attention: used to specify whether it is self attention or cross attention. + Defaults to False. + attention_dropout_prob: dropout probability of attention weights. + Defaults to 0.0. + output_dropout_prob: dropout probability of output. Defaults to 0.0. + init_method: method to initialize the input layer weights. + Defaults to ``init.xavier_normal_``. + output_layer_init_method: method to initialize the output layer weights. + If None, use ``init_method``. + layer_idx: a layer_idx sign which determines the placements. + It will be used in pipeline parallelism. Defaults to 0. + """ + + def __init__( + self, + hidden_size, + num_attention_heads, + head_size, + relative_attention_num_buckets, + is_cross_attention=False, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + padding_idx=None, + *, + layer_idx=0, + has_relative_attention_bias=False, + is_decoder=False, + ): + super().__init__() + self.hidden_size = hidden_size + self.relative_attention_num_buckets = relative_attention_num_buckets + self.has_relative_attention_bias = has_relative_attention_bias + self.is_decoder = is_decoder + self.attention_dropout_prob = attention_dropout_prob + + if output_layer_init_method is None: + output_layer_init_method = init_method + self.num_heads = num_attention_heads + self.head_size = head_size + + self.dropout = nn.Dropout(p=attention_dropout_prob) + self.norm_factor = 1.0 / math.sqrt(float(self.head_size)) + + self.is_cross_attention = is_cross_attention + + self.output_dropout = nn.Dropout(p=output_dropout_prob) + + if self.is_cross_attention: + self.query = Linear( + self.hidden_size, + self.num_heads * self.head_size, + bias=False, + parallel="col", + init_method=init_method, + layer_idx=layer_idx, + ) + self.key_value = Linear( + self.hidden_size, + self.num_heads * self.head_size * 2, + bias=False, + parallel="col", + init_method=init_method, + layer_idx=layer_idx, + ) + else: + self.query_key_value = Linear( + self.hidden_size, + self.num_heads * self.head_size * 3, + bias=False, + parallel="col", + init_method=init_method, + layer_idx=layer_idx, + ) + + self.dense = Linear( + self.num_heads * self.head_size, + self.hidden_size, + bias=False, + parallel="row", + init_method=output_layer_init_method, + skip_bias_add=False, + layer_idx=layer_idx, + ) + if self.has_relative_attention_bias: + self.relative_attention_bias = Embedding( + self.relative_attention_num_buckets, + self.num_heads, + padding_idx=padding_idx, + layer_idx=layer_idx, + ) + + def forward( + self, + hidden_states: flow.Tensor, + encoder_states: flow.Tensor = None, + attention_mask: flow.Tensor = None, + past_key_value: Tuple[flow.Tensor, flow.Tensor] = None, + use_cache: bool = False, + position_bias=None, + query_length=None, + ): + """ + + Args: + hidden_states (flow.Tensor): shape is [bsz, tgt_len, hidden_size]. + encoder_states (flow.Tensor, optional): shape is [bsz, src_len, hidden_size]. + Defaults to None. + attention_mask (flow.Tensor, optional): shape is [bsz, 1, tgt_len, src_len]. + It should be the combination of padding mask and casual mask. + It is the padding mask of source input when used with self-attention in encoder. + And it is the combination of padding mask of target input and casual mask when + used with self-attention in decoder. It is the padding mask of source input when + used with cross-attention in decoder. + Defaults to None. + past_key_value (Tuple[flow.Tensor, flow.Tensor], optional): tuple of key and value, + each shape is [bsz, num_heads, src_len, head_size]. Defaults to None. + use_cache (bool, optional): it will be set to True, when the model is in the inference + phase and used for incremental decoding. Defaults to False. + """ + + if encoder_states is not None: + encoder_states = encoder_states.to_global(placement=hidden_states.placement) + + if attention_mask is not None: + attention_mask = attention_mask.to_global(placement=hidden_states.placement) + + # hidden_states shape: [seq_len, batch_size, hidden_size] + real_seq_length, bsz = hidden_states.size()[:2] + + if past_key_value is not None: + assert ( + len(past_key_value) == 2 + ), "past_key_value should have 2 past states: keys and values." + f"Got {len(past_key_value)} past states.\n" + real_seq_length += past_key_value[0].shape[2] if query_length is None else query_length + + key_length = real_seq_length if encoder_states is None else encoder_states.shape[0] + + if self.is_cross_attention: + query = self.query(hidden_states) + query = query.view(-1, bsz, self.num_heads, self.head_size) + query = query.permute(1, 2, 0, 3) # bsz, num_head, seq_len, head_size + + if past_key_value is not None: + key, value = past_key_value + elif encoder_states is not None: + key_value = self.key_value(encoder_states) + key_value = key_value.view(-1, bsz, self.num_heads, 2 * self.head_size) + key_value = key_value.permute(1, 2, 0, 3) + key, value = flow.chunk(key_value, chunks=2, dim=-1) + else: + raise ValueError( + "past_key_value and encoder_states cannot be None at the same time." + ) + else: + query_key_value = self.query_key_value(hidden_states) + if use_cache: + query_key_value = query_key_value.view(bsz, -1, self.num_heads, 3 * self.head_size) + query_key_value = query_key_value.permute( + 0, 2, 1, 3 + ) # [bsz, num_heads, src_len, 3 * head_size] + query, key, value = flow.chunk(query_key_value, chunks=3, dim=-1) + else: + attention_scores, value = flow._C.fused_self_attention( + query_key_value, head_size=self.head_size, alpha=1 + ) + if past_key_value is not None: + past_key, past_value = past_key_value + key = flow.cat((past_key.type_as(key), key), dim=2) + value = flow.cat((past_value.type_as(value), value), dim=2) + + if use_cache: + past_key_value = (key, value) + + if self.is_cross_attention or use_cache: + attention_scores = flow.matmul(query, key, transpose_b=True, alpha=1) + + if position_bias is None: + if not self.has_relative_attention_bias: + position_bias = flow.zeros( + (1, self.num_heads, real_seq_length, key_length), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=attention_scores.placement, + ) + else: + position_bias = self.compute_bias( + real_seq_length, key_length, placement=attention_mask.placement + ) + + if past_key_value is not None: + position_bias = position_bias[:, :, -hidden_states.size(1) :, :] + + if attention_mask is not None: + if use_cache: + attention_mask = attention_mask.expand_as(attention_scores) + + attention_weights = flow._C.fused_bias_add_scale_mask_softmax_dropout( + attention_scores, + position_bias, + attention_mask, + fill_value=-10000.0, + scale=1, + p=self.attention_dropout_prob, + )[0] + else: + attention_scores = attention_scores + position_bias + attention_weights = flow.softmax(attention_scores, dim=-1) + attention_weights = self.dropout(attention_weights) + + context = flow.matmul(attention_weights, value) + + """ transpose [batch_size, num_head, seq_len, head_size] to + [seq_len, batch_size, num_head, head_size] + """ + context = flow._C.transpose(context, perm=(2, 0, 1, 3)) + + output = self.dense(context.flatten(2)) + + output = self.output_dropout(output) + + if use_cache: + output = (output, past_key_value) + + output = (output,) + (position_bias,) + return output + + def extra_repr(self) -> str: + return "hidden_size={}, num_heads={}, is_cross_attention={}".format( + self.hidden_size, + self.num_heads, + self.is_cross_attention, + ) + + def _relative_position_bucket( + self, relative_position, bidirectional=True, num_buckets=32, max_distance=128 + ): + relative_buckets = 0 + if bidirectional: + num_buckets //= 2 + relative_buckets = ( + relative_buckets + (relative_position > 0).to(flow.long) * num_buckets + ) + relative_position = flow.abs(relative_position) + else: + relative_position = ( + -1 + * flow.min( + relative_position, + flow.zeros( + relative_position.size(), + sbp=relative_position.sbp, + placement=relative_position.placement, + ), + ).to(flow.long) + ) + + max_exact = num_buckets // 2 + is_small = relative_position < max_exact + + relative_postion_if_large = max_exact + ( + flow.log(relative_position.float() / max_exact) + / math.log(max_distance / max_exact) + * (num_buckets - max_exact) + ).to(flow.long) + + relative_postion_if_large = flow.min( + relative_postion_if_large, + flow.zeros( + relative_postion_if_large.size(), + dtype=relative_postion_if_large.dtype, + sbp=relative_postion_if_large.sbp, + placement=relative_postion_if_large.placement, + ).fill_(num_buckets - 1), + ) + + relative_buckets = relative_buckets + flow.where( + is_small, relative_position, relative_postion_if_large + ) + return relative_buckets + + def compute_bias(self, query_length, key_length, placement=None): + """Compute binned relative position bias""" + context_position = flow.arange( + query_length, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=placement, + ) + memory_position = flow.arange( + key_length, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=placement, + ) + relative_position = ( + memory_position[None, :] - context_position[:, None] + ) # shape (query_length, key_length) + + relative_position_bucket = self._relative_position_bucket( + relative_position, + bidirectional=(not self.is_decoder), + num_buckets=self.relative_attention_num_buckets, + ) # shape (query_length, key_length) + + values = self.relative_attention_bias( + relative_position_bucket + ) # shape (query_length, key_length, num_heads) + values = values.permute([2, 0, 1]).unsqueeze( + 0 + ) # shape (1, num_heads, query_length, key_length) + return values diff --git a/projects/MT5/layers/embed_layer.py b/projects/MT5/layers/embed_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..93bd81062f4b5f5cbfe97a2cb6136274599c8185 --- /dev/null +++ b/projects/MT5/layers/embed_layer.py @@ -0,0 +1,121 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +import oneflow.nn as nn +from oneflow.nn import init + +import libai.utils.distributed as dist +from libai.layers.embedding import VocabEmbedding + + +class MT5Embedding(flow.nn.Module): + def __init__( + self, + hidden_size, + vocab_size, + embedding_dropout_prob, + pad_token_id=0, + init_method=flow.nn.init.xavier_normal_, + amp_enabled=False, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.vocab_size = vocab_size + + self.word_embeddings = VocabEmbedding( + num_embeddings=vocab_size, + embedding_dim=hidden_size, + init_method=init_method, + amp_enabled=amp_enabled, + padding_idx=pad_token_id, + ) + + self.embedding_dropout = flow.nn.Dropout(embedding_dropout_prob) + + def forward(self, input_ids): + word_embeddings = self.word_embeddings(input_ids) + embeddings = self.embedding_dropout(word_embeddings) + return embeddings + + +class Embedding(nn.Module): + """Construct the trainable embedding module, which does not support parallelization. + This can be used for positional embedding and token type embedding. + + Arguments: + num_embeddings: size of vocabulary. + embedding_dim: dimension of embeddings. + padding_idx: pad index. Defaults to None. + init_method: method to initialize weights. Defaults to ``flow.nn.init.xavier_normal_``. + amp_enabled: fp16 option for embedding weight. Defaults to False. + """ + + def __init__( + self, + num_embeddings, + embedding_dim, + padding_idx=None, + init_method=init.xavier_normal_, + amp_enabled=False, + layer_idx=0, + ): + super().__init__() + self.num_embeddings = num_embeddings + self.embedding_dim = embedding_dim + if padding_idx is not None: + if padding_idx > 0: + assert ( + padding_idx < self.num_embeddings + ), "Padding_idx must be within num_embeddings" + elif padding_idx < 0: + assert ( + padding_idx >= -self.num_embeddings + ), "Padding_idx must be within num_embeddings" + padding_idx = self.num_embeddings + padding_idx + self.padding_idx = padding_idx + self.init_method = init_method + self.amp_enabled = amp_enabled + + assert num_embeddings > 0 + self.weight = nn.Parameter( + flow.empty( + (num_embeddings, embedding_dim), + dtype=flow.float32, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + ) + self.init_method(self.weight) + + def forward(self, input_ids): + weight = flow._C.amp_white_identity(self.weight) if self.amp_enabled else self.weight + input_embeds = flow._C.gather(weight, input_ids, axis=0) + return input_embeds + + def _fill_padding_idx_with_zero(self) -> None: + if self.padding_idx is not None: + with flow.no_grad(): + self.weight[self.padding_idx] = flow.zeros( + self.embedding_dim, + placement=dist.get_layer_placement(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + + def extra_repr(self) -> str: + s = "num_embeddings={num_embeddings}, embedding_dim={embedding_dim}" + if self.padding_idx is not None: + s += ", padding_idx={padding_idx}" + return s.format(**self.__dict__) diff --git a/projects/MT5/layers/lm_head_layer.py b/projects/MT5/layers/lm_head_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..4911a30c45f34d96abf87d92788b87af2daa9df0 --- /dev/null +++ b/projects/MT5/layers/lm_head_layer.py @@ -0,0 +1,21 @@ +from oneflow import nn + +from libai.layers import Linear, LMLogits + + +class LMHead(nn.Module): + def __init__(self, model_type, hidden_size, vocab_size, hidden_layers): + super().__init__() + if model_type == "mt5": + self.lm_head = Linear( + hidden_size, vocab_size, bias=False, layer_idx=2 * hidden_layers - 1 + ) + else: + self.lm_head = LMLogits(vocab_size, bias=True) + + def forward(self, decoder_states, embed_weight=None): + if isinstance(self.lm_head, Linear): + logits = self.lm_head(decoder_states) + else: + logits = self.lm_head(decoder_states, embed_weight) + return logits diff --git a/projects/MT5/layers/logits_layer.py b/projects/MT5/layers/logits_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..27fd2ae018484b254ea216bd18625de142ceb562 --- /dev/null +++ b/projects/MT5/layers/logits_layer.py @@ -0,0 +1,52 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +from oneflow import nn + +from libai.layers import Linear +from libai.utils import distributed as dist + + +class LMLogits(nn.Module): + def __init__(self, vocab_size, hidden_size=None, bias=False, model_type="t5", layer_idx=-1): + super().__init__() + self.model_type = model_type + if model_type == "t5": + self.bias = ( + nn.Parameter( + flow.zeros( + (vocab_size,), + dtype=flow.float32, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.split(0)]), + ) + ) + if bias + else None + ) + elif model_type == "mt5": + self.linear = Linear(hidden_size, vocab_size, bias=False, layer_idx=layer_idx) + + def forward(self, input, word_embeddings=None): + if self.model_type == "t5": + w = word_embeddings.to_global(placement=input.placement) + input = input.to_global(grad_sbp=input.sbp) + logits = flow._C.matmul(input, w, transpose_b=True) + if self.bias is not None: + logits = logits + self.bias + else: + logits = self.linear(input) + return logits diff --git a/projects/MT5/layers/loss_layer.py b/projects/MT5/layers/loss_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..b13bc8fee6727833dd71edfcb5e8280ad581b8bd --- /dev/null +++ b/projects/MT5/layers/loss_layer.py @@ -0,0 +1,60 @@ +import oneflow as flow + +from libai.layers import ParallelCrossEntropyLoss +from libai.utils import distributed as dist + + +class MT5Loss(flow.nn.Module): + def __init__(self) -> None: + super().__init__() + self.lm_loss = ParallelCrossEntropyLoss() + + def forward(self, logits, lm_labels, loss_mask): + lm_labels = lm_labels.to_global(placement=logits.placement) + lm_loss = self.lm_loss(logits, lm_labels) + loss_mask = loss_mask.to_global(placement=lm_loss.placement) + loss_mask = loss_mask.float() + denominator = loss_mask.sum().to_global( + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]) + ) + lm_loss = flow._C.amp_white_identity(lm_loss) + lm_loss = flow._C.amp_black_identity(lm_loss) + masked_lm_loss = flow.sum(lm_loss.view(-1) * loss_mask.view(-1)) / denominator + masked_lm_loss = masked_lm_loss.to_global( + sbp=dist.get_nd_sbp([flow.sbp.partial_sum, flow.sbp.broadcast]) + ) + + if self.training: + # token throughput + done_tokens = ( + flow.zeros( + 1, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=lm_labels.placement, + ) + + logits.shape[0] * logits.shape[1] + ) + + # correct token + correct_tokens = flow.sum( + ( + logits.to_global( + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=lm_labels.placement, + ) + .argmax(dim=-1) + .eq(lm_labels) + ).float() + ) + + return { + "mlm_loss": masked_lm_loss, + "done_tokens": done_tokens, + "correct_tokens": correct_tokens, + "denominator": denominator, + } + + else: + return { + "mlm_loss": masked_lm_loss, + } diff --git a/projects/MT5/layers/mask_layer.py b/projects/MT5/layers/mask_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..3fb0b90dacaa3267907a77bb07a3ea9893380dad --- /dev/null +++ b/projects/MT5/layers/mask_layer.py @@ -0,0 +1,63 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow + +from libai.utils import distributed as dist + + +class ExtendedMask(flow.nn.Module): + def forward(self, x, input_tensor=None, is_decoder=False): + if x.dim() == 3: + extended_mask = x[:, None, :, :] + elif x.dim() == 2: + if is_decoder: + extended_mask = self.create_extended_mask_for_decoder(x, input_tensor) + else: + extended_mask = x[:, None, None, :] + + return extended_mask + + def create_extended_mask_for_decoder(self, x, input_tensor): + batch_size, seq_len = input_tensor.size() + seq_ids = flow.arange( + seq_len, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=x.placement, + ) + causal_mask = ( + seq_ids[None, None, :].repeat(batch_size, seq_len, 1) <= seq_ids[None, :, None] + ) + + causal_mask = causal_mask.to(x.dtype) + causal_mask = causal_mask.to_global(sbp=x.sbp) + if causal_mask.shape[1] < x.shape[1]: + prefix_seq_len = x.shape[1] - causal_mask.shape[1] + ones = flow.ones( + (batch_size, seq_len, prefix_seq_len), + dtype=causal_mask.dtype, + sbp=causal_mask.sbp, + placement=causal_mask.placement, + ) + causal_mask = flow.cat( + [ + ones, + causal_mask, + ], + dim=-1, + ) + + extended_mask = causal_mask[:, None, :, :] * x[:, None, None, :] + return extended_mask diff --git a/projects/MT5/layers/mlp_layer.py b/projects/MT5/layers/mlp_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..d26b2e66aba98a15ef8f1de717c1257490d78b2b --- /dev/null +++ b/projects/MT5/layers/mlp_layer.py @@ -0,0 +1,126 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +from oneflow import nn + +from libai.layers import Linear, build_activation + + +class T5MLP(nn.Module): + def __init__( + self, + hidden_size, + ffn_hidden_size, + output_dropout_prob=0.0, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + *, + layer_idx=0, + ): + super().__init__() + self.output_dropout_prob = output_dropout_prob + + if output_layer_init_method is None: + output_layer_init_method = init_method + + self.dense_h_to_4h = Linear( + hidden_size, + ffn_hidden_size, + bias=False, + parallel="col", + skip_bias_add=False, + init_method=init_method, + layer_idx=layer_idx, + ) + + self.activation_func = build_activation("relu") + + self.dense_4h_to_h = Linear( + ffn_hidden_size, + hidden_size, + bias=False, + parallel="row", + skip_bias_add=False, + init_method=output_layer_init_method, + layer_idx=layer_idx, + ) + + self.dropout = nn.Dropout(self.output_dropout_prob) + + def forward(self, hidden_states): + intermediate = self.dense_h_to_4h(hidden_states) + intermediate = self.activation_func(intermediate) + output = self.dense_4h_to_h(intermediate) + output = self.dropout(output) + return output + + +class MT5MLP(nn.Module): + def __init__( + self, + hidden_size, + ffn_hidden_size, + output_dropout_prob=0.0, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + *, + layer_idx=0, + ): + super().__init__() + self.output_dropout_prob = output_dropout_prob + + if output_layer_init_method is None: + output_layer_init_method = init_method + + self.wi_0 = Linear( + hidden_size, + ffn_hidden_size, + bias=False, + parallel="col", + skip_bias_add=False, + init_method=init_method, + layer_idx=layer_idx, + ) + + self.wi_1 = Linear( + hidden_size, + ffn_hidden_size, + bias=False, + parallel="col", + skip_bias_add=False, + init_method=init_method, + layer_idx=layer_idx, + ) + + self.wo = Linear( + ffn_hidden_size, + hidden_size, + bias=False, + parallel="row", + skip_bias_add=False, + init_method=output_layer_init_method, + layer_idx=layer_idx, + ) + + self.dropout = nn.Dropout(self.output_dropout_prob) + + def forward(self, hidden_states): + wi_0_out = self.wi_0(hidden_states) + hidden_linear = self.wi_1(hidden_states) + hidden_states = flow._C.fused_fast_gelu_mul(wi_0_out, hidden_linear) + output = self.wo(hidden_states) + output = self.dropout(output) + return output diff --git a/projects/MT5/layers/transformer_layer.py b/projects/MT5/layers/transformer_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..c87d30f49bbcddfb0145d3866dca064842a9f351 --- /dev/null +++ b/projects/MT5/layers/transformer_layer.py @@ -0,0 +1,255 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow.nn as nn + +from libai.layers.droppath import DropPath +from libai.layers.layer_norm import RMSLayerNorm as LayerNorm +from libai.utils import distributed as dist +from projects.MT5.layers.attention_layer import MultiheadAttention +from projects.MT5.layers.mlp_layer import MT5MLP, T5MLP + + +class TransformerLayer(nn.Module): + """A single transformer layer. + + Transformer layer takes input with size [bsz, seq_length, hidden size] and returns an + output of the same size. + The input and output has same sbp sign, (S(0), B). + + Arguments: + hidden_size: size of hidden state. + ffn_hidden_size: size of feed forword neural network. + num_attention_heads: number of attention heads. + is_decoder: used to specify whether this is transformer encoder layer or transformer + decoder layer. Default: ``False``. + attention_dropout_prob: dropout probability of attention weights. + output_dropout_prob: dropout probability of output. + layernorm_epsilon: epsilon used in layernorm layer. Default: `1e-5`. + init_method: method to initialize the input layer weights. + output_layer_init_method: method to initialize the output layer weights. + If None, use `init_method`. + layer_idx: the layer index, which determines the placement. + """ + + def __init__( + self, + hidden_size, + ffn_hidden_size, + num_attention_heads, + head_size, + relative_attention_num_buckets, + is_decoder=False, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + drop_path_prob=0.0, + layernorm_epsilon=1e-5, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + padding_idx=None, + *, + layer_idx=0, + model_type="t5", + has_relative_attention_bias=False + ): + super().__init__() + self.hidden_size = hidden_size + self.ffn_hidden_size = ffn_hidden_size + self.num_attention_heads = num_attention_heads + self.head_size = head_size + self.attention_dropout_prob = attention_dropout_prob + self.output_dropout_prob = output_dropout_prob + self.layernorm_epsilon = layernorm_epsilon + self.layer_idx = layer_idx + self.is_decoder = is_decoder + + self.init_method = init_method + if output_layer_init_method is None: + output_layer_init_method = init_method + self.output_layer_init_method = output_layer_init_method + + self.drop_path = DropPath(drop_path_prob) if drop_path_prob > 0.0 else nn.Identity() + + self.input_layernorm = LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon, layer_idx=self.layer_idx + ) + + self.self_attention = self.build_attention( + is_cross_attention=False, + relative_attention_num_buckets=relative_attention_num_buckets, + padding_idx=padding_idx, + has_relative_attention_bias=has_relative_attention_bias, + is_decoder=self.is_decoder, + ) + self.post_attention_layernorm = LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon, layer_idx=self.layer_idx + ) + + if self.is_decoder: + self.cross_attention = self.build_attention( + is_cross_attention=True, + relative_attention_num_buckets=relative_attention_num_buckets, + padding_idx=padding_idx, + is_decoder=self.is_decoder, + ) + self.post_cross_attention_layernorm = LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon, layer_idx=self.layer_idx + ) + if model_type == "mt5": + self.mlp = MT5MLP( + self.hidden_size, + self.ffn_hidden_size, + self.output_dropout_prob, + self.init_method, + output_layer_init_method=self.output_layer_init_method, + layer_idx=self.layer_idx, + ) + elif model_type == "t5": + self.mlp = T5MLP( + self.hidden_size, + self.ffn_hidden_size, + self.output_dropout_prob, + self.init_method, + output_layer_init_method=self.output_layer_init_method, + layer_idx=self.layer_idx, + ) + + def forward( + self, + hidden_states, + attention_mask=None, + encoder_states=None, + encoder_attention_mask=None, + past_key_value=None, + use_cache=False, + position_bias=None, + encoder_decoder_position_bias=None, + ): + """ + Args: + hidden_states: shape is (batch_size, seq_length, hidden_size), + sbp signature is (S(0), B). + attention_mask: the combination of key padding mask and casual mask of hidden states + with shape (batch_size, 1, seq_length, seq_length) and the sbp + signature is (S(0), B), + encoder_states: encoder output with shape (batch_size, seq_length, hidden_size) + and the sbp signature is (S(0), B), which will be used in cross attention. + encoder_attention_mask: key padding mask of encoder states with shape + (batch_size, 1, seq_length, seq_length) and the sbp signature is (S(0), B). + past_key_value: tuple of key and value, each shape is + (seq_length, bsz, num_heads, head_size), For decoder layer, + the past_key_value contains the states both from self attention + and cross attention. + use_cache: it will be set to `True` when the model is in the inference phase and + used for incremental decoding. + """ + hidden_states = hidden_states.to_global(placement=dist.get_layer_placement(self.layer_idx)) + + if attention_mask is not None: + attention_mask = attention_mask.to_global( + placement=dist.get_layer_placement(self.layer_idx) + ) + + if past_key_value is not None: + if self.is_decoder: + assert len(past_key_value) == 4 + self_attn_past_key_value = past_key_value[:2] + cross_attn_past_key_value = past_key_value[2:] + else: + self_attn_past_key_value = past_key_value + cross_attn_past_key_value = None + else: + self_attn_past_key_value, cross_attn_past_key_value = None, None + + layernorm_output = self.input_layernorm(hidden_states) + + attention_output, position_bias = self.self_attention( + layernorm_output, + attention_mask=attention_mask, + past_key_value=self_attn_past_key_value, + position_bias=position_bias, + use_cache=use_cache, + ) + + attention_output = self.drop_path(attention_output) + + if use_cache: + attention_output, presents = attention_output + else: + presents = None + + hidden_states = hidden_states + attention_output + + layernorm_output = self.post_attention_layernorm(hidden_states) + + if self.is_decoder: + if presents is not None: + query_length = presents[0].shape[2] + else: + query_length = None + + attention_output, encoder_decoder_position_bias = self.cross_attention( + layernorm_output, + encoder_states, + attention_mask=encoder_attention_mask, + past_key_value=cross_attn_past_key_value, + position_bias=encoder_decoder_position_bias, + use_cache=use_cache, + query_length=query_length, + ) + if use_cache: + attention_output, decoder_presents = attention_output + presents = presents + decoder_presents + + attention_output = self.drop_path(attention_output) + + hidden_states = hidden_states + attention_output + layernorm_output = self.post_cross_attention_layernorm(hidden_states) + + mlp_output = self.mlp(layernorm_output) + mlp_output = self.drop_path(mlp_output) + + output = hidden_states + mlp_output + + if use_cache: + output = (output, presents) + output = (output,) + (position_bias,) + if self.is_decoder: + output = output + (encoder_decoder_position_bias,) + return output + + def build_attention( + self, + is_cross_attention=False, + relative_attention_num_buckets=None, + padding_idx=None, + has_relative_attention_bias=False, + is_decoder=False, + ): + return MultiheadAttention( + self.hidden_size, + self.num_attention_heads, + head_size=self.head_size, + relative_attention_num_buckets=relative_attention_num_buckets, + is_cross_attention=is_cross_attention, + attention_dropout_prob=self.attention_dropout_prob, + output_dropout_prob=self.output_dropout_prob, + init_method=self.init_method, + output_layer_init_method=self.output_layer_init_method, + padding_idx=padding_idx, + layer_idx=self.layer_idx, + has_relative_attention_bias=has_relative_attention_bias, + is_decoder=is_decoder, + ) diff --git a/projects/MT5/mt5_model.py b/projects/MT5/mt5_model.py new file mode 100644 index 0000000000000000000000000000000000000000..a16144ff267b3bfa4c8bafa38020dd18b91050d2 --- /dev/null +++ b/projects/MT5/mt5_model.py @@ -0,0 +1,440 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +import oneflow.nn as nn + +from libai.config import configurable +from libai.inference.generator.generation_utils import Generator +from libai.layers import Linear, LMLogits, RMSLayerNorm +from libai.models.utils import init_method_normal, scaled_init_method_normal +from libai.utils import distributed as dist +from projects.MT5.layers.embed_layer import MT5Embedding +from projects.MT5.layers.loss_layer import MT5Loss +from projects.MT5.layers.mask_layer import ExtendedMask +from projects.MT5.layers.transformer_layer import TransformerLayer +from projects.MT5.utils.mt5_loader import T5LoaderHuggerFace + + +class MT5Model(flow.nn.Module, Generator): + @configurable + def __init__( + self, + vocab_size, + hidden_size, + hidden_layers, + num_attention_heads, + head_size, + intermediate_size, + embedding_dropout_prob, + hidden_dropout_prob, + attention_probs_dropout_prob, + relative_attention_num_buckets, + padding_idx=None, + initializer_range=0.02, + layernorm_eps=1e-12, + amp_enabled=False, + model_type="mt5", + cfg=None, + ) -> None: + super().__init__() + self.cfg = cfg + self.model_type = model_type + init_method = init_method_normal(initializer_range) + scaled_init_method = scaled_init_method_normal(initializer_range, hidden_layers) + self.embedding = MT5Embedding( + hidden_size=hidden_size, + vocab_size=vocab_size, + embedding_dropout_prob=embedding_dropout_prob, + init_method=init_method, + amp_enabled=amp_enabled, + ) + self.extended_attn_mask = ExtendedMask() + + encoder_layers = flow.nn.ModuleList( + [ + TransformerLayer( + hidden_size=hidden_size, + ffn_hidden_size=intermediate_size, + num_attention_heads=num_attention_heads, + head_size=head_size, + relative_attention_num_buckets=relative_attention_num_buckets, + is_decoder=False, + attention_dropout_prob=attention_probs_dropout_prob, + output_dropout_prob=hidden_dropout_prob, + layernorm_epsilon=layernorm_eps, + init_method=init_method, + output_layer_init_method=scaled_init_method, + padding_idx=padding_idx, + layer_idx=i, + model_type=model_type, + has_relative_attention_bias=bool(i == 0), + ) + for i in range(hidden_layers) + ] + ) + + encoder_final_layernorm = RMSLayerNorm( + (hidden_size,), + eps=layernorm_eps, + layer_idx=hidden_layers - 1, + ) + + self.encoder = flow.nn.Sequential() + self.encoder.add_module("layers", encoder_layers) + self.encoder.add_module("final_layernorm", encoder_final_layernorm) + + decoder_layers = flow.nn.ModuleList( + [ + TransformerLayer( + hidden_size=hidden_size, + ffn_hidden_size=intermediate_size, + num_attention_heads=num_attention_heads, + head_size=head_size, + relative_attention_num_buckets=relative_attention_num_buckets, + is_decoder=True, + attention_dropout_prob=attention_probs_dropout_prob, + output_dropout_prob=hidden_dropout_prob, + layernorm_epsilon=layernorm_eps, + init_method=init_method, + output_layer_init_method=scaled_init_method, + padding_idx=padding_idx, + layer_idx=i, + model_type=model_type, + has_relative_attention_bias=bool(i - hidden_layers == 0), + ) + for i in range(hidden_layers, 2 * hidden_layers) + ] + ) + + decoder_final_layernorm = RMSLayerNorm( + (hidden_size,), + eps=layernorm_eps, + layer_idx=2 * hidden_layers - 1, + ) + + self.decoder = flow.nn.Sequential() + self.decoder.add_module("layers", decoder_layers) + self.decoder.add_module("final_layernorm", decoder_final_layernorm) + self.past_key_values = [None] * len(self.decoder.layers) + self.encoder_states = None + self.past_length = 0 + + if model_type == "mt5": + self.lm_head = Linear( + hidden_size, vocab_size, bias=False, layer_idx=2 * hidden_layers - 1 + ) + else: + self.lm_head = LMLogits(vocab_size, bias=False) + + @classmethod + def from_config(cls, cfg): + return { + "vocab_size": cfg.vocab_size, + "hidden_size": cfg.hidden_size, + "hidden_layers": cfg.hidden_layers, + "num_attention_heads": cfg.num_attention_heads, + "head_size": cfg.head_size, + "intermediate_size": cfg.intermediate_size, + "embedding_dropout_prob": cfg.embedding_dropout_prob, + "hidden_dropout_prob": cfg.hidden_dropout_prob, + "attention_probs_dropout_prob": cfg.attention_probs_dropout_prob, + "relative_attention_num_buckets": cfg.relative_attention_num_buckets, + "padding_idx": cfg.padding_idx, + "initializer_range": cfg.initializer_range, + "layernorm_eps": cfg.layernorm_eps, + "amp_enabled": cfg.amp_enabled, + "model_type": cfg.model_type, + "cfg": cfg, + } + + def forward( + self, + encoder_input_ids=None, + decoder_input_ids=None, + encoder_attn_mask=None, + decoder_attn_mask=None, + encoder_decoder_attn_mask=None, + use_cache=False, + only_encoder=False, + ): + + encoder_input_ids = ( + encoder_input_ids.to_global(placement=dist.get_layer_placement(0)) + if encoder_input_ids is not None + else encoder_input_ids + ) + decoder_input_ids = ( + decoder_input_ids.to_global(placement=dist.get_layer_placement(0)) + if decoder_input_ids is not None + else decoder_input_ids + ) + encoder_attn_mask = ( + encoder_attn_mask.to_global(placement=dist.get_layer_placement(0)) + if encoder_attn_mask is not None + else encoder_attn_mask + ) + decoder_attn_mask = ( + decoder_attn_mask.to_global(placement=dist.get_layer_placement(0)) + if decoder_attn_mask is not None + else decoder_attn_mask + ) + encoder_decoder_attn_mask = ( + encoder_decoder_attn_mask.to_global(placement=dist.get_layer_placement(0)) + if encoder_decoder_attn_mask is not None + else encoder_decoder_attn_mask + ) + + if use_cache and self.encoder_states is not None: + encoder_states = self.encoder_states + else: + position_bias = None + encoder_decoder_position_bias = None + self.set_cache(encoder_states=None, past_key_values=None) + encoder_attn_mask = self.extended_attn_mask(encoder_attn_mask) + enc_embedding_output = self.embedding(encoder_input_ids) + # transpose [batch_size, seq_len, embed_size] to [seq_len, batch_size, embed_size] + enc_hidden_states = enc_embedding_output.transpose(0, 1) + + for layer in self.encoder.layers: + enc_hidden_states, position_bias = layer( + enc_hidden_states, + encoder_attn_mask, + position_bias=position_bias, + ) + encoder_states = self.encoder.final_layernorm(enc_hidden_states) + + if only_encoder: + return encoder_states + + decoder_attn_mask = self.extended_attn_mask( + decoder_attn_mask, decoder_input_ids, is_decoder=True + ) + encoder_decoder_attn_mask = self.extended_attn_mask(encoder_decoder_attn_mask) + + dec_embedding_output = self.embedding(decoder_input_ids) + # transpose [batch_size, seq_len, embed_size] to [seq_len, batch_size, embed_size] + dec_hidden_states = dec_embedding_output.transpose(0, 1) + if use_cache: + presents = [] + + position_bias = None + encoder_decoder_position_bias = None + for layer, past_key_value in zip(self.decoder.layers, self.past_key_values): + dec_hidden_states, position_bias, encoder_decoder_position_bias = layer( + dec_hidden_states, + decoder_attn_mask, + encoder_states, + encoder_decoder_attn_mask, + past_key_value=past_key_value, + position_bias=position_bias, + encoder_decoder_position_bias=encoder_decoder_position_bias, + use_cache=use_cache, + ) + if use_cache: + dec_hidden_states, present = dec_hidden_states + presents.append(present) + if use_cache: + self.set_cache(encoder_states, past_key_values=presents) + + decoder_states = self.decoder.final_layernorm(dec_hidden_states) + + if self.cfg.tie_word_embeddings: + decoder_states = decoder_states * (self.cfg.hidden_size ** -0.5) + + if self.model_type == "mt5": + logits = self.lm_head(decoder_states) + else: + logits = self.lm_head(decoder_states, self.embedding.word_embeddings.weight) + + return {"logits": logits} + + def set_cache(self, encoder_states, past_key_values): + self.encoder_states = encoder_states + self.past_length = 0 if past_key_values is None else past_key_values[0][0].shape[2] + + if past_key_values is None: + past_key_values = [None] * len(self.decoder.layers) + assert len(past_key_values) == len(self.decoder.layers), ( + f"past_key_values's length {len(past_key_values)} doesn't match " + f"decoder num_layers' length {self.decoder.layers}" + ) + self.past_key_values = past_key_values + + def _reorder_cache(self, beam_idx): + past_key_values = self.past_key_values + reordered_decoder_past = () + for layer_past_states in past_key_values: + # get the correct batch idx from layer past batch dim + # batch dim of `past` is at 2nd position + reordered_layer_past_states = () + for layer_past_state in layer_past_states: + # need to set correct `past` for each of the four key / value states + beam_idx = beam_idx.to_global(placement=layer_past_state.placement) + reordered_layer_past_states = reordered_layer_past_states + ( + layer_past_state.index_select(0, beam_idx), + ) + + assert reordered_layer_past_states[0].shape == layer_past_states[0].shape + assert len(reordered_layer_past_states) == len(layer_past_states) + + reordered_decoder_past = reordered_decoder_past + (reordered_layer_past_states,) + return reordered_decoder_past + + def prepare_inputs_for_generation( + self, + input_ids, + past=None, + encoder_attn_mask=None, + encoder_decoder_attn_mask=None, + use_cache=None, + encoder_outputs=None, + ): + # cut decoder_input_ids if past is used + if past is not None: + input_ids = input_ids[:, -1:] + self.past_key_values = past + + self.encoder_states = encoder_outputs + decoder_attn_maks = flow.ones( + input_ids.size(), + dtype=flow.bool, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=flow.placement("cuda", list(range(dist.get_world_size()))), + ) + return { + "decoder_input_ids": input_ids, + "decoder_attn_mask": decoder_attn_maks, + "encoder_attn_mask": encoder_attn_mask, + "encoder_decoder_attn_mask": encoder_decoder_attn_mask, + "use_cache": use_cache, + } + + +class MT5ForPreTraining(flow.nn.Module): + def __init__(self, cfg) -> None: + super().__init__() + if cfg.pretrained_model_path is not None: + loader = T5LoaderHuggerFace(MT5Model, cfg, cfg.pretrained_model_path) + self.mt5_model = loader.load() + else: + self.mt5_model = MT5Model(cfg) + self.loss_func = MT5Loss() + + def set_cache(self, encoder_states, past_key_values): + self.mt5_model.set_cache(encoder_states, past_key_values) + + def forward( + self, + encoder_input_ids, + decoder_input_ids, + encoder_attn_mask, + decoder_attn_mask, + encoder_decoder_attn_mask, + lm_labels=None, + loss_mask=None, + use_cache=False, + ): + logits = self.mt5_model( + encoder_input_ids, + decoder_input_ids, + encoder_attn_mask, + decoder_attn_mask, + encoder_decoder_attn_mask, + use_cache=use_cache, + )["logits"] + # transpose [seq_len, batch_size, vocab_size] to [batch_size, seq_len, vocab_size] + logits = logits.transpose(0, 1) + if lm_labels is not None: + lm_loss = self.loss_func(logits, lm_labels, loss_mask) + return lm_loss + else: + return { + "prediction_scores": logits, + } + + @staticmethod + def set_pipeline_stage_id(model): + dist_utils = dist.get_dist_util() + + # Set pipeline parallelism stage_id + if hasattr(model.mt5_model.encoder.final_layernorm, "config"): + # Old API in OneFlow 0.8 + for module_block in model.modules(): + if isinstance(module_block.origin, MT5Embedding): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, ExtendedMask): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, TransformerLayer): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.origin, MT5Loss): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + model.mt5_model.encoder.final_layernorm.config.set_stage( + dist_utils.get_layer_stage_id(model.mt5_model.encoder.final_layernorm.layer_idx), + dist.get_layer_placement(model.mt5_model.encoder.final_layernorm.layer_idx), + ) + model.mt5_model.decoder.final_layernorm.config.set_stage( + dist_utils.get_layer_stage_id(model.mt5_model.decoder.final_layernorm.layer_idx), + dist.get_layer_placement(model.mt5_model.decoder.final_layernorm.layer_idx), + ) + else: + for module_block in model.modules(): + if isinstance(module_block.to(nn.Module), MT5Embedding): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), ExtendedMask): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.to(nn.Module), MT5Loss): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + model.mt5_model.encoder.final_layernorm.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(model.mt5_model.encoder.final_layernorm.layer_idx), + dist.get_layer_placement(model.mt5_model.encoder.final_layernorm.layer_idx), + ) + model.mt5_model.decoder.final_layernorm.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(model.mt5_model.decoder.final_layernorm.layer_idx), + dist.get_layer_placement(model.mt5_model.decoder.final_layernorm.layer_idx), + ) + + @staticmethod + def set_activation_checkpoint(model): + for module_block in model.modules(): + # Old API in OneFlow 0.8 + if hasattr(module_block, "origin"): + if isinstance(module_block.origin, TransformerLayer): + module_block.config.activation_checkpointing = True + else: + if isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(nn.graph.GraphModule).activation_checkpointing = True diff --git a/projects/MT5/readme.md b/projects/MT5/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..25f50cfda7650a4ef83d975d18f4b725a6fd23d5 --- /dev/null +++ b/projects/MT5/readme.md @@ -0,0 +1,54 @@ +# MT5 + +Reproduce T5Model and MT5Model with OneFlow, which effect are equivalent to HuggingFace's [T5](https://huggingface.co/docs/transformers/v4.19.4/en/model_doc/t5#overview) and [T5v1.1](https://github.com/google-research/text-to-text-transfer-transformer/blob/main/released_checkpoints.md#t511). + +## Introduce +The t5 and mt5 pretraining project can support 3D parallel and [ZERO](https://arxiv.org/abs/2202.10435). + +## Training MT5 +Training MT5 on 8 GPUs using 3D parallelism and ZERO. + +### 1. Prepare your training config file + +> set the pretrain parameters in `MT5/configs/mt5_pretrain.py`, such as `vocab_file` and `data_prefix`. + +> If you would like to use the t5 model, please set `model_type`="t5". + +### 2. Prepare the demo training data + +Prepare the demo training data by running: +```bash +# path/to/libai +wget https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/bert-base-chinese-vocab.txt -P ./data_test/bert_data/ +wget https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/loss_compara_content_sentence.bin -P ./data_test/bert_data/ +wget https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/loss_compara_content_sentence.idx -P ./data_test/bert_data/ +``` + +### 3. Prepare your own training data + +If you want to use your own training data, please skip the step2, and refer [Preprocessing Dataset](https://libai.readthedocs.io/en/latest/tutorials/basics/Preprocessing_Dataset.html#). + +```bash +IMPL=mmap +KEYS=text + +python tools/preprocess_data.py \ + --input /path/to/libai/projects/MT5/data/test.json \ + --json-keys ${KEYS} \ + --vocab-file /path/to/libai/projects/MT5/data/vocab.txt \ + --dataset-impl ${IMPL} \ + --tokenizer-name BertTokenizer \ + --do-lower-case \ + --do-chinese-wwm \ + --split-sentences \ + --output-prefix magic_prompt_${IMPL} \ + --workers 4 \ + --log-interval 2 + +``` + +### 4. Run the following code to start training +```bash +# cd /path/to/libai +bash tools/train.sh projects/MT5/train_net.py projects/MT5/configs/mt5_pretrain.py 8 +``` diff --git a/projects/MT5/train_net.py b/projects/MT5/train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..4cc34ef696b4e8f20c3db66bfcd463293b50581f --- /dev/null +++ b/projects/MT5/train_net.py @@ -0,0 +1,109 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import random +import sys + +import numpy as np +import oneflow as flow + +from libai.config import LazyConfig, default_argument_parser, try_get_key +from libai.engine import DefaultTrainer, default_setup +from libai.utils.checkpoint import Checkpointer +from libai.utils.events import JSONWriter, TensorboardXWriter +from projects.MT5.utils.mt5_metrc_printer import MT5MetricPrinter + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) + + +logger = logging.getLogger("libai." + __name__) + + +class Mt5Trainer(DefaultTrainer): + def __init__(self, cfg): + super().__init__(cfg) + + def build_writers(self): + """ + Build a list of writers to be used. By default it contains + writers that write metrics to the screen, + a json file, and a tensorboard event file respectively. + If you'd like a different list of writers, you can overwrite it in + your trainer. + + Returns: + list[EventWriter]: a list of :class:`EventWriter` objects. + + It is now implemented by: + + .. code-block:: python + + return [ + MT5MetricPrinter(self.global_batch_size, self.max_iter), + JSONWriter(os.path.join(self.cfg.train.output_dir, "metrics.json")), + TensorboardXWriter(self.cfg.train.output_dir), + ] + """ + # Assume the default print/log frequency. + return [ + # It may not always print what you want to see, since it prints "common" metrics only. + MT5MetricPrinter(self.global_batch_size, self.max_iter, self.cfg.train.log_period), + JSONWriter(os.path.join(self.cfg.train.output_dir, "metrics.json")), + TensorboardXWriter(self.cfg.train.output_dir), + ] + + +def main(args): + cfg = LazyConfig.load(args.config_file) + cfg = LazyConfig.apply_overrides(cfg, args.opts) + default_setup(cfg, args) + + seed_for_rank = cfg.train.seed + flow.env.get_rank() + flow.manual_seed(seed_for_rank) + flow.cuda.manual_seed(seed_for_rank) + np.random.seed(seed_for_rank) + random.seed(seed_for_rank) + + if args.fast_dev_run: + cfg.train.train_epoch = 0 + cfg.train.train_iter = 20 + cfg.train.evaluation.eval_period = 10 + cfg.train.log_period = 1 + + if args.eval_only: + tokenizer = None + if try_get_key(cfg, "tokenization") is not None: + tokenizer = Mt5Trainer.build_tokenizer(cfg) + model = Mt5Trainer.build_model(cfg) + Checkpointer(model, save_dir=cfg.train.output_dir).resume_or_load( + cfg.train.load_weight, resume=args.resume + ) + if try_get_key(cfg, "train.graph.enabled", default=False): + model = Mt5Trainer.build_graph(cfg, model, is_train=False) + test_loader = Mt5Trainer.build_test_loader(cfg, tokenizer) + if len(test_loader) == 0: + logger.info("No dataset in dataloader.test, please set dataset for dataloader.test") + _ = Mt5Trainer.test(cfg, test_loader, model) + return + + trainer = Mt5Trainer(cfg) + return trainer.train() + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + main(args) diff --git a/projects/MT5/utils/mt5_loader.py b/projects/MT5/utils/mt5_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..cf573064645ffcaa6dbaad5380c70702f6e1b762 --- /dev/null +++ b/projects/MT5/utils/mt5_loader.py @@ -0,0 +1,301 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import oneflow as flow + +from libai.models.utils import ModelLoaderHuggerFace, ModelLoaderLiBai + + +class T5LoaderHuggerFace(ModelLoaderHuggerFace): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + + """NOTE: base_model_prefix_1 is T5's prefix in Transformers. + base_model_prefix_2 is T5's prefix in LiBai.""" + self.base_model_prefix_1 = "transformer" + self.base_model_prefix_2 = "mt5_model" + + def _convert_state_dict(self, flow_state_dict, cfg): + """Convert state_dict's keys to match model. + + Args: + flow_state_dict (OrderedDict): model state dict. + cfg (dict): model's default config dict in LiBai. + + Returns: + OrderedDict: flow state dict. + """ + # The converted checkpoint. + oneflow_state_dict = flow_state_dict.copy() + old_keys = list(oneflow_state_dict.keys()) + # Get configs + num_heads = cfg.get("num_attention_heads") + hidden_size = cfg.get("hidden_size") + head_size = cfg.get("head_size", None) + if head_size is None: + head_size = int(hidden_size / num_heads) + + has_prefix = any(s.startswith(self.base_model_prefix_1) for s in oneflow_state_dict) + prefix1 = self.base_model_prefix_1 + "." if has_prefix else "" + prefix2 = self.base_model_prefix_2 + "." if has_prefix else "" + encoder_decoder_idx = 1 if has_prefix else 0 + layer_idx1 = 3 if has_prefix else 2 + layer_idx2 = 5 if has_prefix else 4 + op_idx = 6 if has_prefix else 5 + + # Convert T5's Embedding layers. + # NOTE: Transformers' T5 has no position embedding layer. + new_key = prefix2 + "embedding.word_embeddings.weight" + old_keys.remove(prefix1 + "shared.weight") + oneflow_state_dict[new_key] = oneflow_state_dict.pop(prefix1 + "shared.weight") + + # Convert T5's final_layer_norm + new_key = prefix2 + "encoder.final_layernorm.weight" + old_keys.remove(prefix1 + "encoder.final_layer_norm.weight") + oneflow_state_dict[new_key] = oneflow_state_dict.pop( + prefix1 + "encoder.final_layer_norm.weight" + ) + new_key = prefix2 + "decoder.final_layernorm.weight" + old_keys.remove(prefix1 + "decoder.final_layer_norm.weight") + oneflow_state_dict[new_key] = oneflow_state_dict.pop( + prefix1 + "decoder.final_layer_norm.weight" + ) + + # Convert MT5's lm_head + if cfg.model_type == "mt5" and "lm_head.weight" in oneflow_state_dict: + new_key = prefix2 + "lm_head.weight" + old_keys.remove("lm_head.weight") + oneflow_state_dict[new_key] = oneflow_state_dict.pop("lm_head.weight") + + # NOTE: Each layers has no bias in Transformer's T5. + for key in old_keys: + keys = key.split(".") + if layer_idx1 > len(keys) or layer_idx2 > len(keys): + continue + layer1 = keys[layer_idx1] + layer2 = keys[layer_idx2] + op_name = keys[op_idx] + + if keys[op_idx + 1] == "relative_attention_bias" and keys[op_idx] == "SelfAttention": + new_key = ( + prefix2 + + keys[encoder_decoder_idx] + + ".layers.0.self_attention.relative_attention_bias.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + # Convert T5's Encoder layers. + if keys[encoder_decoder_idx] == "encoder": + if op_name == "SelfAttention": + new_key = ( + prefix2 + + "encoder.layers." + + layer1 + + ".self_attention.query_key_value.weight" + ) + if new_key in oneflow_state_dict.keys(): + continue + q_w = ".".join(keys[: op_idx + 1]) + ".q." + "weight" + k_w = ".".join(keys[: op_idx + 1]) + ".k." + "weight" + v_w = ".".join(keys[: op_idx + 1]) + ".v." + "weight" + qkv_w = flow.cat( + ( + oneflow_state_dict.pop(q_w), + oneflow_state_dict.pop(k_w), + oneflow_state_dict.pop(v_w), + ), + dim=0, + ) + qkv_w = self._fix_qkv_ordering(qkv_w, head_size, num_heads, hidden_size) + oneflow_state_dict[new_key] = qkv_w + + o_w = ".".join(keys[: op_idx + 1]) + ".o." + "weight" + new_key = prefix2 + "encoder.layers." + layer1 + ".self_attention.dense.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(o_w) + elif op_name == "layer_norm": + if layer2 == "0": + new_key = prefix2 + "encoder.layers." + layer1 + ".input_layernorm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif layer2 == "1": + new_key = ( + prefix2 + + "encoder.layers." + + layer1 + + ".post_attention_layernorm.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif op_name == "DenseReluDense": + if cfg.get("model_type") == "t5": + if keys[op_idx + 1] == "wi": + new_key = ( + prefix2 + "encoder.layers." + layer1 + ".mlp.dense_h_to_4h.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif keys[op_idx + 1] == "wo": + new_key = ( + prefix2 + "encoder.layers." + layer1 + ".mlp.dense_4h_to_h.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif cfg.get("model_type") == "mt5": + if keys[op_idx + 1] == "wi_0": + new_key = prefix2 + "encoder.layers." + layer1 + ".mlp.wi_0.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif keys[op_idx + 1] == "wi_1": + new_key = prefix2 + "encoder.layers." + layer1 + ".mlp.wi_1.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif keys[op_idx + 1] == "wo": + new_key = prefix2 + "encoder.layers." + layer1 + ".mlp.wo.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + + # Convert T5's decoder Layers. + elif keys[encoder_decoder_idx] == "decoder": + if op_name == "SelfAttention": + new_key = ( + prefix2 + + "decoder.layers." + + layer1 + + ".self_attention.query_key_value.weight" + ) + if new_key in oneflow_state_dict.keys(): + continue + q_w = ".".join(keys[: op_idx + 1]) + ".q." + "weight" + k_w = ".".join(keys[: op_idx + 1]) + ".k." + "weight" + v_w = ".".join(keys[: op_idx + 1]) + ".v." + "weight" + qkv_w = flow.cat( + ( + oneflow_state_dict.pop(q_w), + oneflow_state_dict.pop(k_w), + oneflow_state_dict.pop(v_w), + ), + dim=0, + ) + qkv_w = self._fix_qkv_ordering(qkv_w, head_size, num_heads, hidden_size) + + oneflow_state_dict[new_key] = qkv_w + + o_w = ".".join(keys[: op_idx + 1]) + ".o." + "weight" + new_key = prefix2 + "decoder.layers." + layer1 + ".self_attention.dense.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(o_w) + elif op_name == "layer_norm": + if layer2 == "0": + new_key = prefix2 + "decoder.layers." + layer1 + ".input_layernorm.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif layer2 == "1": + new_key = ( + prefix2 + + "decoder.layers." + + layer1 + + ".post_attention_layernorm.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif layer2 == "2": + new_key = ( + prefix2 + + "decoder.layers." + + layer1 + + ".post_cross_attention_layernorm.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif op_name == "EncDecAttention": + new_key = prefix2 + "decoder.layers." + layer1 + ".cross_attention.query.weight" + if new_key in oneflow_state_dict.keys(): + continue + q_w = ".".join(keys[: op_idx + 1]) + ".q." + "weight" + k_w = ".".join(keys[: op_idx + 1]) + ".k." + "weight" + v_w = ".".join(keys[: op_idx + 1]) + ".v." + "weight" + + q_w = oneflow_state_dict.pop(q_w) + kv_w = flow.cat( + ( + oneflow_state_dict.pop(k_w), + oneflow_state_dict.pop(v_w), + ), + dim=0, + ) + q_w = self._fix_qkv_ordering(q_w, head_size, num_heads, hidden_size) + kv_w = self._fix_qkv_ordering(kv_w, head_size, num_heads, hidden_size) + + oneflow_state_dict[new_key] = q_w + new_key = ( + prefix2 + "decoder.layers." + layer1 + ".cross_attention.key_value.weight" + ) + oneflow_state_dict[new_key] = kv_w + + o_w = ".".join(keys[: op_idx + 1]) + ".o." + "weight" + new_key = prefix2 + "decoder.layers." + layer1 + ".cross_attention.dense.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(o_w) + elif op_name == "DenseReluDense": + if cfg.get("model_type") == "t5": + if keys[op_idx + 1] == "wi": + new_key = ( + prefix2 + "decoder.layers." + layer1 + ".mlp.dense_h_to_4h.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif keys[op_idx + 1] == "wo": + new_key = ( + prefix2 + "decoder.layers." + layer1 + ".mlp.dense_4h_to_h.weight" + ) + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif cfg.get("model_type") == "mt5": + if keys[op_idx + 1] == "wi_0": + new_key = prefix2 + "decoder.layers." + layer1 + ".mlp.wi_0.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif keys[op_idx + 1] == "wi_1": + new_key = prefix2 + "decoder.layers." + layer1 + ".mlp.wi_1.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + elif keys[op_idx + 1] == "wo": + new_key = prefix2 + "decoder.layers." + layer1 + ".mlp.wo.weight" + oneflow_state_dict[new_key] = oneflow_state_dict.pop(key) + return oneflow_state_dict + + def _load_config_from_json(self, config_file): + """load config from `config.json`, and update default config. + + Args: + config_file (str): Path of config file. + """ + with open(config_file, mode="r", encoding="utf-8") as f: + cfg_dict = json.load(f) + + self._update_cfg("vocab_size", cfg_dict["vocab_size"]) + self._update_cfg("hidden_size", cfg_dict["d_model"]) + self._update_cfg("hidden_layers", cfg_dict["num_layers"]) + self._update_cfg("num_attention_heads", cfg_dict["num_heads"]) + self._update_cfg("intermediate_size", cfg_dict["d_ff"]) + self._update_cfg("hidden_dropout_prob", cfg_dict["dropout_rate"]) + self._update_cfg("attention_probs_dropout_prob", cfg_dict["dropout_rate"]) + self._update_cfg( + "relative_attention_num_buckets", cfg_dict["relative_attention_num_buckets"] + ) + self._update_cfg("embedding_dropout_prob", cfg_dict["dropout_rate"]) + self._update_cfg("initializer_range", cfg_dict["initializer_factor"]) + self._update_cfg("layernorm_eps", cfg_dict["layer_norm_epsilon"]) + self._update_cfg("head_size", cfg_dict["d_kv"]) + if "tie_word_embeddings" in self.libai_cfg: + self._update_cfg("tie_word_embeddings", cfg_dict.get("tie_word_embeddings", True)) + + # update libai_cfg by kwargs + for k, v in self.kwargs.items(): + self._update_cfg(k, v) + + self._update_cfg_log() + + +class T5LoaderLibai(ModelLoaderLiBai): + def __init__(self, model, libai_cfg, pretrained_model_path, **kwargs): + super().__init__(model, libai_cfg, pretrained_model_path, **kwargs) + self.base_model_prefix_2 = "mt5_model" diff --git a/projects/MT5/utils/mt5_metrc_printer.py b/projects/MT5/utils/mt5_metrc_printer.py new file mode 100644 index 0000000000000000000000000000000000000000..9e6ce85d6adbe2828e64b0ed4a0b16ae4506be81 --- /dev/null +++ b/projects/MT5/utils/mt5_metrc_printer.py @@ -0,0 +1,112 @@ +import datetime +import logging +import time + +from libai.utils.events import EventWriter, get_event_storage + + +class MT5MetricPrinter(EventWriter): + """ + Print **MT5** metrics to the terminal, including + iteration time, ETA, memory, all losses, and the learning rate. + It also applies smoothing using a window of 20 elements. + It's meant to print MT5 metrics in MT5 ways. + To print something in more customized ways, please implement a similar printer by yourself. + """ + + def __init__(self, batch_size, max_iter, log_period): + """ + Args: + max_iter (int): the maximum number of iterations to train. + Used to compute ETA. + """ + self.logger = logging.getLogger("libai." + __name__) + self._batch_size = batch_size + self._max_iter = max_iter + self._last_write = None + self._log_period = log_period + + def write(self): + storage = get_event_storage() + iteration = storage.iter + consumed_samples = storage.samples + + try: + done_tokens = storage.history("done_tokens").avg(self._log_period) + token_time = storage.history("time").avg(self._log_period) + except KeyError: + done_tokens = None + + try: + correct_tokens = storage.history("correct_tokens").avg(self._log_period) + denominator = storage.history("denominator").avg(self._log_period) + acc_mlm = correct_tokens / denominator + except KeyError: + acc_mlm = None + + if iteration == self._max_iter: + # This hook only reports training progress (loss, ETA, etc) but not other data, + # therefore do not write anything after training succeeds, even if this method + # is called. + return + + try: + data_time = storage.history("data_time").avg(self._log_period) + except KeyError: + # they may not exist in the first few iterations (due to warmup) + # or when SimpleTrainer is not used + data_time = None + + eta_string = None + try: + iter_time = storage.history("time").global_avg() + eta_seconds = storage.history("time").median(1000) * (self._max_iter - iteration - 1) + storage.put_scalar("eta_seconds", eta_seconds, smoothing_hint=False) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + except KeyError: + iter_time = None + # estimate eta on our own - more noisy + if self._last_write is not None: + estimate_iter_time = (time.perf_counter() - self._last_write[1]) / ( + iteration - self._last_write[0] + ) + eta_seconds = estimate_iter_time * (self._max_iter - iteration - 1) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + self._last_write = (iteration, time.perf_counter()) + + try: + lr = "{:.2e}".format(storage.history("lr").latest()) + except KeyError: + lr = "N/A" + + max_mem_mb = None + + # NOTE: max_mem is parsed by grep in "dev/parse_results.sh" + self.logger.info( + " {eta} {iter} {sample} {losses} {time} {data_time} {tpt} lr: {lr} {memory} " + " {tokens_speed} {acc_mlm}".format( + eta=f"eta: {eta_string}" if eta_string else "", + iter=f"iteration: {iteration}/{self._max_iter}", + sample=f"consumed_samples: {consumed_samples}", + losses=" ".join( + [ + "{}: {:.4g}".format(k, v.median(200)) + for k, v in storage.histories().items() + if "loss" in k + ] + ), + time="time: {:.4f} s/iter ".format(iter_time) if iter_time is not None else "", + data_time="data_time: {:.4f} s/iter".format(data_time) + if data_time is not None + else "", + tpt="total_throughput: {:.2f} samples/s".format(self._batch_size / iter_time) + if iter_time is not None + else "", + lr=lr, + memory="max_mem: {:.0f}M".format(max_mem_mb) if max_mem_mb is not None else "", + tokens_speed="tokens_throughput: {:.4f} tokens/s".format(done_tokens / token_time) + if done_tokens is not None + else "", + acc_mlm="acc_mlm: {:.4f}".format(acc_mlm) if acc_mlm is not None else "", + ) + ) diff --git a/projects/MagicPrompt/configs/gpt2_dataset.py b/projects/MagicPrompt/configs/gpt2_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..ef5c75735bfb6c7b55e4099da1a01f33f291bf1d --- /dev/null +++ b/projects/MagicPrompt/configs/gpt2_dataset.py @@ -0,0 +1,37 @@ +from libai.config import LazyCall +from libai.data import build_nlp_train_val_test_loader +from configs.common.data.gpt_dataset import tokenization, dataloader +from libai.tokenizer import GPT2Tokenizer +from libai.data.datasets import GPT2Dataset +from libai.data.data_utils import get_indexed_dataset + + +data_prefix = "/data/home/magicprompt/train/en_train_mmap_text_sentence" + +tokenization.tokenizer = LazyCall(GPT2Tokenizer)( + vocab_file="/data/home/magicprompt/vocab.json", + merges_file="/data/home/magicprompt/merges.txt", + do_lower_case=True, + do_chinese_wwm=True, +) +tokenization.append_eod = False + +dataloader.train = LazyCall(build_nlp_train_val_test_loader)( + dataset=[ + LazyCall(GPT2Dataset)( + name="gpt-2", + data_prefix=data_prefix, + indexed_dataset=LazyCall(get_indexed_dataset)( + data_prefix=data_prefix, + data_impl="mmap", + skip_warmup=False, + ), + max_seq_length=1024, + seed=1234, + ), + ], + train_val_test_num_samples=None, # a hint for deferred assignment + splits=[[949.0, 50.0, 1.0]], + weights=[1.0], + num_workers=4, +) diff --git a/projects/MagicPrompt/configs/gpt2_inference.py b/projects/MagicPrompt/configs/gpt2_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..a78b51f7804215e4f11121d8a9cdf5a2f0061ff3 --- /dev/null +++ b/projects/MagicPrompt/configs/gpt2_inference.py @@ -0,0 +1,69 @@ +from configs.common.models.gpt import cfg +from libai.config import LazyCall +from libai.tokenizer.tokenization_gpt2 import GPT2Tokenizer +from projects.MagicPrompt.gpt2 import GPTModel, GPTForPreTraining +from configs.common.data.gpt_dataset import tokenization +from configs.common.train import train + + +cfg.update( + # Model + embedding_dropout_prob=0, + attention_dropout_prob=0, + output_dropout_prob=0, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + apply_residual_post_layernorm=False, + amp_enabled=False, + num_attention_heads=12, + hidden_size=768, + ffn_hidden_size=4 * 768, + hidden_layers=12, + max_seq_length=1024, + initializer_range=0.02, + vocab_size=50304, + # Inference + is_encoder_decoder=False, + max_length=20, + min_length=0, + do_sample=False, + early_stopping=False, + num_beams=1, + num_beam_groups=1, + diversity_penalty=0.0, + temperature=1.0, + top_k=50, + top_p=1.0, + typical_p=1.0, + repetition_penalty=1.0, + length_penalty=1.0, + no_repeat_ngram_size=0, + encoder_no_repeat_ngram_size=0, + num_return_sequences=1, + chunk_size_feed_forward=0, + output_scores=False, + forced_bos_token_id=None, + forced_eos_token_id=None, + remove_invalid_values=False, + exponential_decay_length_penalty=None, + use_cache=True, + # Tokenizer + pad_token_id=0, + eos_token_id=50256, + bos_token_id=50256, + sep_token_id=None, + decoder_start_token_id=None, + # train + pretrained_model_path="/data/home/magicprompt", +) + + +model = LazyCall(GPTModel)(cfg=cfg) +pretrain_model = LazyCall(GPTForPreTraining)(cfg=cfg) +tokenization.tokenizer = LazyCall(GPT2Tokenizer)( + vocab_file="/data/home/magicprompt/vocab.json", + merges_file="/data/home/magicprompt/merges.txt", + add_bos_token=True, +) diff --git a/projects/MagicPrompt/configs/gpt2_training.py b/projects/MagicPrompt/configs/gpt2_training.py new file mode 100644 index 0000000000000000000000000000000000000000..69406f1a6e201337c3cc6b19df4439d45411b7c8 --- /dev/null +++ b/projects/MagicPrompt/configs/gpt2_training.py @@ -0,0 +1,83 @@ +from libai.config import LazyCall +from libai.evaluation import PPLEvaluator +from projects.MagicPrompt.configs.gpt2_inference import pretrain_model as model +from projects.MagicPrompt.configs.gpt2_dataset import dataloader, tokenization +from configs.common.optim import optim + +from libai.scheduler import WarmupExponentialLR + +from configs.common.train import train +from configs.common.models.graph import graph + + +vocab_file = "/data/home/magicprompt/vocab.json" +merge_files = "/data/home/magicprompt/merges.txt" +train_data_prefix = "/data/home/magicprompt/train/en_train_mmap_text_sentence" + +tokenization.tokenizer.vocab_file = vocab_file +tokenization.tokenizer.merges_file = merge_files +dataloader.train.dataset[0].data_prefix = train_data_prefix +dataloader.train.dataset[0].indexed_dataset.data_prefix = train_data_prefix + +# gpt2 model config +model.cfg.pretrained_model_path = None +model.cfg.embedding_dropout_prob = 0.1 +model.cfg.attention_dropout_prob = 0.1 +model.cfg.output_dropout_prob = 0.1 +model.cfg.num_attention_heads = 12 +model.cfg.hidden_size = 768 +model.cfg.ffn_hidden_size = 4 * 768 +model.cfg.hidden_layers = 12 +model.cfg.max_seq_length = 1024 +model.cfg.initializer_range = 0.02 +model.cfg.vocab_size = 50257 +model.cfg.layernorm_epsilon = 1e-5 +model.cfg.use_scaled_init_for_output_weights = True +model.cfg.bias_gelu_fusion = True +model.cfg.bias_dropout_fusion = True +model.cfg.scale_mask_softmax_fusion = True +model.cfg.apply_query_key_layer_scaling = True +model.cfg.apply_residual_post_layernorm = False +model.cfg.amp_enabled = True + +train.input_placement_device = "cpu" + +train.dist.pipeline_num_layers = model.cfg.hidden_layers + +for ds in dataloader.train.dataset: + ds.max_seq_length = model.cfg.max_seq_length + +optim.lr = 5.0e-05 + +train.update( + dict( + output_dir="projects/MagicPrompt/oneflow_magicprompt", + train_micro_batch_size=4, + test_micro_batch_size=4, + train_epoch=33, + train_iter=10000, + log_period=50, + amp=dict(enabled=True), + warmup_ratio=0, + checkpointer=dict(period=8000, max_to_keep=20), + dist=dict( + data_parallel_size=1, + tensor_parallel_size=1, + pipeline_parallel_size=1, + # pipeline_num_layers=2 * model.cfg.hidden_layers, + ), + scheduler=LazyCall(WarmupExponentialLR)( + warmup_factor=0.0, + gamma=1.0, + warmup_method="linear", + warmup_iter=0.0, + ), + evaluation=dict( + enabled=True, + evaluator=LazyCall(PPLEvaluator)(), + eval_iter=250, + eval_period=4000, + ), + rdma_enabled=False, + ) +) diff --git a/projects/MagicPrompt/datasets/datasets.py b/projects/MagicPrompt/datasets/datasets.py new file mode 100644 index 0000000000000000000000000000000000000000..e1874f6fa614922927ddfca968fb5c2b6ecc01a7 --- /dev/null +++ b/projects/MagicPrompt/datasets/datasets.py @@ -0,0 +1,25 @@ +import os + + +def convert_txt2json(file_path): + """Store the dataset in loose JSON format file, you can refer: + https://libai.readthedocs.io/en/latest/tutorials/basics/Preprocessing_Dataset.html + """ + filename, ext = os.path.splitext(file_path) + filename = filename.split("/")[-1] + + with open(file_path) as f: + lines = f.readlines() + print(len(lines)) + + target_file = "/home/xiezipeng/libai/projects/MagicPrompt/" + filename + "_magicprompy.txt" + with open(target_file, "w", encoding="utf-8") as f: + for line in lines: + line = "{" + '"' + "text" + '"' + ": " + '"' + line.strip() + '"' + "}" + "\n" + f.write(line) + os.rename(target_file, target_file[:-4] + ".json") + + +if __name__ == "__main__": + convert_txt2json("/path/to/test.txt") + convert_txt2json("/path/to/train.txt") diff --git a/projects/MagicPrompt/gpt2.py b/projects/MagicPrompt/gpt2.py new file mode 100644 index 0000000000000000000000000000000000000000..03ed0f336970b8a96858dc0ceb78164354e5e13b --- /dev/null +++ b/projects/MagicPrompt/gpt2.py @@ -0,0 +1,430 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +from oneflow import nn +from oneflow.nn import init + +from libai.config import configurable +from libai.inference.generator.generation_utils import Generator +from libai.layers import Embedding, LayerNorm, LMLogits, VocabEmbedding +from libai.layers.attention import AttnMaskType +from libai.models.gpt_model import GPTLoss +from libai.models.utils import GPT2LoaderHuggerFace, init_method_normal, scaled_init_method_normal +from libai.utils import distributed as dist +from projects.MagicPrompt.layers.transformer_layer import TransformerLayer + + +class GPTModel(nn.Module, Generator): + """GPT-2 language model. The output of the forward method is logits. + + Args: + hidden_layers (int): The number of ``TransformerLayer`` in the gpt model. + vocab_size (int): The size of vocabulary file. + hidden_size (int): The size of hidden states. + ffn_hidden_size (int): + The size of intermediate layer in feed-forward network for each ``TransformerLayer``. + num_attention_heads (int): + The number of attention heads for each attention layer of ``TransformerLayer``. + max_seq_length (int, optional): + Max sequence length of input, defines the shape of Position Embeddings in GPTEmebedding. + Defaults to 1024. + embedding_dropout_prob (float, optional): + The dropout ratio for the output of GPTEmbedding Layer. Defaults to 0.0. + attention_dropout_prob (float, optional): + The dropout ratio for the output of each attention layer in ``TransformerLayer``. + Defaults to 0.0. + output_dropout_prob (float, optional): + The dropout ratio for the output for each TransformerLayer. Defaults to 0.0. + layernorm_epsilon (float, optional): + The epsilon of LayerNorm layer. Defaults to 1e-5. + initializer_range (float, optional): + Sigma of the normal distribution in the initialization method. Defaults to 0.02. + use_scaled_init_for_output_weights (bool, optional): Defaults to ``True``. + bias_gelu_fusion (bool, optional): + Whether or not to fuse the computing of bias and gelu. Defaults to ``False``. + bias_dropout_fusion (bool, optional): + Whether or not to fuse the computing of dropout and bias. Defaults to ``False``. + scale_mask_softmax_fusion (bool, optional): + Whether to fuse the computing of mask and softmax in attention layers. + Defaults to ``False``. + apply_query_key_layer_scaling (bool, optional): + Whether or not to use layer index related scaling in computing attention scores. + If ``True``, the scaling factor equals to sqrt(d) * (layer_index + 1). + Defaults to ``False``. + apply_residual_post_layernorm (bool, optional): + If set ``True``, use original BERT residual connection ordering otherwise use Megatron + BERT residual connection which is more stable when scaling model size introduced in + https://arxiv.org/pdf/1909.08053.pdf. + Default: ``False``. + amp_enabled (bool, optional): + Whether or not to set fp16 for embedding weight in T5 model. Defaults to ``False``. + """ + + @configurable + def __init__( + self, + hidden_layers, + vocab_size, + hidden_size, + ffn_hidden_size, + num_attention_heads, + max_seq_length=1024, + embedding_dropout_prob=0.0, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + layernorm_epsilon=1e-5, + initializer_range=0.02, + use_scaled_init_for_output_weights=True, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + apply_residual_post_layernorm=False, + amp_enabled=False, + cfg=None, + ): + super().__init__() + self.cfg = cfg + init_method = init_method_normal(sigma=initializer_range) + if use_scaled_init_for_output_weights: + output_layer_init_method = scaled_init_method_normal(initializer_range, hidden_layers) + else: + output_layer_init_method = init_method + + self.embeddings = GPTEmbedding( + vocab_size, + hidden_size, + max_seq_length, + init_method=init_method, + embedding_dropout_prob=embedding_dropout_prob, + amp_enabled=amp_enabled, + ) + + self.transformer = Transformer( + hidden_layers, + hidden_size, + ffn_hidden_size, + num_attention_heads, + attention_dropout_prob=attention_dropout_prob, + output_dropout_prob=output_dropout_prob, + layernorm_epsilon=layernorm_epsilon, + init_method=init_method, + output_layer_init_method=output_layer_init_method, + bias_gelu_fusion=bias_gelu_fusion, + bias_dropout_fusion=bias_dropout_fusion, + scale_mask_softmax_fusion=scale_mask_softmax_fusion, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + apply_residual_post_layernorm=apply_residual_post_layernorm, + set_cache=self.set_cache, + ) + + self.past_key_values = [None] * hidden_layers + self.past_length = 0 + + self.lm_head = LMLogits(vocab_size, bias=False) + + @classmethod + def from_config(cls, cfg): + return { + "hidden_layers": cfg.hidden_layers, + "vocab_size": cfg.vocab_size, + "hidden_size": cfg.hidden_size, + "ffn_hidden_size": cfg.ffn_hidden_size, + "num_attention_heads": cfg.num_attention_heads, + "max_seq_length": cfg.max_seq_length, + "embedding_dropout_prob": cfg.embedding_dropout_prob, + "attention_dropout_prob": cfg.attention_dropout_prob, + "output_dropout_prob": cfg.output_dropout_prob, + "layernorm_epsilon": cfg.layernorm_epsilon, + "initializer_range": cfg.initializer_range, + "use_scaled_init_for_output_weights": cfg.use_scaled_init_for_output_weights, + "bias_gelu_fusion": cfg.bias_gelu_fusion, + "bias_dropout_fusion": cfg.bias_dropout_fusion, + "scale_mask_softmax_fusion": cfg.scale_mask_softmax_fusion, + "apply_query_key_layer_scaling": cfg.apply_query_key_layer_scaling, + "apply_residual_post_layernorm": cfg.apply_residual_post_layernorm, + "amp_enabled": cfg.amp_enabled, + "cfg": cfg, + } + + def forward(self, input_ids, use_cache=False): + """ + + Args: + input_ids (flow.LongTensor): Indices of input sequence tokens in vocabulary. + + Returns: + flow.Tensor: logits + """ + + input_ids = input_ids.to_global(placement=dist.get_layer_placement(0)) + + if use_cache and self.past_key_values[0] is not None: + self.past_length = self.past_key_values[0][0].size(-2) + else: + self.past_length = 0 + + input_embeds = self.embeddings(input_ids, self.past_length) + + transformer_output = self.transformer( + input_embeds, + attention_mask=None, + past_key_values=self.past_key_values, + use_cache=use_cache, + ) + + logits = self.lm_head(transformer_output, self.embeddings.token_embeddings.weight) + + return {"logits": logits} + + def set_cache(self, past_key_values): + self.past_length = 0 if past_key_values is None else past_key_values[0][0].shape[2] + + if past_key_values is None: + past_key_values = [None] * self.cfg.hidden_layers + + assert len(past_key_values) == self.cfg.hidden_layers, ( + f"past_key_values's length {len(past_key_values)} doesn't match " + f"num_layers:' {self.cfg.hidden_layers}" + ) + + self.past_key_values = past_key_values + + def _reorder_cache(self, beam_idx): + past_key_values = self.past_key_values + return tuple( + tuple( + past_state.index_select(0, beam_idx.to(past_state.device)) + for past_state in layer_past + ) + for layer_past in past_key_values + ) + + def prepare_inputs_for_generation( + self, + input_ids, + past=None, + use_cache=None, + ): + if past is not None: + input_ids = input_ids[:, -1:] + self.past_key_values = past + + return {"input_ids": input_ids, "use_cache": use_cache} + + +class GPTEmbedding(nn.Module): + def __init__( + self, + vocab_size, + hidden_size, + max_seq_length, + init_method=init.xavier_normal_, + embedding_dropout_prob=0.0, + amp_enabled=False, + ): + super().__init__() + self.token_embeddings = VocabEmbedding( + vocab_size, hidden_size, init_method=init_method, amp_enabled=amp_enabled + ) + self.position_embeddings = Embedding( + max_seq_length, hidden_size, init_method=init_method, amp_enabled=amp_enabled + ) + self.dropout = nn.Dropout(embedding_dropout_prob) + + self.position_ids = flow.arange( + max_seq_length, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ).unsqueeze(0) + + def forward(self, input_ids, past_length=0): + bsz, seq_length = input_ids.size() + + position_ids = self.position_ids[:, past_length : past_length + seq_length] + position_ids = position_ids.expand_as(input_ids).to_global(sbp=input_ids.sbp) + + token_embeds = self.token_embeddings(input_ids) + position_embeds = self.position_embeddings(position_ids) + input_embeds = token_embeds + position_embeds + input_embeds = self.dropout(input_embeds) + return input_embeds + + +class Transformer(nn.Module): + def __init__( + self, + hidden_layers, + hidden_size, + ffn_hidden_size, + num_attention_heads, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + layernorm_epsilon=1e-5, + init_method=init.xavier_normal_, + output_layer_init_method=None, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + apply_residual_post_layernorm=False, + set_cache=None, + ): + super().__init__() + self.hidden_layers = hidden_layers + self.set_cache = set_cache + + def build_layer(layer_number): + return TransformerLayer( + hidden_size, + ffn_hidden_size, + num_attention_heads, + attention_dropout_prob=attention_dropout_prob, + output_dropout_prob=output_dropout_prob, + layernorm_epsilon=layernorm_epsilon, + init_method=init_method, + output_layer_init_method=output_layer_init_method, + bias_gelu_fusion=bias_gelu_fusion, + bias_dropout_fusion=bias_dropout_fusion, + scale_mask_softmax_fusion=scale_mask_softmax_fusion, + apply_query_key_layer_scaling=apply_query_key_layer_scaling, + apply_residual_post_layernorm=apply_residual_post_layernorm, + attn_mask_type=AttnMaskType.causal, + layer_idx=layer_number, + ) + + self.layers = nn.ModuleList([build_layer(i) for i in range(self.hidden_layers)]) + self.layernorm_f = LayerNorm(hidden_size, eps=layernorm_epsilon, layer_idx=-1) + + def forward(self, hidden_states, attention_mask, past_key_values=None, use_cache=False): + if use_cache: + presents = [] + + for layer, past_key_value in zip(self.layers, past_key_values): + hidden_states = layer( + hidden_states, + attention_mask, + past_key_value=past_key_value, + use_cache=use_cache, + ) + if use_cache: + hidden_states, present = hidden_states + presents.append(present) + + output = self.layernorm_f(hidden_states) + + if use_cache: + self.set_cache(presents) + + return output + + +class GPTForPreTraining(nn.Module): + """ + GPT Model with classification head on top. + """ + + def __init__(self, cfg) -> None: + super().__init__() + if cfg.pretrained_model_path is not None: + loader = GPT2LoaderHuggerFace(GPTModel, cfg, cfg.pretrained_model_path) + self.GPT_model = loader.load() + else: + self.GPT_model = GPTModel(cfg) + + self.loss_func = GPTLoss() + + def forward( + self, + input_ids, + labels=None, + ): + """ + + Args: + input_ids (flow.LongTensor): Indices of input sequence tokens in vocabulary. + labels (flow.LongTensor, optional): Labels for computing language modeling loss. + None for evaluating. Defaults to None. + + Returns: + dict: + A dict containing :code:`loss_value` or :code:`logits` + depending on training or evaluation. + :code:`{"masked_lm_loss": loss_value}` when training, + :code:`{"prediction_scores": logits}` when evaluating. + """ + logits = self.GPT_model(input_ids)["logits"] + if labels is not None: + lm_loss = self.loss_func(logits, labels) + return lm_loss + else: + return {"prediction_scores": logits} + + @staticmethod + def set_pipeline_stage_id(model: nn.Module): + dist_utils = dist.get_dist_util() + + if hasattr(model.GPT_model.transformer.layernorm_f, "config"): + # Old API in OneFlow 0.8 + for module_block in model.modules(): + if isinstance(module_block.origin, (GPTEmbedding)): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, TransformerLayer): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.origin, (LMLogits, GPTLoss)): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + model.GPT_model.transformer.layernorm_f.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + else: + for module_block in model.modules(): + if isinstance(module_block.to(nn.Module), (GPTEmbedding)): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.to(nn.Module), (LMLogits, GPTLoss)): + module_block.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + model.GPT_model.transformer.layernorm_f.to(nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + @staticmethod + def set_activation_checkpoint(model): + for module_block in model.modules(): + # Old API in OneFlow 0.8 + if hasattr(module_block, "origin"): + if isinstance(module_block.origin, TransformerLayer): + module_block.config.activation_checkpointing = True + else: + if isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(nn.graph.GraphModule).activation_checkpointing = True diff --git a/projects/MagicPrompt/layers/__init__.py b/projects/MagicPrompt/layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/projects/MagicPrompt/layers/attention_layer.py b/projects/MagicPrompt/layers/attention_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..c8859b41b8802b75c596fef9abca2b21e8ef7dcd --- /dev/null +++ b/projects/MagicPrompt/layers/attention_layer.py @@ -0,0 +1,276 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import Tuple + +import oneflow as flow +from oneflow import nn + +from libai.layers.attention import AttnMaskType +from libai.layers.linear import Linear +from libai.utils import distributed as dist + + +class MultiheadAttention(nn.Module): + """Multi-head attention layer, support self attention and cross attention. + + Args: + hidden_size: size of hidden state. + num_attention_heads: number of attention heads. + is_cross_attention: used to specify whether it is self attention or cross attention. + Defaults to False. + attention_dropout_prob: dropout probability of attention weights. + Defaults to 0.0. + output_dropout_prob: dropout probability of output. Defaults to 0.0. + init_method: method to initialize the input layer weights. + Defaults to ``init.xavier_normal_``. + output_layer_init_method: method to initialize the output layer weights. + If None, use ``init_method``. + bias_dropout_fusion: whether to fuse add bias and dropout. + Defaults to False. + scale_mask_softmax_fusion: whether to fuse scale, mask and softmax. + Defaults to False. + apply_query_key_layer_scaling: if `True`, scaling the attention score by layer index. + Defaults to False. + layer_idx: a layer_idx sign which determines the placements. + It will be used in pipeline parallelism. Defaults to 0. + """ + + def __init__( + self, + hidden_size, + num_attention_heads, + is_cross_attention=False, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + attn_mask_type=AttnMaskType.padding, + *, + layer_idx=0 + ): + super().__init__() + self.hidden_size = hidden_size + if output_layer_init_method is None: + output_layer_init_method = init_method + + assert ( + hidden_size % num_attention_heads == 0 + ), "hidden_size must be divisible by num_attention_heads." + + self.num_heads = num_attention_heads + self.head_size = hidden_size // num_attention_heads + self.attn_mask_type = attn_mask_type + + self.attention_dropout_prob = attention_dropout_prob + self.dropout = nn.Dropout(p=attention_dropout_prob) + self.norm_factor = 1.0 / math.sqrt(float(self.head_size)) + self.coeff = None + if apply_query_key_layer_scaling: + self.coeff = layer_idx + 1 + self.norm_factor /= self.coeff + + self.is_cross_attention = is_cross_attention + self.scale_mask_softmax_fusion = scale_mask_softmax_fusion + self.bias_dropout_fusion = bias_dropout_fusion + + if self.bias_dropout_fusion: + self.output_dropout_prob = output_dropout_prob + else: + self.output_dropout = nn.Dropout(p=output_dropout_prob) + + if self.is_cross_attention: + self.query = Linear( + self.hidden_size, + self.hidden_size, + parallel="col", + init_method=init_method, + layer_idx=layer_idx, + ) + self.key_value = Linear( + self.hidden_size, + self.hidden_size * 2, + parallel="col", + init_method=init_method, + layer_idx=layer_idx, + ) + else: + self.query_key_value = Linear( + self.hidden_size, + self.hidden_size * 3, + parallel="col", + init_method=init_method, + layer_idx=layer_idx, + ) + + self.dense = Linear( + self.hidden_size, + self.hidden_size, + parallel="row", + init_method=output_layer_init_method, + skip_bias_add=self.bias_dropout_fusion, + layer_idx=layer_idx, + ) + + self.bias = flow.tril(flow.ones((1024, 1024), dtype=flow.uint8)).view(1, 1, 1024, 1024) + self.bias = self.bias.to_global( + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(layer_idx), + ) + + def forward( + self, + hidden_states: flow.Tensor, + encoder_states: flow.Tensor = None, + attention_mask: flow.Tensor = None, + past_key_value: Tuple[flow.Tensor, flow.Tensor] = None, + use_cache: bool = False, + ): + """ + + Args: + hidden_states (flow.Tensor): shape is [bsz, tgt_len, hidden_size]. + encoder_states (flow.Tensor, optional): shape is [bsz, src_len, hidden_size]. + Defaults to None. + attention_mask (flow.Tensor, optional): shape is [bsz, 1, tgt_len, src_len]. + It should be the combination of padding mask and casual mask. + It is the padding mask of source input when used with self-attention in encoder. + And it is the combination of padding mask of target input and casual mask when + used with self-attention in decoder. It is the padding mask of source input when + used with cross-attention in decoder. + Defaults to None. + past_key_value (Tuple[flow.Tensor, flow.Tensor], optional): tuple of key and value, + each shape is [bsz, num_heads, src_len, head_size]. Defaults to None. + use_cache (bool, optional): it will be set to True, when the model is in the inference + phase and used for incremental decoding. Defaults to False. + """ + if encoder_states is not None: + encoder_states = encoder_states.to_global(placement=hidden_states.placement) + + if attention_mask is not None: + attention_mask = attention_mask.to_global(placement=hidden_states.placement) + + bsz, tgt_len = hidden_states.size()[:2] + + if self.is_cross_attention: + query = self.query(hidden_states) + query = query.view(bsz, -1, self.num_heads, self.head_size) + query = query.permute(0, 2, 1, 3) + if past_key_value is not None: + key, value = past_key_value + elif encoder_states is not None: + key_value = self.key_value(encoder_states) + key_value = key_value.view(bsz, -1, self.num_heads, 2 * self.head_size) + key_value = key_value.permute(0, 2, 1, 3) + key, value = flow.chunk(key_value, chunks=2, dim=-1) + else: + raise ValueError( + "past_key_value and encoder_states cannot be None at the same time." + ) + else: + query_key_value = self.query_key_value(hidden_states) + query_key_value = query_key_value.view(bsz, -1, self.num_heads, 3 * self.head_size) + query_key_value = query_key_value.permute( + 0, 2, 1, 3 + ) # [bsz, num_heads, src_len, 3 * head_size] + query, key, value = flow.chunk(query_key_value, chunks=3, dim=-1) + if past_key_value is not None: + past_key, past_value = past_key_value + key = flow.cat((past_key.type_as(key), key), dim=2) + value = flow.cat((past_value.type_as(value), value), dim=2) + + if use_cache: + past_key_value = (key, value) + + attention_scores = flow.matmul(query, key, transpose_b=True, alpha=self.norm_factor) + + if not self.is_cross_attention: + query_length, key_length = query.size(-2), key.size(-2) + causal_mask = self.bias[:, :, key_length - query_length : key_length, :key_length].to( + flow.bool + ) + causal_mask = causal_mask.repeat(attention_scores.size(0), 1, 1, 1) + causal_mask = causal_mask.to_global( + sbp=attention_scores.sbp, placement=attention_scores.placement + ) + fill_value = flow.finfo(attention_scores.dtype).min + mask_value = flow.ones( + causal_mask.size(), + dtype=attention_scores.dtype, + sbp=attention_scores.sbp, + placement=attention_scores.placement, + ).fill_(fill_value) + attention_scores = flow.where(causal_mask, attention_scores, mask_value) + + if attention_mask is not None: + if self.scale_mask_softmax_fusion: + if self.attn_mask_type == AttnMaskType.padding: + attention_mask = ( + attention_mask.expand_as(attention_scores) if use_cache else attention_mask + ) + attention_weights = flow._C.fused_scale_mask_softmax_dropout( + attention_scores, + attention_mask, + fill_value=-10000.0, + scale=self.coeff, + p=self.attention_dropout_prob, + )[0] + else: + if self.coeff is not None: + attention_scores *= self.coeff + attention_scores = flow.mul(attention_scores, attention_mask) + attention_scores = attention_scores - 10000.0 * (1 - attention_mask) + attention_weights = flow.softmax(attention_scores, dim=-1) + attention_weights = self.dropout(attention_weights) + else: + if self.scale_mask_softmax_fusion and self.attn_mask_type == AttnMaskType.causal: + attention_weights = flow._C.fused_scale_tril_softmax_mask_scale( + attention_scores, + p=self.attention_dropout_prob, + diagonal=0, + tril_scale_value=self.coeff, + tril_fill_value=-10000.0, + )[0] + else: + attention_weights = flow.softmax(attention_scores, dim=-1) + attention_weights = self.dropout(attention_weights) + + context = flow.matmul(attention_weights, value) + context = context.transpose(1, 2) + output = self.dense(context.flatten(2)) + + if self.bias_dropout_fusion: + output, bias = output + output = flow._C.fused_bias_add_dropout( + output, bias, p=self.output_dropout_prob, axis=output.ndim - 1 + ) + else: + output = self.output_dropout(output) + + if use_cache: + output = (output, past_key_value) + + return output + + def extra_repr(self) -> str: + return "hidden_size={}, num_heads={}, is_cross_attention={}".format( + self.hidden_size, + self.num_heads, + self.is_cross_attention, + ) diff --git a/projects/MagicPrompt/layers/transformer_layer.py b/projects/MagicPrompt/layers/transformer_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..d26f8c13a9ccf9b6b7973b74787f954a52350fb3 --- /dev/null +++ b/projects/MagicPrompt/layers/transformer_layer.py @@ -0,0 +1,250 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow.nn as nn + +from libai.layers import build_activation +from libai.layers.droppath import DropPath +from libai.layers.layer_norm import LayerNorm +from libai.layers.mlp import MLP +from libai.utils import distributed as dist +from projects.MagicPrompt.layers.attention_layer import AttnMaskType, MultiheadAttention + + +class TransformerLayer(nn.Module): + """A single transformer layer. + + Transformer layer takes input with size [bsz, seq_length, hidden size] and returns an + output of the same size. + The input and output has same sbp sign, (S(0), B). + + Arguments: + hidden_size: size of hidden state. + ffn_hidden_size: size of feed forword neural network. + num_attention_heads: number of attention heads. + is_decoder: used to specify whether this is transformer encoder layer or transformer + decoder layer. Default: ``False``. + attention_dropout_prob: dropout probability of attention weights. + output_dropout_prob: dropout probability of output. + layernorm_epsilon: epsilon used in layernorm layer. Default: `1e-5`. + init_method: method to initialize the input layer weights. + output_layer_init_method: method to initialize the output layer weights. + If None, use `init_method`. + bias_gelu_fusion: whether fuse add bias and gelu. Default: ``False``. + bias_dropout_fusion: whether fuse add bias and dropout. Default: ``False``. + scale_mask_softmax_fusion: whether to fuse scale, mask and softmax. Default: ``False``. + apply_query_key_layer_scaling: if `true`, scaling the attention score by layer index. + Default: ``False``. + apply_residual_post_layernorm: if ``true``, use original BERT residual + connection ordering. Otherwise, use Megatron BERT residual connection which + is more stable when scaling model size introduced in + https://arxiv.org/pdf/1909.08053.pdf. + Default: ``False``. + layer_idx: the layer index, which determines the placement. + """ + + def __init__( + self, + hidden_size, + ffn_hidden_size, + num_attention_heads, + is_decoder=False, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + drop_path_prob=0.0, + layernorm_epsilon=1e-5, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + apply_residual_post_layernorm=False, + attn_mask_type=AttnMaskType.padding, + *, + layer_idx=0 + ): + super().__init__() + self.hidden_size = hidden_size + self.ffn_hidden_size = ffn_hidden_size + self.num_attention_heads = num_attention_heads + self.attention_dropout_prob = attention_dropout_prob + self.output_dropout_prob = output_dropout_prob + self.layernorm_epsilon = layernorm_epsilon + self.attn_mask_type = attn_mask_type + + self.layer_idx = layer_idx + self.is_decoder = is_decoder + + self.bias_gelu_fusion = bias_gelu_fusion + self.bias_dropout_fusion = bias_dropout_fusion + self.scale_mask_softmax_fusion = scale_mask_softmax_fusion + self.apply_query_key_layer_scaling = apply_query_key_layer_scaling + self.apply_residual_post_layernorm = apply_residual_post_layernorm + + self.init_method = init_method + if output_layer_init_method is None: + output_layer_init_method = init_method + self.output_layer_init_method = output_layer_init_method + + self.drop_path = DropPath(drop_path_prob) if drop_path_prob > 0.0 else nn.Identity() + + self.input_layernorm = LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon, layer_idx=self.layer_idx + ) + + self.self_attention = self.build_attention(is_cross_attention=False) + self.post_attention_layernorm = LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon, layer_idx=self.layer_idx + ) + + if self.is_decoder: + self.cross_attention = self.build_attention(is_cross_attention=True) + self.post_cross_attention_layernorm = LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon, layer_idx=self.layer_idx + ) + + self.mlp = MLP( + self.hidden_size, + self.ffn_hidden_size, + self.output_dropout_prob, + self.init_method, + output_layer_init_method=self.output_layer_init_method, + bias_gelu_fusion=self.bias_gelu_fusion, + bias_dropout_fusion=self.bias_dropout_fusion, + layer_idx=self.layer_idx, + ) + + # use gelu_tanh activation + if not bias_gelu_fusion: + self.mlp.activation_func = build_activation("gelu_tanh") + + def forward( + self, + hidden_states, + attention_mask=None, + encoder_states=None, + encoder_attention_mask=None, + past_key_value=None, + use_cache=False, + ): + """ + Args: + hidden_states: shape is (batch_size, seq_length, hidden_size), + sbp signature is (S(0), B). + attention_mask: the combination of key padding mask and casual mask of hidden states + with shape (batch_size, 1, seq_length, seq_length) and the sbp + signature is (S(0), B), + encoder_states: encoder output with shape (batch_size, seq_length, hidden_size) + and the sbp signature is (S(0), B), which will be used in cross attention. + encoder_attention_mask: key padding mask of encoder states with shape + (batch_size, 1, seq_length, seq_length) and the sbp signature is (S(0), B). + past_key_value: tuple of key and value, each shape is + (seq_length, bsz, num_heads, head_size), For decoder layer, + the past_key_value contains the states both from self attention + and cross attention. + use_cache: it will be set to `True` when the model is in the inference phase and + used for incremental decoding. + """ + hidden_states = hidden_states.to_global(placement=dist.get_layer_placement(self.layer_idx)) + + if attention_mask is not None: + attention_mask = attention_mask.to_global( + placement=dist.get_layer_placement(self.layer_idx) + ) + + if past_key_value is not None: + if self.is_decoder: + assert len(past_key_value) == 4 + self_attn_past_key_value = past_key_value[:2] + cross_attn_past_key_value = past_key_value[2:] + else: + self_attn_past_key_value = past_key_value + cross_attn_past_key_value = None + else: + self_attn_past_key_value, cross_attn_past_key_value = None, None + + layernorm_output = self.input_layernorm(hidden_states) + attention_output = self.self_attention( + layernorm_output, + attention_mask=attention_mask, + past_key_value=self_attn_past_key_value, + use_cache=use_cache, + ) + attention_output = self.drop_path(attention_output) + + if use_cache: + attention_output, presents = attention_output + + if self.apply_residual_post_layernorm: + residual = layernorm_output + else: + residual = hidden_states + + hidden_states = residual + attention_output + + layernorm_output = self.post_attention_layernorm(hidden_states) + + if self.is_decoder: + attention_output = self.cross_attention( + layernorm_output, + encoder_states, + attention_mask=encoder_attention_mask, + past_key_value=cross_attn_past_key_value, + use_cache=use_cache, + ) + + if use_cache: + attention_output, decoder_presents = attention_output + presents += decoder_presents + + attention_output = self.drop_path(attention_output) + if self.apply_residual_post_layernorm: + residual = layernorm_output + else: + residual = hidden_states + + hidden_states = residual + attention_output + layernorm_output = self.post_cross_attention_layernorm(hidden_states) + + mlp_output = self.mlp(layernorm_output) + mlp_output = self.drop_path(mlp_output) + + if self.apply_residual_post_layernorm: + residual = layernorm_output + else: + residual = hidden_states + + output = residual + mlp_output + + if use_cache: + output = (output, presents) + return output + + def build_attention(self, is_cross_attention=False): + return MultiheadAttention( + self.hidden_size, + self.num_attention_heads, + is_cross_attention=is_cross_attention, + attention_dropout_prob=self.attention_dropout_prob, + output_dropout_prob=self.output_dropout_prob, + init_method=self.init_method, + output_layer_init_method=self.output_layer_init_method, + bias_dropout_fusion=self.bias_dropout_fusion, + scale_mask_softmax_fusion=self.scale_mask_softmax_fusion, + apply_query_key_layer_scaling=self.apply_query_key_layer_scaling, + attn_mask_type=self.attn_mask_type, + layer_idx=self.layer_idx, + ) diff --git a/projects/MagicPrompt/pipeline.py b/projects/MagicPrompt/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..1da8e5b4e51041134d0a97986296127f14aa5bb0 --- /dev/null +++ b/projects/MagicPrompt/pipeline.py @@ -0,0 +1,106 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libai.inference.basic import BasePipeline +from libai.utils import distributed as dist + + +class TextGenerationPipeline(BasePipeline): + def load_pretrain_weight(self, libai_cfg_model, model_path, mode="huggingface"): + """load pretrained model. + + Args: + libai_cfg_model (libai.models): Lazy config Model in Libai, you can import it + by `from libai.config.configs.common.models.bert + import pretrain_model as libai_cfg_model` + model_path (str): The directory path of pretrained model, + """ + if mode == "huggingface": + from libai.models.utils import GPT2LoaderHuggerFace + + model_loader = GPT2LoaderHuggerFace( + libai_cfg_model, + libai_cfg_model.cfg, + model_path, + ) + model = model_loader.load() + model.eval() + return model + + elif mode == "libai": + from libai.models.utils import GPT2LoaderLiBai + + model_loader = GPT2LoaderLiBai( + libai_cfg_model, + libai_cfg_model.cfg, + model_path, + ) + model = model_loader.load() + model.eval() + return model + + elif mode == "random": + from libai.engine import DefaultTrainer + + return DefaultTrainer.build_model(self.cfg) + else: + raise NotImplementedError + + def _parse_parameters(self, **pipeline_parameters): + preprocess_params = {} + forward_params = {**pipeline_parameters} + postprocess_params = {} + + return preprocess_params, forward_params, postprocess_params + + def preprocess(self, inputs, **kwargs) -> dict: + # tokenizer encoder + input_ids = self.tokenizer.encode(inputs, return_tensors="of", is_global=True) + + inputs = { + "input_ids": input_ids, + } + + return inputs + + def forward(self, inputs, **kwargs) -> dict: + outputs = self.model.generate(inputs["input_ids"], do_sample=True, max_length=50, **kwargs) + return {"return_ids": outputs} + + def postprocess(self, model_output_dict, **kwargs) -> dict: + return_ids = model_output_dict["return_ids"] + records = [ + {"generated_text": self.tokenizer.decode(return_ids[i], skip_special_tokens=True)} + for i in range(return_ids.size(0)) + ] + return records + + +if __name__ == "__main__": + pipeline = TextGenerationPipeline( + "projects/MagicPrompt/configs/gpt2_inference.py", + data_parallel=1, + tensor_parallel=1, + pipeline_parallel=1, + # pipeline_stage_id=[0] * 6 + [1] * 6, + # pipeline_num_layers=12, + model_path="/path/to/oneflow-model", + mode="libai", + ) + + text = ["a dog"] + output = pipeline(inputs=text) + if dist.is_main_process(): + print(output) diff --git a/projects/MagicPrompt/readme.md b/projects/MagicPrompt/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..da6dbbaf35dac4dba03e3de3d6a9727bcd24baed --- /dev/null +++ b/projects/MagicPrompt/readme.md @@ -0,0 +1,84 @@ +# MagicPrompt + +This project is a NLP text generate tasks, which based on `gpt2` model to generate prompt-texts for AI drawings, such as `stable-diffusion`. we provides a [pipeline](./pipeline.py) that can be directly used for inference, and you can also use your own dataset to `customize` the gpt2 with your own data. + +## How to use pipeline + +- Prepare the gpt2 checkpoint, If you don't have suitable checkpoint, you can use [OneFlow/MagicPrompt-Stable-Diffusion](https://oneflow-public.oss-cn-beijing.aliyuncs.com/datasets/libai/magicprompt/OneFlow-MagicPrompt-Stable_Diffusion.zip) or [Gustavosta/MagicPrompt-Stable-Diffusion](https://huggingface.co/Gustavosta/MagicPrompt-Stable-Diffusion?text=My+name+is+Merve+and+my+favorite). + +- Adjust the parameters in the pipeline.py, and run it. + +- Here is an example showing MagicPrompt, which combines the [oneflow version of diffusion](https://github.com/Oneflow-Inc/diffusers/wiki/How-to-Run-OneFlow-Stable-Diffusion): + +```python +from projects.MagicPrompt.pipeline import MagicPromptPipeline +import oneflow as torch +from diffusers import OneFlowStableDiffusionPipeline + + +pipeline = MagicPromptPipeline( + "/projects/MagicPrompt/configs/gpt_inference.py", + model_path="path/to/gpt2-checkpoint", + mode="huggingface", +) + +text = ["a dog"] +output = pipeline(inputs=text) +if dist.is_main_process(): + print(output) + +pipe = OneFlowStableDiffusionPipeline.from_pretrained( + "prompthero/midjourney-v4-diffusion", + use_auth_token=True, +) + +pipe = pipe.to("cuda") +prompt = output[0]['generated_text'] +with torch.autocast("cuda"): + images = pipe(prompt).images + for i, image in enumerate(images): + image.save(f"result.png") + +``` +- Generated prompt: `a dog in a astronaut suit and luffy, intricate, luffy, highly detailed, digital painting, artstation, concept art, smooth, sharp focus, illustration, luffy, unreal engine 5, 8 k, art by artgerm` +
+ + +## How to customize the gpt2 with your own data + +### 1. Prepare your own datasets + +- Official dataset address: [https://huggingface.co/datasets/Gustavosta/Stable-Diffusion-Prompts](https://huggingface.co/datasets/Gustavosta/Stable-Diffusion-Prompts). + +- If you want to customize datasts, please prepare the data in txt format, and use `projects/MagicPrompt/datasets/datasets.py` to process it. + +- Use `tools/preprocess_data.py` to process the json files, you can refer [https://libai.readthedocs.io/en/latest/tutorials/basics/Preprocessing_Dataset.html](https://libai.readthedocs.io/en/latest/tutorials/basics/Preprocessing_Dataset.html). + +```python +IMPL=lazy +KEYS=text + +python tools/preprocess_data.py \ + --input path/to/test_sample_cn.json \ + --json-keys ${KEYS} \ + --vocab-file path/to/vocab.txt \ + --merges-file path/to/merges.txt + --dataset-impl ${IMPL} \ + --tokenizer-name GPT2Tokenizer \ + --do-lower-case \ + --do-chinese-wwm \ + --split-sentences \ + --output-prefix magic_prompt_${IMPL} \ + --workers 4 \ + --log-interval 2 +``` + +> You can directly get the processed dataset by running the following command: +> `wget http://oneflow-public.oss-cn-beijing.aliyuncs.com/datasets/libai/magicprompt/magicprompt.zip` + + +### 2. Training + +```bash +bash tools/train.sh tools/train_net.py projects/MagicPrompt/configs/finetune.py 1 +``` diff --git a/projects/NeRF/configs/config_model.py b/projects/NeRF/configs/config_model.py new file mode 100644 index 0000000000000000000000000000000000000000..f514906f7710b6fe64769784091ff581ea31a514 --- /dev/null +++ b/projects/NeRF/configs/config_model.py @@ -0,0 +1,24 @@ +from omegaconf import DictConfig +from libai.config import LazyCall + +from projects.NeRF.modeling.System import NerfSystem + + +cfg = dict( + D=8, + W=256, + in_channels_xyz=63, + in_channels_dir=27, + skips=[4], + N_samples=64, + use_disp=False, + perturb=1.0, + noise_std=0.0, + N_importance=128, + chunk=64 * 1204, + dataset_type="Blender", +) + +cfg = DictConfig(cfg) + +model = LazyCall(NerfSystem)(cfg=cfg) diff --git a/projects/NeRF/configs/config_nerf.py b/projects/NeRF/configs/config_nerf.py new file mode 100644 index 0000000000000000000000000000000000000000..6c44212dc12edde07ef3622f1de73b6c7085b0f0 --- /dev/null +++ b/projects/NeRF/configs/config_nerf.py @@ -0,0 +1,174 @@ +from omegaconf import OmegaConf + +import oneflow as flow +import oneflow.nn as nn + +from libai.data.build import build_image_train_loader, build_image_test_loader +from libai.config import LazyCall, get_config +from libai.optim import get_default_optimizer_params +from libai.scheduler.lr_scheduler import WarmupCosineAnnealingLR, WarmupMultiStepLR + +from projects.NeRF.datasets import BlenderDataset, LLFFDataset +from projects.NeRF.optimizers import Ranger, RAdam +from projects.NeRF.evaluation.nerf_evaluator import NerfEvaluator +from projects.NeRF.configs.config_model import model + + +def get_nerf_dataset(dataset_type="Blender"): + """ + Args: + dataset_type: Blender or LLFF + """ + assert dataset_type in ["Blender", "LLFF"], "The Nerf dataset must be one of Blender and LLFF" + if dataset_type == "Blender": + return BlenderDataset + else: + return LLFFDataset + + +graph = get_config("common/models/graph.py").graph +graph.enabled = False +train = get_config("common/train.py").train + +# Refine train cfg for Nerf System +train.train_micro_batch_size = 1024 # Verification by ray +train.test_micro_batch_size = 1 # Verification by picture +train.dataset_type = "Blender" # Blender or LLFF +train.blender_dataset_path = "/path/to/blender" +train.llff_dataset_path = "/path/to/llff" +train.train_epoch = 16 if train.dataset_type == "Blender" else 30 +train.warmup_ratio = int(1 / train.train_epoch) +train.evaluation.eval_period = 1000 +train.log_period = 50 +train.optim_type = "adam" +train.lr_scheduler_type = "cosine" + +# Redefining model config +model.cfg.dataset_type = train.dataset_type +model.cfg.loss_func = nn.MSELoss() +model.cfg.noise_std = 0.0 if train.dataset_type == "Blender" else 1.0 +# Redefining evaluator +train.evaluation = dict( + enabled=True, + # evaluator for calculating psnr + evaluator=LazyCall(NerfEvaluator)( + img_wh=(400, 400) if train.dataset_type == "Blender" else (504, 378) + ), + eval_period=train.evaluation.eval_period, + eval_iter=1e5, # running steps for validation/test + # Metrics to be used for best model checkpoint. + eval_metric="psnr", + eval_mode="max", +) + +# Refine optimizer cfg for Nerf System +# NOTE: In theory, both datasets used by Nerf are optimized using the Adam optimizer, but +# since the borrowed code base also implements three other optimizer configurations, libai +# also implements the corresponding optimizer. +if train.optim_type == "adam": + optimizer = flow.optim.Adam + lr = 5e-4 +elif train.optim_type == "sgd": + optimizer = flow.optim.SGD + lr = 5e-2 +elif train.optim_type == "radam": + optimizer = RAdam + lr = 5e-4 +elif train.optim_type == "ranger": + optimizer = Ranger + lr = 5e-4 +else: + raise NotImplementedError("Nerf does not support this type of optimizer!") + +optim = LazyCall(optimizer)( + params=LazyCall(get_default_optimizer_params)( + # params.model is meant to be set to the model object, + # before instantiating the optimizer. + clip_grad_max_norm=None, + clip_grad_norm_type=None, + weight_decay_norm=None, + weight_decay_bias=None, + ), + lr=lr, + weight_decay=0, +) + +if train.optim_type == "sgd": + optim.momentum = 0.9 + +if train.lr_scheduler_type == "steplr": + scheduler = WarmupMultiStepLR +elif train.lr_scheduler_type == "cosine": + scheduler = WarmupCosineAnnealingLR +else: + raise NotImplementedError("Nerf does not support this type of scheduler!") + +train.scheduler = LazyCall(scheduler)( + warmup_factor=0.001, + warmup_method="linear", +) + +if train.lr_scheduler_type == "steplr": + if train.dataset_type == "Blender": + milestones = [2 / 16, 4 / 16, 8 / 16] + else: + milestones = [10 / 30, 20 / 30] + train.scheduler.milestones = milestones + train.scheduler.gamma = 0.5 +elif train.lr_scheduler_type == "cosine": + train.scheduler.eta_min = 1e-8 + +train.warmup_ratio = ( + train.warmup_ratio + if train.warmup_ratio > 0 and train.optim_type not in ["radam", "ranger"] + else 0.0 +) + +# Set fp16 ON +train.amp.enabled = True + +dataset = LazyCall(get_nerf_dataset)(dataset_type=train.dataset_type) + +dataloader = OmegaConf.create() + +dataloader.train = LazyCall(build_image_train_loader)( + dataset=[ + LazyCall(dataset)( + split="train", + img_wh=(400, 400) if dataset.dataset_type == "Blender" else (504, 378), + root_dir=train.blender_dataset_path + if dataset.dataset_type == "Blender" + else train.llff_dataset_path, + spheric_poses=None if dataset.dataset_type == "Blender" else False, + val_num=None if dataset.dataset_type == "Blender" else 1, # Number of your GPUs + batchsize=train.train_micro_batch_size, + ) + ], + num_workers=4, + train_batch_size=1, + test_batch_size=train.test_micro_batch_size, +) + +dataloader.test = [ + LazyCall(build_image_test_loader)( + dataset=LazyCall(dataset)( + split="val", + img_wh=(400, 400) if dataset.dataset_type == "Blender" else (504, 378), + root_dir=train.blender_dataset_path + if dataset.dataset_type == "Blender" + else train.llff_dataset_path, + spheric_poses=None if dataset.dataset_type == "Blender" else False, + val_num=None if dataset.dataset_type == "Blender" else 1, # Number of your GPUs + ), + num_workers=0, + test_batch_size=train.test_micro_batch_size, + ) +] + +# Distributed Settings +depth = None +train.train_micro_batch_size = 1 +train.dist.pipeline_num_layers = depth +train.dist.data_parallel_size = 1 +train.dist.tensor_parallel_size = 1 +train.dist.pipeline_parallel_size = 1 diff --git a/projects/NeRF/configs/config_nerf_for_rendering.py b/projects/NeRF/configs/config_nerf_for_rendering.py new file mode 100644 index 0000000000000000000000000000000000000000..2c27b02c6d7f0e8cc20548e87e7a98ef7f3e1d74 --- /dev/null +++ b/projects/NeRF/configs/config_nerf_for_rendering.py @@ -0,0 +1,49 @@ +from projects.NeRF.configs.config_nerf import ( + train, + dataset, + dataloader, + graph, + model, + LazyCall, + build_image_test_loader, +) +from projects.NeRF.evaluation.nerf_evaluator import NerfVisEvaluator +from libai.data.samplers import SingleRoundSampler + +# NOTE: Used for generating MP4 format files +# Redefining evaluator +train.evaluation = dict( + enabled=True, + # evaluator for calculating psnr + evaluator=LazyCall(NerfVisEvaluator)( + img_wh=(400, 400) if train.dataset_type == "Blender" else (504, 378), + pose_dir_len=40 if train.dataset_type == "Blender" else 120, + name="blender_rendering_result" + if train.dataset_type == "Blender" + else "llff_rendering_result", + ), + eval_period=train.evaluation.eval_period, + eval_iter=1e5, # running steps for validation/test + # Metrics to be used for best model checkpoint. + eval_metric="psnr", + eval_mode="max", +) + +dataloader.test = [ + LazyCall(build_image_test_loader)( + dataset=LazyCall(dataset)( + split="vis", + img_wh=(400, 400) if dataset.dataset_type == "Blender" else (504, 378), + root_dir=train.blender_dataset_path + if dataset.dataset_type == "Blender" + else train.llff_dataset_path, + spheric_poses=None if dataset.dataset_type == "Blender" else False, + val_num=None if dataset.dataset_type == "Blender" else 1, # Number of your GPUs + ), + sampler=LazyCall(SingleRoundSampler)(shuffle=False, drop_last=False), + num_workers=0, + test_batch_size=train.test_micro_batch_size, + ) +] + +train.load_weight = "/path/to/ckpt" # Already trained NeRF checkpoint location diff --git a/projects/NeRF/datasets/__init__.py b/projects/NeRF/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0860a1b1267f692723ece3eb58f5f5ddda908bc4 --- /dev/null +++ b/projects/NeRF/datasets/__init__.py @@ -0,0 +1 @@ +from .nerf_dataset import BlenderDataset, LLFFDataset diff --git a/projects/NeRF/datasets/nerf_dataset.py b/projects/NeRF/datasets/nerf_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..d475c11d662192e1b7677dee0c2e8d3edec3b1db --- /dev/null +++ b/projects/NeRF/datasets/nerf_dataset.py @@ -0,0 +1,879 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import json +import os +import re +import sys +from collections import OrderedDict +from typing import Optional + +import numpy as np +import oneflow as flow +from flowvision import transforms as T +from oneflow.utils.data import Dataset +from PIL import Image + +from libai.data.structures import DistTensorData, Instance + + +def read_pfm(filename): + file = open(filename, "rb") + + header = file.readline().decode("utf-8").rstrip() + if header == "PF": + color = True + elif header == "Pf": + color = False + else: + raise Exception("Not a PFM file.") + + dim_match = re.match(r"^(\d+)\s(\d+)\s$", file.readline().decode("utf-8")) + if dim_match: + width, height = map(int, dim_match.groups()) + else: + raise Exception("Malformed PFM header.") + + scale = float(file.readline().rstrip()) + if scale < 0: # little-endian + endian = "<" + scale = -scale + else: + endian = ">" # big-endian + + data = np.fromfile(file, endian + "f") + shape = (height, width, 3) if color else (height, width) + + data = np.reshape(data, shape) + data = np.flipud(data) + file.close() + return data, scale + + +def save_pfm(filename, image, scale=1): + file = open(filename, "wb") + image = np.flipud(image) + + if image.dtype.name != "float32": + raise Exception("Image dtype must be float32.") + + if len(image.shape) == 3 and image.shape[2] == 3: # color image + color = True + elif len(image.shape) == 2 or len(image.shape) == 3 and image.shape[2] == 1: # greyscale + color = False + else: + raise Exception("Image must have H x W x 3, H x W x 1 or H x W dimensions.") + + file.write("PF\n".encode("utf-8") if color else "Pf\n".encode("utf-8")) + file.write("{} {}\n".format(image.shape[1], image.shape[0]).encode("utf-8")) + + endian = image.dtype.byteorder + + if endian == "<" or endian == "=" and sys.byteorder == "little": + scale = -scale + + file.write(("%f\n" % scale).encode("utf-8")) + + image.tofile(file) + file.close() + + +# Preparatory conversion tools for 3D rendering +def create_meshgrid( + height: int, + width: int, + normalized_coordinates: bool = True, + device: Optional[flow.device] = flow.device("cpu"), + dtype: flow.dtype = flow.float32, +): + """Generate a coordinate grid for an image. + + When the flag ``normalized_coordinates`` is set to True, the grid is + normalized to be in the range :math:`[-1,1]` to be consistent with the pytorch + function :py:func:`torch.nn.functional.grid_sample`. + + Args: + height: the image height (rows). + width: the image width (cols). + normalized_coordinates: whether to normalize + coordinates in the range :math:`[-1,1]` in order to be consistent with the + PyTorch function :py:func:`torch.nn.functional.grid_sample`. + device: the device on which the grid will be generated. + dtype: the data type of the generated grid. + + Return: + grid tensor with shape :math:`(1, H, W, 2)`. + + Example: + >>> create_meshgrid(2, 2) + tensor([[[[-1., -1.], + [ 1., -1.]], + + [[-1., 1.], + [ 1., 1.]]]]) + + >>> create_meshgrid(2, 2, normalized_coordinates=False) + tensor([[[[0., 0.], + [1., 0.]], + + [[0., 1.], + [1., 1.]]]]) + """ + xs = flow.linspace(0, width - 1, width, device=device, dtype=dtype) + ys = flow.linspace(0, height - 1, height, device=device, dtype=dtype) + if normalized_coordinates: + xs = (xs / (width - 1) - 0.5) * 2 + ys = (ys / (height - 1) - 0.5) * 2 + # generate grid by stacking coordinates + base_grid = flow.stack(flow.meshgrid([xs, ys], indexing="ij"), dim=-1) # WxHx2 + return base_grid.permute(1, 0, 2).unsqueeze(0) # 1xHxWx2 + + +def get_rays(directions, c2w): + """ + Get ray origin and normalized directions in world coordinate for all pixels in one image. + Inputs: + directions: (H, W, 3) precomputed ray directions in camera coordinate + c2w: (3, 4) transformation matrix from camera coordinate to world coordinate + Outputs: + rays_o: (H*W, 3), the origin of the rays in world coordinate + rays_d: (H*W, 3), the normalized direction of the rays in world coordinate + """ + # Rotate ray directions from camera coordinate to the world coordinate + rays_d = directions @ c2w[:, :3].T # (H, W, 3) + # rays_d = rays_d / flow.norm(rays_d, dim=-1, keepdim=True) + # The origin of all rays is the camera origin in world coordinate + rays_o = c2w[:, 3].expand(rays_d.shape) # (H, W, 3) + + rays_d = rays_d.view(-1, 3) + rays_o = rays_o.view(-1, 3) + + return rays_o, rays_d + + +def get_ray_directions(H, W, focal): + """ + Get ray directions for all pixels in camera coordinate. + Inputs: + H, W, focal: image height, width and focal length + Outputs: + directions: (H, W, 3), the direction of the rays in camera coordinate + """ + grid = create_meshgrid(H, W, normalized_coordinates=False)[0] + i, j = grid.unbind(-1) + i = flow.tensor(i.numpy()) + j = flow.tensor(j.numpy()) + directions = flow.stack( + [(i - W / 2) / focal, -(j - H / 2) / focal, -flow.ones_like(i)], -1 + ) # compute about tanx (H, W, 3) + + return directions + + +def get_ndc_rays(H, W, focal, near, rays_o, rays_d): + """ + Transform rays from world coordinate to NDC. + NDC: Space such that the canvas is a cube with sides [-1, 1] in each axis. + For detailed derivation, please see: + http://www.songho.ca/opengl/gl_projectionmatrix.html + https://github.com/bmild/nerf/files/4451808/ndc_derivation.pdf + https://pengfeixc.com/blogs/computer-graphics/3D-matrix-transformation-part-three + + In practice, use NDC "if and only if" the scene is unbounded (has a large depth). + See https://github.com/bmild/nerf/issues/18 + + Inputs: + H, W, focal: image height, width and focal length + near: (N_rays) or float, the depths of the near plane + rays_o: (N_rays, 3), the origin of the rays in world coordinate + rays_d: (N_rays, 3), the direction of the rays in world coordinate + + Outputs: + rays_o: (N_rays, 3), the origin of the rays in NDC + rays_d: (N_rays, 3), the direction of the rays in NDC + """ + # Shift ray origins to near plane + t = -(near + rays_o[..., 2]) / rays_d[..., 2] + rays_o = rays_o + t[..., None] * rays_d + + # Store some intermediate homogeneous results + ox_oz = rays_o[..., 0] / rays_o[..., 2] + oy_oz = rays_o[..., 1] / rays_o[..., 2] + + # Projection + o0 = -1.0 / (W / (2.0 * focal)) * ox_oz + o1 = -1.0 / (H / (2.0 * focal)) * oy_oz + o2 = 1.0 + 2.0 * near / rays_o[..., 2] + + d0 = -1.0 / (W / (2.0 * focal)) * (rays_d[..., 0] / rays_d[..., 2] - ox_oz) + d1 = -1.0 / (H / (2.0 * focal)) * (rays_d[..., 1] / rays_d[..., 2] - oy_oz) + d2 = 1 - o2 + rays_o = flow.stack([o0, o1, o2], -1) # (B, 3) + rays_d = flow.stack([d0, d1, d2], -1) # (B, 3) + + return rays_o, rays_d + + +def normalize(v): + """Normalize a vector.""" + return v / np.linalg.norm(v) + + +def average_poses(poses): + """ + Calculate the average pose, which is then used to center all poses + using @center_poses. Its computation is as follows: + 1. Compute the center: the average of pose centers. + 2. Compute the z axis: the normalized average z axis. + 3. Compute axis y': the average y axis. + 4. Compute x' = y' cross product z, then normalize it as the x axis. + 5. Compute the y axis: z cross product x. + + Note that at step 3, we cannot directly use y' as y axis since it's + not necessarily orthogonal to z axis. We need to pass from x to y. + + Inputs: + poses: (N_images, 3, 4) + + Outputs: + pose_avg: (3, 4) the average pose + """ + # 1. Compute the center + center = poses[..., 3].mean(0) # (3) + + # 2. Compute the z axis + z = normalize(poses[..., 2].mean(0)) # (3) + + # 3. Compute axis y' (no need to normalize as it's not the final output) + y_ = poses[..., 1].mean(0) # (3) + + # 4. Compute the x axis + x = normalize(np.cross(y_, z)) # (3) + + # 5. Compute the y axis (as z and x are normalized, y is already of norm 1) + y = np.cross(z, x) # (3) + + pose_avg = np.stack([x, y, z, center], 1) # (3, 4) + + return pose_avg + + +def center_poses(poses): + """ + Center the poses so that we can use NDC. + See https://github.com/bmild/nerf/issues/34 + + Inputs: + poses: (N_images, 3, 4) + + Outputs: + poses_centered: (N_images, 3, 4) the centered poses + pose_avg: (3, 4) the average pose + """ + + pose_avg = average_poses(poses) # (3, 4) + pose_avg_homo = np.eye(4) + pose_avg_homo[:3] = pose_avg # convert to homogeneous coordinate for faster computation + # by simply adding 0, 0, 0, 1 as the last row + last_row = np.tile(np.array([0, 0, 0, 1]), (len(poses), 1, 1)) # (N_images, 1, 4) + poses_homo = np.concatenate([poses, last_row], 1) # (N_images, 4, 4) homogeneous coordinate + + poses_centered = np.linalg.inv(pose_avg_homo) @ poses_homo # (N_images, 4, 4) + poses_centered = poses_centered[:, :3] # (N_images, 3, 4) + + return poses_centered, np.linalg.inv(pose_avg_homo) + + +def create_spiral_poses(radii, focus_depth, n_poses=120): + """ + Computes poses that follow a spiral path for rendering purpose. + See https://github.com/Fyusion/LLFF/issues/19 + In particular, the path looks like: + https://tinyurl.com/ybgtfns3 + + Inputs: + radii: (3) radii of the spiral for each axis + focus_depth: float, the depth that the spiral poses look at + n_poses: int, number of poses to create along the path + + Outputs: + poses_spiral: (n_poses, 3, 4) the poses in the spiral path + """ + + poses_spiral = [] + for t in np.linspace(0, 4 * np.pi, n_poses + 1)[:-1]: # rotate 4pi (2 rounds) + # the parametric function of the spiral (see the interactive web) + center = np.array([np.cos(t), -np.sin(t), -np.sin(0.5 * t)]) * radii + + # the viewing z axis is the vector pointing from the @focus_depth plane + # to @center + z = normalize(center - np.array([0, 0, -focus_depth])) + + # compute other axes as in @average_poses + y_ = np.array([0, 1, 0]) # (3) + x = normalize(np.cross(y_, z)) # (3) + y = np.cross(z, x) # (3) + + poses_spiral += [np.stack([x, y, z, center], 1)] # (3, 4) + + return np.stack(poses_spiral, 0) # (n_poses, 3, 4) + + +def spheric_pose(theta, phi, radius): + def trans_t(t): + return np.array( + [ + [1, 0, 0, 0], + [0, 1, 0, -0.9 * t], + [0, 0, 1, t], + [0, 0, 0, 1], + ] + ) + + def rot_phi(phi): + return np.array( + [ + [1, 0, 0, 0], + [0, np.cos(phi), -np.sin(phi), 0], + [0, np.sin(phi), np.cos(phi), 0], + [0, 0, 0, 1], + ] + ) + + def rot_theta(th): + return np.array( + [ + [np.cos(th), 0, -np.sin(th), 0], + [0, 1, 0, 0], + [np.sin(th), 0, np.cos(th), 0], + [0, 0, 0, 1], + ] + ) + + c2w = rot_theta(theta) @ rot_phi(phi) @ trans_t(radius) + c2w = np.array([[-1, 0, 0, 0], [0, 0, 1, 0], [0, 1, 0, 0], [0, 0, 0, 1]]) @ c2w + return c2w[:3] + + +def create_spheric_poses(radius, n_poses=120): + """ + Create circular poses around z axis. + Inputs: + radius: the (negative) height and the radius of the circle. + + Outputs: + spheric_poses: (n_poses, 3, 4) the poses in the circular path + """ + + spheric_poses = [] + for th in np.linspace(0, 2 * np.pi, n_poses + 1)[:-1]: + spheric_poses += [spheric_pose(th, -np.pi / 5, radius)] # 36 degree view downwards + return np.stack(spheric_poses, 0) + + +def pose_spherical(theta, phi, radius): + def trans_t(t): + return flow.Tensor( + [ + [1, 0, 0, 0], + [0, 1, 0, 0], + [0, 0, 1, t], + [0, 0, 0, 1], + ] + ).float() + + def rot_phi(phi): + return flow.Tensor( + [ + [1, 0, 0, 0], + [0, np.cos(phi), -np.sin(phi), 0], + [0, np.sin(phi), np.cos(phi), 0], + [0, 0, 0, 1], + ] + ).float() + + def rot_theta(th): + return flow.Tensor( + [ + [np.cos(th), 0, -np.sin(th), 0], + [0, 1, 0, 0], + [np.sin(th), 0, np.cos(th), 0], + [0, 0, 0, 1], + ] + ).float() + + c2w = trans_t(radius) + c2w = rot_phi(phi / 180.0 * np.pi) @ c2w + c2w = rot_theta(theta / 180.0 * np.pi) @ c2w + c2w = flow.Tensor(np.array([[-1, 0, 0, 0], [0, 0, 1, 0], [0, 1, 0, 0], [0, 0, 0, 1]])) @ c2w + return c2w + + +def viewmatrix(z, up, pos): + vec2 = normalize(z) + vec1_avg = up + vec0 = normalize(np.cross(vec1_avg, vec2)) + vec1 = normalize(np.cross(vec2, vec0)) + m = np.stack([vec0, vec1, vec2, pos], 1) + return m + + +def render_path_spiral(c2w, hwf, up, rads, focal, zdelta, zrate, rots, N): + render_poses = [] + hwf = hwf[:, None] + rads = np.array(list(rads) + [1.0]) + for theta in np.linspace(0.0, 2.0 * np.pi * rots, N + 1)[:-1]: + c = np.dot( + c2w, np.array([np.cos(theta), -np.sin(theta), -np.sin(theta * zrate), 1.0]) * rads + ) + z = normalize(c - np.dot(c2w, np.array([0, 0, -focal, 1.0]))) + render_poses.append(np.concatenate([viewmatrix(z, up, c), hwf], 1)) + return render_poses + + +# Blender and LLFF Datasets +def trun_dict_to_instance(dict): + return Instance(**{key: DistTensorData(flow.tensor(value)) for key, value in dict.items()}) + + +class NerfBaseDataset(Dataset): + def __init__(self, root_dir, split, img_wh): + super(NerfBaseDataset, self).__init__() + self.root_dir = root_dir + self.split = split + self.img_wh = img_wh + self.transform = T.Compose([T.ToTensor()]) + os.environ["ONEFLOW_DISABLE_VIEW"] = "true" + + def load_meta(self): + pass + + +class BlenderDataset(NerfBaseDataset): + def __init__(self, root_dir, split="train", img_wh=(800, 800), batchsize=1024, **kwargs): + """ + Args: + root_dir: str, + split: str, + img_wh: tuple, + """ + super(BlenderDataset, self).__init__(root_dir, split, img_wh) + self.white_back = True + self.batchsize = batchsize + self.load_meta() + self.render_poses = flow.stack( + [pose_spherical(angle, -30.0, 4.0) for angle in np.linspace(-180, 180, 40 + 1)[:-1]], 0 + ) # use for test + + def load_meta(self): + if self.split == "vis": + with open(os.path.join(self.root_dir, "transforms_train.json"), "r") as f: + self.meta = json.load(f) + else: + with open(os.path.join(self.root_dir, f"transforms_{self.split}.json"), "r") as f: + self.meta = json.load(f) + w, h = self.img_wh + camera_angle_x = float(self.meta["camera_angle_x"]) + self.focal = 0.5 * w / np.tan(0.5 * camera_angle_x) + self.near = 2.0 + self.far = 6.0 + self.bounds = np.array([self.near, self.far]) + self.directions = get_ray_directions(h, w, self.focal) # (h, w, 3) + + if self.split == "train": # create buffer of all rays and rgb data + self.image_paths = [] + self.poses = [] + self.all_rays = [] + self.all_rgbs = [] + self.indexs = [ + i * self.img_wh[0] * self.img_wh[1] for i in range(len(self.meta["frames"])) + ] + for frame in self.meta["frames"]: + pose = np.array(frame["transform_matrix"])[:3, :4] + self.poses += [pose] + c2w = flow.Tensor(pose) + image_path = os.path.join(self.root_dir, f"{frame['file_path']}.png") + self.image_paths += [image_path] + img = Image.open(image_path) + img = img.resize(self.img_wh, Image.LANCZOS) + img = self.transform(img) # (4, h, w) + img = img.view(4, -1).permute(1, 0) # (h*w, 4) RGBA + img = img[:, :3] * img[:, -1:] + (1 - img[:, -1:]) # blend A to RGB + self.all_rgbs += [img] + rays_o, rays_d = get_rays(self.directions, c2w) # both (h*w, 3) + self.all_rays += [ + flow.cat( + [ + rays_o, + rays_d, + self.near * flow.ones_like(rays_o[:, :1]), + self.far * flow.ones_like(rays_o[:, :1]), + ], + 1, + ) + ] # (h*w, 8) + self.all_rays = flow.cat(self.all_rays, 0) # (len(self.meta['frames])*h*w, 3) + self.all_rgbs = flow.cat(self.all_rgbs, 0) # (len(self.meta['frames])*h*w, 3) + self.num_iter = 0 + self.dH = int(self.img_wh[0] // 2 * 0.5) + self.dW = int(self.img_wh[1] // 2 * 0.5) + + def __len__(self): + if self.split == "train": + return int(len(self.all_rays) / self.batchsize) + elif self.split == "val": + return 8 # only validate 8 images (to support <=8 gpus) + elif self.split == "vis": + return len(self.render_poses) + elif self.split == "test": + return len(self.meta["frames"]) + + def __getitem__(self, idx): + if self.split == "train": # use data in the buffers + idx = idx % len(self.indexs) + if self.num_iter < 500: + coords = flow.stack( + flow.meshgrid( + flow.linspace( + self.img_wh[1] // 2 - self.dH, + self.img_wh[1] // 2 + self.dH - 1, + 2 * self.dH, + ), + flow.linspace( + self.img_wh[0] // 2 - self.dW, + self.img_wh[0] // 2 + self.dW - 1, + 2 * self.dW, + ), + ), + -1, + ) + else: + coords = flow.stack( + flow.meshgrid( + flow.linspace(0, self.img_wh[1] - 1, self.img_wh[1]), + flow.linspace(0, self.img_wh[0] - 1, self.img_wh[0]), + ), + -1, + ) # (H, W, 2) + coords = flow.reshape(coords, [-1, 2]) # (H * W, 2) + select_inds = np.random.choice( + coords.shape[0], size=[self.batchsize], replace=False + ) # (N_rand,) + select_coords = coords[select_inds].long() # (N_rand, 2) + rays = self.all_rays[ + self.indexs[idx] : self.indexs[idx] + self.img_wh[0] * self.img_wh[1] + ] + rgbs = self.all_rgbs[ + self.indexs[idx] : self.indexs[idx] + self.img_wh[0] * self.img_wh[1] + ] + rays = rays.view(self.img_wh[1], self.img_wh[0], -1) + rgbs = rgbs.view(self.img_wh[1], self.img_wh[0], -1) + rays = rays[select_coords[:, 0], select_coords[:, 1]] # (N_rand, 3) + rgbs = rgbs[select_coords[:, 0], select_coords[:, 1]] # (N_rand, 3) + self.num_iter += 1 + sample = OrderedDict(rays=rays, rgbs=rgbs) # # a alignment point with nerf_pytorch + + elif self.split == "val" or self.split == "test": # create data for each image separately + frame = self.meta["frames"][idx] + c2w = flow.Tensor(frame["transform_matrix"])[:3, :4] + + img = Image.open(os.path.join(self.root_dir, f"{frame['file_path']}.png")) + img = img.resize(self.img_wh, Image.LANCZOS) + img = self.transform(img) # (4, H, W) + valid_mask = (img[-1] > 0).flatten() # (H*W) valid color area + img = img.view(4, -1).permute(1, 0) # (H*W, 4) RGBA + img = img[:, :3] * img[:, -1:] + (1 - img[:, -1:]) # blend A to RGB + + rays_o, rays_d = get_rays(self.directions, c2w) + + rays = flow.concat( + [ + rays_o, + rays_d, + self.near * flow.ones_like(rays_o[:, :1]), + self.far * flow.ones_like(rays_o[:, :1]), + ], + 1, + ) # (H*W, 8) + sample = OrderedDict(rays=rays, rgbs=img, c2w=c2w, valid_mask=valid_mask) + else: + c2w = self.render_poses[idx][:3, :4] + rays_o, rays_d = get_rays(self.directions, c2w) + + rays = flow.concat( + [ + rays_o, + rays_d, + self.near * flow.ones_like(rays_o[:, :1]), + self.far * flow.ones_like(rays_o[:, :1]), + ], + 1, + ) # (H*W, 8) + sample = OrderedDict(rays=rays, c2w=c2w) + return trun_dict_to_instance(sample) + + +class LLFFDataset(NerfBaseDataset): + def __init__( + self, + root_dir, + split="train", + img_wh=(504, 378), + spheric_poses=False, + val_num=1, + batchsize=1024, + ): + """ + Args: + root_dir: str, + split: str, + img_wh: tuple, + spheric_poses: bool, whether the images are taken in a spheric inward-facing manner + default: False (forward-facing) + val_num: int, number of val images (used for multigpu training, validate same image + for all gpus) + batchsize: int, batchsize of rays + """ + super(LLFFDataset, self).__init__(root_dir, split, img_wh) + self.spheric_poses = spheric_poses + self.val_num = max(1, val_num) # at least 1 + self.batchsize = batchsize + self.load_meta() + # build render_poses for inference + up = normalize(self.poses[:, :3, 1].sum(0)) + tt = self.poses[:, :3, 3] + rads = np.percentile(np.abs(tt), 90, 0) + close_depth, inf_depth = self.bounds.min() * 0.9, self.bounds.max() * 5.0 + dt = 0.75 + focal = 1.0 / (((1.0 - dt) / close_depth + dt / inf_depth)) + zdelta = close_depth * 0.2 + N_views = 120 + N_rots = 2 + hwf = self.hwf + center = self.poses[:, :3, 3].mean(0) + vec2 = normalize(self.poses[:, :3, 2].sum(0)) + up = self.poses[:, :3, 1].sum(0) + c2w = viewmatrix(vec2, up, center) + self.render_poses = flow.Tensor( + render_path_spiral( + c2w, hwf, normalize(up), rads, focal, zdelta, zrate=0.5, rots=N_rots, N=N_views + ) + ) # use for test + self.white_back = False + + def load_meta(self): + poses_bounds = np.load(os.path.join(self.root_dir, "poses_bounds.npy")) # (N_images, 17) + self.image_paths = sorted(glob.glob(os.path.join(self.root_dir, "images/*"))) + if self.split in ["train", "val"]: + assert len(poses_bounds) == len( + self.image_paths + ), "Mismatch between number of images and number of poses! Please rerun COLMAP!" + poses = poses_bounds[:, :15].reshape(-1, 3, 5) # (N_images, 3, 5) + self.bounds = poses_bounds[:, -2:] # (N_images, 2) + H, W, self.focal = poses[0, :, -1] # original intrinsics, same for all images + H, W, self.focal = H.item(), W.item(), self.focal.item() + assert ( + H * self.img_wh[0] == W * self.img_wh[1] + ), f"You must set @img_wh to have the same aspect ratio as ({W}, {H}) !" + self.focal *= self.img_wh[0] / W + poses = np.concatenate([poses[..., 1:2], -poses[..., :1], poses[..., 2:4]], -1) + # (N_images, 3, 4) exclude H, W, focal + self.poses, self.pose_avg = center_poses(poses) + distances_from_center = np.linalg.norm(self.poses[..., 3], axis=1) + val_idx = np.argmin(distances_from_center) # choose val image as the closest to + near_original = self.bounds.min() + scale_factor = near_original * 0.75 # 0.75 is the default parameter + self.bounds /= scale_factor + self.poses[..., 3] /= scale_factor + self.directions = get_ray_directions( + self.img_wh[1], self.img_wh[0], self.focal + ) # (H, W, 3) + self.hwf = np.array([self.img_wh[1], self.img_wh[0], self.focal]) + if self.split == "train": # create buffer of all rays and rgb data + # use first N_images-1 to train, the LAST is val + self.all_rays = [] + self.all_rgbs = [] + self.indexs = [ + i * self.img_wh[0] * self.img_wh[1] for i in range(len(self.image_paths) - 1) + ] + for i, image_path in enumerate(self.image_paths): + if i == val_idx: # exclude the val image + continue + c2w = flow.Tensor(self.poses[i]) + img = Image.open(image_path).convert("RGB") + assert ( + img.size[1] * self.img_wh[0] == img.size[0] * self.img_wh[1] + ), f"{image_path} has different aspect ratio than img_wh, please check your data!" + img = img.resize(self.img_wh, Image.LANCZOS) + img = self.transform(img) # (3, h, w) + img = img.view(3, -1).permute(1, 0) # (h*w, 3) RGB + self.all_rgbs += [img] + + rays_o, rays_d = get_rays(self.directions, c2w) # both (h*w, 3) + if not self.spheric_poses: + near, far = 0, 1 + rays_o, rays_d = get_ndc_rays( + self.img_wh[1], self.img_wh[0], self.focal, 1.0, rays_o, rays_d + ) + else: + near = self.bounds.min() + far = min(8 * near, self.bounds.max()) # focus on central object only + + self.all_rays += [ + flow.concat( + [ + rays_o, + rays_d, + near * flow.ones_like(rays_o[:, :1]), + far * flow.ones_like(rays_o[:, :1]), + ], + 1, + ) + ] # (h*w, 8) + + self.all_rays = flow.cat(self.all_rays, 0) # ((N_images-1)*h*w, 8) + self.all_rgbs = flow.cat(self.all_rgbs, 0) # ((N_images-1)*h*w, 3) + self.num_iter = 0 + self.dH = int(self.img_wh[0] // 2 * 0.5) + self.dW = int(self.img_wh[1] // 2 * 0.5) + + elif self.split == "val": + self.c2w_val = self.poses[val_idx] + self.image_path_val = self.image_paths[val_idx] + + else: # for testing, create a parametric rendering path + if self.split.endswith("train"): # test on training set + self.poses_test = self.poses + elif not self.spheric_poses: + focus_depth = 3.5 # hardcoded, this is numerically close to the formula + # given in the original repo. Mathematically if near=1 + # and far=infinity, then this number will converge to 4 + radii = np.percentile(np.abs(self.poses[..., 3]), 90, axis=0) + self.poses_test = create_spiral_poses(radii, focus_depth) + else: + radius = 1.1 * self.bounds.min() + self.poses_test = create_spheric_poses(radius) + + def __len__(self): + if self.split == "train": + return int(len(self.all_rays) / self.batchsize) + elif self.split == "vis": + return len(self.render_poses) + elif self.split == "val": + return self.val_num + elif self.split == "test": + return len(self.poses_test) + + def __getitem__(self, idx): + if self.split == "train": # use data in the buffers + idx = idx % len(self.indexs) + if self.num_iter < 500: + coords = flow.stack( + flow.meshgrid( + flow.linspace( + self.img_wh[1] // 2 - self.dH, + self.img_wh[1] // 2 + self.dH - 1, + 2 * self.dH, + ), + flow.linspace( + self.img_wh[0] // 2 - self.dW, + self.img_wh[0] // 2 + self.dW - 1, + 2 * self.dW, + ), + ), + -1, + ) + else: + coords = flow.stack( + flow.meshgrid( + flow.linspace(0, self.img_wh[1] - 1, self.img_wh[1]), + flow.linspace(0, self.img_wh[0] - 1, self.img_wh[0]), + ), + -1, + ) # (H, W, 2) + coords = flow.reshape(coords, [-1, 2]) # (H * W, 2) + select_inds = np.random.choice( + coords.shape[0], size=[self.batchsize], replace=False + ) # (N_rand,) + select_coords = coords[select_inds].long() # (N_rand, 2) + rays = self.all_rays[ + self.indexs[idx] : self.indexs[idx] + self.img_wh[0] * self.img_wh[1] + ] + rgbs = self.all_rgbs[ + self.indexs[idx] : self.indexs[idx] + self.img_wh[0] * self.img_wh[1] + ] + rays = rays.view(self.img_wh[1], self.img_wh[0], -1) + rgbs = rgbs.view(self.img_wh[1], self.img_wh[0], -1) + rays = rays[select_coords[:, 0], select_coords[:, 1]] # (N_rand, 3) + rgbs = rgbs[select_coords[:, 0], select_coords[:, 1]] # (N_rand, 3) + self.num_iter += 1 + sample = OrderedDict(rays=rays, rgbs=rgbs) # a alignment point with nerf_pytorch + + elif self.split in ["val", "test"]: + if self.split == "val": + c2w = flow.Tensor(self.c2w_val) + else: + c2w = flow.Tensor(self.poses_test[idx]) + + rays_o, rays_d = get_rays(self.directions, c2w) + if not self.spheric_poses: + near, far = 0, 1 + rays_o, rays_d = get_ndc_rays( + self.img_wh[1], self.img_wh[0], self.focal, 1.0, rays_o, rays_d + ) + else: + near = self.bounds.min() + far = min(8 * near, self.bounds.max()) + + rays = flow.cat( + [ + rays_o, + rays_d, + near * flow.ones_like(rays_o[:, :1]), + far * flow.ones_like(rays_o[:, :1]), + ], + 1, + ) # (h*w, 8) + + sample = OrderedDict(rays=rays, c2w=c2w) + if self.split == "val": + img = Image.open(self.image_path_val).convert("RGB") + img = img.resize(self.img_wh, Image.LANCZOS) + img = self.transform(img) # (3, h, w) + img = img.view(3, -1).permute(1, 0) # (h*w, 3) + sample["rgbs"] = img + else: + c2w = self.render_poses[idx][:3, :4] + rays_o, rays_d = get_rays(self.directions, c2w) + if not self.spheric_poses: + near, far = 0, 1 + rays_o, rays_d = get_ndc_rays( + self.img_wh[1], self.img_wh[0], self.focal, 1.0, rays_o, rays_d + ) + else: + near = self.bounds.min() + far = min(8 * near, self.bounds.max()) + rays = flow.concat( + [ + rays_o, + rays_d, + near * flow.ones_like(rays_o[:, :1]), + far * flow.ones_like(rays_o[:, :1]), + ], + 1, + ) # (H*W, 8) + sample = OrderedDict(rays=rays, c2w=c2w) + return trun_dict_to_instance(sample) diff --git a/projects/NeRF/evaluation/nerf_evaluator.py b/projects/NeRF/evaluation/nerf_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..ec8ad690f93b718f276787fb30379634f9c06fd8 --- /dev/null +++ b/projects/NeRF/evaluation/nerf_evaluator.py @@ -0,0 +1,162 @@ +import copy +import math +import os +from collections import OrderedDict +from datetime import datetime + +import cv2 +import flowvision.transforms as T +import imageio +import numpy as np +import oneflow as flow +from PIL import Image + +from libai.evaluation.cls_evaluator import ClsEvaluator +from libai.utils import distributed as dist + + +class NerfEvaluator(ClsEvaluator): + def __init__(self, img_wh, image_save_path=None): + """ + Args: + img_wh (tuple(int)): the width and height of the images in the validation set + image_save_path (str): location of image storage + """ + super().__init__(topk=(1, 5)) + self.img_wh = img_wh + self.image_save_path = ( + str(os.path.dirname(os.path.realpath(__file__))) + "/../images" + if image_save_path is None + else image_save_path + ) + if not os.path.exists(self.image_save_path): + os.makedirs(self.image_save_path) + self.toimage = T.ToPILImage() + + def current_time(self): + currentDateAndTime = datetime.now() + currentTime = currentDateAndTime.strftime("%H_%M_%S") + return currentTime + + def process(self, inputs, outputs): + """ + Inputs: + inputs (dict): Inputs to NeRF System + outputs (dict): Outputs to NeRF System + + Outputs: + None + """ + losses, rgbs = ( + outputs["losses"], + outputs["rgbs"].squeeze(0), + ) + typ = list(outputs.keys())[1] + outputs.pop(typ) + outputs.pop("losses") + outputs.pop("rgbs") + results = {k: v.squeeze(0) for k, v in outputs.items()} + if len(self._predictions) == 0: + W, H = self.img_wh + img = results[f"rgb_{typ}"].view(H, W, 3).cpu() + img = img.permute(2, 0, 1) # (3, H, W) + img_gt = rgbs.view(H, W, 3).permute(2, 0, 1).cpu() # (3, H, W) + depth = self.visualize_depth(results[f"depth_{typ}"].view(H, W)) # (3, H, W) + img = self.toimage(img) + img_gt = self.toimage(img_gt) + depth = self.toimage(depth) + img.save( + os.path.join(self.image_save_path, f"img_{self.current_time()}.png"), quality=100 + ) + img_gt.save( + os.path.join(self.image_save_path, f"img_gt_{self.current_time()}.png"), quality=100 + ) + depth.save( + os.path.join(self.image_save_path, f"depth_{self.current_time()}.png"), quality=100 + ) + psnr = self.psnr(results[f"rgb_{typ}"], rgbs) + self._predictions.append({"losses": losses.item(), "psnr": psnr.item()}) + + def evaluate(self): + if not dist.is_main_process(): + return {} + else: + predictions = self._predictions + + total_correct_num = OrderedDict() + total_correct_num["losses"] = 0 + total_correct_num["psnr"] = 0 + total_samples = 0 + for prediction in predictions: + losses = prediction["losses"] + psnr = prediction["psnr"] + total_correct_num["losses"] += losses + total_correct_num["psnr"] += psnr + total_samples += 1 + + self._results = OrderedDict() + for key, value in total_correct_num.items(): + self._results[key] = value / total_samples + + return copy.deepcopy(self._results) + + def visualize_depth(self, depth, cmap=cv2.COLORMAP_JET): + x = depth.cpu().numpy() + x = np.nan_to_num(x) # change nan to 0 + mi = np.min(x) # get minimum depth + ma = np.max(x) + x = (x - mi) / (ma - mi + 1e-8) # normalize to 0~1 + x = (255 * x).astype(np.uint8) + x_ = Image.fromarray(cv2.applyColorMap(x, cmap)) + x_ = T.ToTensor()(x_) # (3, H, W) + return x_ + + def mse(self, image_pred, image_gt, valid_mask=None, reduction="mean"): + value = (image_pred - image_gt) ** 2 + if valid_mask is not None: + value = value[valid_mask] + if reduction == "mean": + return flow.mean(value) + return value + + def psnr(self, image_pred, image_gt, valid_mask=None, reduction="mean"): + return -10 * flow.log(self.mse(image_pred, image_gt, valid_mask, reduction)) / math.log(10) + + +class NerfVisEvaluator(NerfEvaluator): + def __init__(self, img_wh, pose_dir_len, name): + """ + Args: + img_wh (tuple(int)): the width and height of the images in the validation set + """ + super().__init__(img_wh=img_wh) + self.image_list = [] + self.pose_dir_len = pose_dir_len + self.name = name + self.mp4_save_path = self.image_save_path + + def to8b(self, x): + return (255 * np.clip(x, 0, 1)).astype(np.uint8) + + def process(self, inputs, outputs): + """ + Inputs: + inputs (dict): Inputs to NeRF System + outputs (dict): Outputs to NeRF System + + Outputs: + None + """ + typ = list(outputs.keys())[0] + outputs.pop(typ) + results = {k: v.squeeze(0) for k, v in outputs.items()} + W, H = self.img_wh + img = results[f"rgb_{typ}"].view(H, W, 3).cpu().numpy() + self.image_list.append(img) + self._predictions.append({"losses": 0.0, "psnr": 0.0}) + if len(self._predictions) == self.pose_dir_len: + mp4_save_path = os.path.join(self.mp4_save_path, f"{self.name}.mp4") + imageio.mimwrite( + mp4_save_path, self.to8b(np.stack(self.image_list, 0)), fps=30, quality=8 + ) + print("successfully save mp4 file!") diff --git a/projects/NeRF/modeling/NeRF.py b/projects/NeRF/modeling/NeRF.py new file mode 100644 index 0000000000000000000000000000000000000000..50d8c4bedd324e27035612a647089e049d4535d2 --- /dev/null +++ b/projects/NeRF/modeling/NeRF.py @@ -0,0 +1,146 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import oneflow as flow +import oneflow.nn as nn +import oneflow.nn.functional as F + +from libai.utils import distributed as dist + + +class Embedding(nn.Module): + def __init__(self, in_channels, N_freqs, logscale=True): + """ + Defines a function that embeds x to (x, sin(2^k x), cos(2^k x), ...) + in_channels: number of input channels (3 for both xyz and direction) + """ + super(Embedding, self).__init__() + self.N_freqs = N_freqs + self.in_channels = in_channels + self.funcs = [flow.sin, flow.cos] + self.out_channels = in_channels * (len(self.funcs) * N_freqs + 1) + + if logscale: + freq_bands = 2 ** flow.linspace(0, N_freqs - 1, N_freqs) + else: + freq_bands = flow.linspace(1, 2 ** (N_freqs - 1), N_freqs).cuda() + self.register_buffer( + "freq_bands", + freq_bands.to_global( + placement=dist.get_layer_placement(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ), + persistent=False, + ) + + def forward(self, x): + """ + Embeds x to (x, sin(2^k x), cos(2^k x), ...) + Different from the paper, "x" is also in the output + See https://github.com/bmild/nerf/issues/12 + + Inputs: + x (Tensor): (B, self.in_channels) + + Outputs: + out (Tensor): (B, self.out_channels) + """ + out = [x] + for freq in self.freq_bands: + for func in self.funcs: + m = func(freq * x) + out += [m] + + return flow.cat(out, -1) + + +class NeRF(nn.Module): # a alignment point with nerf_pytorch + def __init__( + self, D=8, W=256, input_ch=63, input_ch_views=27, output_ch=5, skips=[4], use_viewdirs=True + ): + """ + D: number of layers for density (sigma) encoder + W: number of hidden units in each layer + input_ch: number of input channels for xyz (3+3*10*2=63 by default) + input_ch_views: number of input channels for direction (3+3*4*2=27 by default) + output_ch: number of output channels + skips: add skip connection in the Dth layer + """ + super(NeRF, self).__init__() + self.D = D + self.W = W + self.input_ch = input_ch + self.input_ch_views = input_ch_views + self.skips = skips + self.use_viewdirs = use_viewdirs + + self.pts_linears = nn.ModuleList( + [nn.Linear(input_ch, W)] + + [ + nn.Linear(W, W) if i not in self.skips else nn.Linear(W + input_ch, W) + for i in range(D - 1) + ] + ) + self.views_linears = nn.ModuleList([nn.Linear(input_ch_views + W, W // 2)]) + if use_viewdirs: + self.feature_linear = nn.Linear(W, W) + self.alpha_linear = nn.Linear(W, 1) + self.rgb_linear = nn.Linear(W // 2, 3) + else: + self.output_linear = nn.Linear(W, output_ch) + + def forward(self, x, sigma_only=False): + """ + Encodes input (xyz+dir) to rgb+sigma (not ready to render yet). + For rendering this ray, please see rendering.py + + Inputs: + x (Tensor): (B, self.in_channels_xyz+self.in_channels_dir) + the embedded vector of position and direction + sigma_only (bool): whether to infer sigma only. If True, + x is of shape (B, self.in_channels_xyz) + + Outputs: + if sigma_ony: + sigma (Tensor): (B, 1) sigma + else: + out (Tensor): (B, 4), rgb and sigma + """ + if not sigma_only: + input_pts, input_views = flow.split(x, [self.input_ch, self.input_ch_views], dim=-1) + else: + input_pts = x + h = input_pts + for i, l in enumerate(self.pts_linears): + h = self.pts_linears[i](h) + h = F.relu(h) + if i in self.skips: + h = flow.cat([input_pts, h], -1) + + if self.use_viewdirs: + alpha = self.alpha_linear(h) + if sigma_only: + return alpha + feature = self.feature_linear(h) + h = flow.cat([feature, input_views], -1) + + for i, l in enumerate(self.views_linears): + h = self.views_linears[i](h) + h = F.relu(h) + + rgb = self.rgb_linear(h).sigmoid() # sigmoid + outputs = flow.cat([rgb, alpha], -1) + else: + outputs = self.output_linear(h) + return outputs diff --git a/projects/NeRF/modeling/System.py b/projects/NeRF/modeling/System.py new file mode 100644 index 0000000000000000000000000000000000000000..befb940ea47845cb0886476c04d5a3ac89281835 --- /dev/null +++ b/projects/NeRF/modeling/System.py @@ -0,0 +1,463 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import collections +from collections import defaultdict + +import oneflow as flow +import oneflow.nn as nn + +from libai.config.config import configurable +from projects.NeRF.modeling.NeRF import Embedding, NeRF + + +class NerfSystem(nn.Module): + @configurable + def __init__( + self, + D=8, + W=256, + in_channels_xyz=63, + in_channels_dir=27, + skips=[4], + N_samples=64, + use_disp=False, + perturb=1.0, + noise_std=1.0, + N_importance=128, + chunk=32 * 1204, + dataset_type="Blender", + loss_func=None, + ): + """ + Args: + D (int): number of layers for density (sigma) encoder + W (int): number of hidden units in each layer + in_channels_xyz (int): number of input channels for xyz (3+3*10*2=63 by default) + in_channels_dir (int): number of input channels for direction (3+3*4*2=27 by default) + skips (list(int)): add skip connection in the Dth layer + N_samples (int): number of coarse samples + use_disp (bool): use disparity depth sampling + perturb (float): factor to perturb depth sampling points + noise_std (float): std dev of noise added to regularize sigma + N_importance (int): number of additional fine samples + chunk (int): chunk size to split the input to avoid OOM + dataset_type (str): the dataset applied for training and evaluating + loss_func (callable): type of loss function + """ + super(NerfSystem, self).__init__() + self.N_samples = N_samples + self.use_disp = use_disp + self.perturb = perturb + self.noise_std = noise_std + self.N_importance = N_importance + self.chunk = chunk + self.white_back = True if dataset_type == "Blender" else False + self.loss_func = nn.MSELoss() if loss_func is None else loss_func + self.embedding_xyz = Embedding(3, 10) # 10 is the default number + self.embedding_dir = Embedding(3, 4) # 4 is the default number + self.nerf_coarse = NeRF( + D=D, + W=W, + input_ch=in_channels_xyz, + input_ch_views=in_channels_dir, + output_ch=5, + skips=skips, + ) + self.models = [self.nerf_coarse] + if N_importance > 0: + self.nerf_fine = NeRF( + D=D, + W=W, + input_ch=in_channels_xyz, + input_ch_views=in_channels_dir, + output_ch=5, + skips=skips, + ) + self.models += [self.nerf_fine] + + @classmethod + def from_config(cls, cfg): + return { + "D": cfg.D, + "W": cfg.W, + "in_channels_xyz": cfg.in_channels_xyz, + "in_channels_dir": cfg.in_channels_dir, + "skips": cfg.skips, + "N_samples": cfg.N_samples, + "use_disp": cfg.use_disp, + "perturb": cfg.perturb, + "noise_std": cfg.noise_std, + "N_importance": cfg.N_importance, + "chunk": cfg.chunk, + "dataset_type": cfg.dataset_type, + "loss_func": cfg.loss_func, + } + + def sample_pdf(self, bins, weights, N_importance, det=False, eps=1e-5): + """ + Sample @N_importance samples from @bins with distribution defined by @weights. + + Inputs: + bins (tensor): (N_rays, N_samples_+1) where N_samples_ is "the number of + coarse samples per ray - 2" + weights (tensor): (N_rays, N_samples_) + N_importance (int): the number of samples to draw from the distribution + det (bool): deterministic or not + eps (float): a small number to prevent division by zero + + Outputs: + samples: the sampled samples + """ + N_rays, N_samples_ = weights.shape + weights = weights + eps # prevent division by zero (don't do inplace op!) + pdf = weights / flow.sum(weights, -1, keepdim=True) # (N_rays, N_samples_) + cdf = flow.cumsum(pdf, -1) # (N_rays, N_samples), cumulative distribution function + cdf = flow.cat([flow.zeros_like(cdf[:, :1]), cdf], -1) # (N_rays, N_samples_+1) + + # padded to 0~1 inclusive + if det: + u = flow.linspace(0, 1, N_importance).to_global(placement=bins.placement, sbp=bins.sbp) + u = u.expand(N_rays, N_importance) + else: + u = flow.rand(N_rays, N_importance).to_global(placement=bins.placement, sbp=bins.sbp) + u = u.contiguous() + + inds = flow.searchsorted(cdf, u, right=True) + below = flow.max(flow.zeros_like(inds - 1), inds - 1) + above = flow.min((cdf.shape[-1] - 1) * flow.ones_like(inds), inds) + inds_g = flow.stack([below, above], -1) # (batch, N_samples, 2) + + # cdf_g = tf.gather(cdf, inds_g, axis=-1, batch_dims=len(inds_g.shape)-2) + # bins_g = tf.gather(bins, inds_g, axis=-1, batch_dims=len(inds_g.shape)-2) + matched_shape = [inds_g.shape[0], inds_g.shape[1], cdf.shape[-1]] + cdf_g = flow.gather(cdf.unsqueeze(1).expand(matched_shape), 2, inds_g) + bins_g = flow.gather(bins.unsqueeze(1).expand(matched_shape), 2, inds_g) + + denom = cdf_g[..., 1] - cdf_g[..., 0] + denom = flow.where(denom < 1e-5, flow.ones_like(denom), denom) + t = (u - cdf_g[..., 0]) / denom + samples = bins_g[..., 0] + t * (bins_g[..., 1] - bins_g[..., 0]) + return samples + + def inference( + self, + N_rays, + model, + embedding_xyz, + xyz_, + no_norm_dir_, + dir_, + dir_embedded, + z_vals, + noise_std=1, + chunk=1024 * 32, + white_back=False, + weights_only=False, + ): + """ + Helper function that performs model inference. + + Inputs: + N_rays (tensor): rays (N_rays, 3+3+2), ray origins, directions and near, + far depth bounds + model (nn.Module): NeRF model (coarse or fine) + embedding_xyz (nn.Module): embedding module for xyz + xyz_ (tensor): (N_rays, N_samples_, 3) sampled positions + N_samples_ is the number of sampled points in each ray; + = N_samples for coarse model + = N_samples+N_importance for fine model + no_norm_dir_ (tensor): (N_rays, 3) ray directions without norm + dir_ (tensor): (N_rays, 3) ray directions with norm + dir_embedded (tensor): (N_rays, embed_dir_channels) embedded directions + z_vals (tensor): (N_rays, N_samples_) depths of the sampled positions + weights_only (tensor): do inference on sigma only or not + + Outputs: + if weights_only: + weights (tensor): (N_rays, N_samples_) weights of each sample + else: + rgb_final (tensor): (N_rays, 3) the final rgb image + depth_final (tensor): (N_rays) depth map + weights (tensor): (N_rays, N_samples_): weights of each sample + """ + N_samples_ = xyz_.shape[1] + # Embed directions + xyz_ = xyz_.view(-1, 3) # (N_rays*N_samples_, 3) + if not weights_only: + dir_embedded = dir_embedded[:, None].expand( + dir_embedded.shape[0], N_samples_, dir_embedded.shape[1] + ) + dir_embedded = dir_embedded.reshape(-1, dir_embedded.shape[-1]) + + # Perform model inference to get rgb and raw sigma + B = xyz_.shape[0] + out_chunks = [] + for i in range(0, B, chunk): + # Embed positions by chunk + xyz_embedded = embedding_xyz(xyz_[i : i + chunk]) + if not weights_only: + xyzdir_embedded = flow.cat([xyz_embedded, dir_embedded[i : i + chunk]], 1) + else: + xyzdir_embedded = xyz_embedded + out_chunk = model(xyzdir_embedded) + out_chunks = out_chunks + [out_chunk] + + out = flow.cat(out_chunks, 0) + if weights_only: + sigmas = out.view(N_rays, N_samples_) + else: + rgbsigma = out.view(N_rays, N_samples_, 4) + rgbs = rgbsigma[..., :3] # (N_rays, N_samples_, 3) + sigmas = rgbsigma[..., 3] # (N_rays, N_samples_) + + # Convert these values using volume rendering (Section 4) + deltas = z_vals[:, 1:].clone() - z_vals[:, :-1].clone() # (N_rays, N_samples_-1) + delta_inf = 1e10 * flow.ones_like(deltas[:, :1]).to_global( + sbp=deltas.sbp, placement=deltas.placement + ) # (N_rays, 1) the last delta is infinity + deltas = flow.cat([deltas, delta_inf], -1) # (N_rays, N_samples_) + + # Multiply each distance by the norm of its corresponding direction ray + # to convert to real world distance (accounts for non-unit directions). + deltas = deltas * flow.norm(no_norm_dir_.unsqueeze(1), dim=-1) + + noise = ( + flow.randn(sigmas.shape).to_global(placement=sigmas.placement, sbp=sigmas.sbp) + * noise_std + ) + + # compute alpha by the formula (3) + alphas = 1 - flow.exp(-deltas * flow.relu(sigmas + noise)) # (N_rays, N_samples_) + ne_alphas = 1 - alphas + 1e-10 + alphas_shifted = flow.cat( + [ + flow.ones_like(alphas[:, :1]).to_global(sbp=alphas.sbp, placement=alphas.placement), + ne_alphas, + ], + -1, + ) # [1, a1, a2, ...] + weights = alphas * flow.cumprod(alphas_shifted, -1)[:, :-1] # (N_rays, N_samples_) + # weights = alphas * alphas_shifted[:, :-1] # (N_rays, N_samples_) + weights_sum = weights.sum(1) # (N_rays), the accumulated opacity along the rays + # equals "1 - (1-a1)(1-a2)...(1-an)" mathematically + if weights_only: + return weights + + # compute final weighted outputs + rgb_final = flow.sum(weights.unsqueeze(-1) * rgbs, -2) # (N_rays, 3) + depth_final = flow.sum(weights * z_vals, -1) # (N_rays) + if white_back: + + rgb_final = rgb_final + (1 - weights_sum.unsqueeze(-1)) + + return rgb_final, depth_final, weights + + def render_rays( + self, + models, + embeddings, + rays, + N_samples=64, + use_disp=False, + perturb=0.0, + N_importance=0.0, + test_time=False, + noise_std=1.0, + chunk=1024 * 32, + white_back=False, + ): + + # Extract models from lists + model_coarse = models[0] + embedding_xyz = embeddings[0] + embedding_dir = embeddings[1] + + # Decompose the inputs + N_rays = rays.shape[0] + rays_o, rays_d = rays[:, 0:3], rays[:, 3:6] # both (N_rays, 3) + near, far = rays[:, 6:7], rays[:, 7:8] # both (N_rays, 1) + viewdirs = rays_d / flow.norm(rays_d, dim=-1, keepdim=True) + + # Embed direction + dir_embedded = embedding_dir(viewdirs) # (N_rays, embed_dir_channels) + + # Sample depth points + z_steps = flow.linspace(0, 1, N_samples).to_global( + sbp=rays.sbp, placement=rays.placement + ) # (N_samples) + if not use_disp: # use linear sampling in depth space + z_vals = near * (1 - z_steps) + far * z_steps + else: # use linear sampling in disparity space + z_vals = 1 / (1 / near * (1 - z_steps) + 1 / far * z_steps) + + z_vals = z_vals.expand(N_rays, N_samples) + + if perturb > 0: # perturb sampling depths (z_vals) + z_vals_mid = 0.5 * ( + z_vals[:, :-1] + z_vals[:, 1:] + ) # (N_rays, N_samples-1) interval mid points + # get intervals between samples + upper = flow.cat([z_vals_mid, z_vals[:, -1:]], -1) + lower = flow.cat([z_vals[:, :1], z_vals_mid], -1) + + v = flow.rand(z_vals.shape).to_global(sbp=rays.sbp, placement=rays.placement) + perturb_rand = perturb * v + z_vals = lower + (upper - lower) * perturb_rand + + xyz_coarse_sampled = rays_o.unsqueeze(1) + rays_d.unsqueeze(1) * z_vals.unsqueeze( + 2 + ) # (N_rays, N_samples, 3) + if test_time: + weights_coarse = self.inference( + rays.shape[0], + model_coarse, + embedding_xyz, + xyz_coarse_sampled, + rays_d, + viewdirs, + dir_embedded, + z_vals, + noise_std, + chunk, + white_back, + weights_only=True, + ) + result = {"opacity_coarse": weights_coarse.sum(1)} + else: + rgb_coarse, depth_coarse, weights_coarse = self.inference( + rays.shape[0], + model_coarse, + embedding_xyz, + xyz_coarse_sampled, + rays_d, + viewdirs, + dir_embedded, + z_vals, + noise_std, + chunk, + white_back, + weights_only=False, + ) + result = { + "rgb_coarse": rgb_coarse, + "depth_coarse": depth_coarse, + "opacity_coarse": weights_coarse.sum(1), + } + + if N_importance > 0: # sample points for fine model + z_vals_mid = 0.5 * ( + z_vals[:, :-1] + z_vals[:, 1:] + ) # (N_rays, N_samples-1) interval mid points + z_vals_ = self.sample_pdf( + z_vals_mid, weights_coarse[:, 1:-1], N_importance, det=(perturb == 0) + ).detach() + # detach so that grad doesn't propogate to weights_coarse from here + + z_vals, _ = flow.sort(flow.cat([z_vals, z_vals_], -1), -1) + + xyz_fine_sampled = rays_o.unsqueeze(1) + rays_d.unsqueeze(1) * z_vals.unsqueeze(2) + # (N_rays, N_samples+N_importance, 3) + + model_fine = models[1] + rgb_fine, depth_fine, weights_fine = self.inference( + rays.shape[0], + model_fine, + embedding_xyz, + xyz_fine_sampled, + rays_d, + viewdirs, + dir_embedded, + z_vals, + noise_std, + chunk, + white_back, + weights_only=False, + ) + result["rgb_fine"] = rgb_fine + result["depth_fine"] = depth_fine + result["opacity_fine"] = weights_fine.sum(1) + + return result + + def forward_features(self, rays): + """Do batched inference on rays using chunk.""" + B = rays.shape[0] + results = defaultdict(list) + for i in range(0, B, self.chunk): + rendered_ray_chunks = self.render_rays( + self.models, + [self.embedding_xyz, self.embedding_dir], + rays[i : i + self.chunk], + self.N_samples, + self.use_disp, + self.perturb, + self.N_importance, + False, + self.noise_std, + self.chunk, # chunk size is effective in val mode + self.white_back, + ) + + for k, v in rendered_ray_chunks.items(): + results[k] += [v] + for k, v in results.items(): + results[k] = flow.cat(v, 0) + return results + + def forward(self, rays, rgbs=None, c2w=None, valid_mask=None): + """ + Inputs: + rays (tensor): (batchsize, 3+3+2) the set of input rays samples + rgbs (tensor): (batchsize, 3) the set of input rgbs samples + c2w (tensor): (3, 4) transformation matrix from camera coordinate to world coordinate + valid_mask (tensor): (H W) valid color area + + Outputs: + re (dict): regarding the series of outputs such as rgbs and loss obtained from the + model predictions. + """ + if c2w is None: + rays = rays.squeeze() # (H*W, 3) + rgbs = rgbs.squeeze() # (H*W, 3) + results = self.forward_features(rays) + losses = self.loss_func(results["rgb_coarse"], rgbs) + if "rgb_fine" in results: + losses += self.loss_func(results["rgb_fine"], rgbs) + return {"losses": losses} + else: + if rgbs is None: + rays = rays.squeeze() # (H*W, 3) + results = self.forward_features(rays) + typ = "fine" if "rgb_fine" in results else "coarse" + re = collections.OrderedDict() + re[typ] = flow.Tensor([0.0]).to_global(sbp=rays.sbp, placement=rays.placement) + for key, value in results.items(): + re[key] = value.unsqueeze(0) + return re + else: + rays = rays.squeeze() # (H*W, 3) + rgbs = rgbs.squeeze() # (H*W, 3) + results = self.forward_features(rays) + losses = self.loss_func(results["rgb_coarse"], rgbs) + if "rgb_fine" in results: + losses += self.loss_func(results["rgb_fine"], rgbs) + typ = "fine" if "rgb_fine" in results else "coarse" + re = collections.OrderedDict() + re["losses"] = losses + re[typ] = flow.Tensor([0.0]).to_global(sbp=losses.sbp, placement=losses.placement) + for key, value in results.items(): + re[key] = value.unsqueeze(0) + re["rgbs"] = rgbs.unsqueeze(0) + return re diff --git a/projects/NeRF/optimizers/Radam.py b/projects/NeRF/optimizers/Radam.py new file mode 100644 index 0000000000000000000000000000000000000000..d431e120d70686fa1945734dc9892bf93f63ea46 --- /dev/null +++ b/projects/NeRF/optimizers/Radam.py @@ -0,0 +1,113 @@ +import math + +import oneflow as flow +from oneflow.optim import Optimizer + + +class RAdam(Optimizer): + def __init__( + self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0, degenerated_to_sgd=True + ): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) + + self.degenerated_to_sgd = degenerated_to_sgd + if isinstance(params, (list, tuple)) and len(params) > 0 and isinstance(params[0], dict): + for param in params: + if "betas" in param and ( + param["betas"][0] != betas[0] or param["betas"][1] != betas[1] + ): + param["buffer"] = [[None, None, None] for _ in range(10)] + defaults = dict( + lr=lr, + betas=betas, + eps=eps, + weight_decay=weight_decay, + buffer=[[None, None, None] for _ in range(10)], + ) + super(RAdam, self).__init__(params, defaults) + + def __setstate__(self, state): + super(RAdam, self).__setstate__(state) + + def step(self, closure=None): + + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + + for p in group["params"]: + if p.grad is None: + continue + grad = p.grad.data.float() + if grad.is_sparse: + raise RuntimeError("RAdam does not support sparse gradients") + + p_data_fp32 = p.data.float() + + state = self.state[p] + + if len(state) == 0: + state["step"] = 0 + state["exp_avg"] = flow.zeros_like(p_data_fp32) + state["exp_avg_sq"] = flow.zeros_like(p_data_fp32) + else: + state["exp_avg"] = state["exp_avg"].type_as(p_data_fp32) + state["exp_avg_sq"] = state["exp_avg_sq"].type_as(p_data_fp32) + + exp_avg, exp_avg_sq = state["exp_avg"], state["exp_avg_sq"] + beta1, beta2 = group["betas"] + + exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) + exp_avg.mul_(beta1).add_(1 - beta1, grad) + + state["step"] += 1 + buffered = group["buffer"][int(state["step"] % 10)] + if state["step"] == buffered[0]: + N_sma, step_size = buffered[1], buffered[2] + else: + buffered[0] = state["step"] + beta2_t = beta2 ** state["step"] + N_sma_max = 2 / (1 - beta2) - 1 + N_sma = N_sma_max - 2 * state["step"] * beta2_t / (1 - beta2_t) + buffered[1] = N_sma + + # more conservative since it's an approximated value + if N_sma >= 5: + step_size = math.sqrt( + (1 - beta2_t) + * (N_sma - 4) + / (N_sma_max - 4) + * (N_sma - 2) + / N_sma + * N_sma_max + / (N_sma_max - 2) + ) / (1 - beta1 ** state["step"]) + elif self.degenerated_to_sgd: + step_size = 1.0 / (1 - beta1 ** state["step"]) + else: + step_size = -1 + buffered[2] = step_size + + # more conservative since it's an approximated value + if N_sma >= 5: + if group["weight_decay"] != 0: + p_data_fp32.add_(-group["weight_decay"] * group["lr"], p_data_fp32) + denom = exp_avg_sq.sqrt().add_(group["eps"]) + p_data_fp32.addcdiv_(-step_size * group["lr"], exp_avg, denom) + p.data.copy_(p_data_fp32) + elif step_size > 0: + if group["weight_decay"] != 0: + p_data_fp32.add_(-group["weight_decay"] * group["lr"], p_data_fp32) + p_data_fp32.add_(-step_size * group["lr"], exp_avg) + p.data.copy_(p_data_fp32) + + return loss diff --git a/projects/NeRF/optimizers/Ranger.py b/projects/NeRF/optimizers/Ranger.py new file mode 100644 index 0000000000000000000000000000000000000000..f61c2467b4263e610e3cc1df34b650ab6287b343 --- /dev/null +++ b/projects/NeRF/optimizers/Ranger.py @@ -0,0 +1,143 @@ +import math + +import oneflow as flow +from oneflow.optim import Optimizer + + +class Ranger(Optimizer): + def __init__( + self, + params, + lr=1e-3, + alpha=0.5, + k=6, + N_sma_threshhold=5, + betas=(0.95, 0.999), + eps=1e-5, + weight_decay=0, + ): + # parameter checks + if not 0.0 <= alpha <= 1.0: + raise ValueError(f"Invalid slow update rate: {alpha}") + if not 1 <= k: + raise ValueError(f"Invalid lookahead steps: {k}") + if not lr > 0: + raise ValueError(f"Invalid Learning Rate: {lr}") + if not eps > 0: + raise ValueError(f"Invalid eps: {eps}") + + defaults = dict( + lr=lr, + alpha=alpha, + k=k, + step_counter=0, + betas=betas, + N_sma_threshhold=N_sma_threshhold, + eps=eps, + weight_decay=weight_decay, + ) + super().__init__(params, defaults) + + # adjustable threshold + self.N_sma_threshhold = N_sma_threshhold + + # now we can get to work... + # removed as we now use step from RAdam...no need for duplicate step counting + # for group in self.param_groups: + # group["step_counter"] = 0 + # print("group step counter init") + + # look ahead params + self.alpha = alpha + self.k = k + + # radam buffer for state + self.radam_buffer = [[None, None, None] for ind in range(10)] + + def __setstate__(self, state): + print("set state called") + super(Ranger, self).__setstate__(state) + + def step(self, closure=None): + loss = None + + for group in self.param_groups: + + for p in group["params"]: + if p.grad is None: + continue + grad = p.grad.data.float() + if grad.is_sparse: + raise RuntimeError("Ranger optimizer does not support sparse gradients") + + p_data_fp32 = p.data.float() + + state = self.state[p] # get state dict for this param + + if len(state) == 0: + state["step"] = 0 + state["exp_avg"] = flow.zeros_like(p_data_fp32) + state["exp_avg_sq"] = flow.zeros_like(p_data_fp32) + + # look ahead weight storage now in state dict + state["slow_buffer"] = p.data.clone() + + else: + state["exp_avg"] = state["exp_avg"].type_as(p_data_fp32) + state["exp_avg_sq"] = state["exp_avg_sq"].type_as(p_data_fp32) + + # begin computations + exp_avg, exp_avg_sq = state["exp_avg"], state["exp_avg_sq"] + beta1, beta2 = group["betas"] + + # compute variance mov avg + exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad) + # compute mean moving avg + exp_avg.mul_(beta1).add_(1 - beta1, grad) + + state["step"] += 1 + + buffered = self.radam_buffer[int(state["step"] % 10)] + if state["step"] == buffered[0]: + N_sma, step_size = buffered[1], buffered[2] + else: + buffered[0] = state["step"] + beta2_t = beta2 ** state["step"] + N_sma_max = 2 / (1 - beta2) - 1 + N_sma = N_sma_max - 2 * state["step"] * beta2_t / (1 - beta2_t) + buffered[1] = N_sma + if N_sma > self.N_sma_threshhold: + step_size = math.sqrt( + (1 - beta2_t) + * (N_sma - 4) + / (N_sma_max - 4) + * (N_sma - 2) + / N_sma + * N_sma_max + / (N_sma_max - 2) + ) / (1 - beta1 ** state["step"]) + else: + step_size = 1.0 / (1 - beta1 ** state["step"]) + buffered[2] = step_size + + if group["weight_decay"] != 0: + p_data_fp32.add_(-group["weight_decay"] * group["lr"], p_data_fp32) + + if N_sma > self.N_sma_threshhold: + denom = exp_avg_sq.sqrt().add_(group["eps"]) + p_data_fp32.addcdiv_(-step_size * group["lr"], exp_avg, denom) + else: + p_data_fp32.add_(-step_size * group["lr"], exp_avg) + + p.data.copy_(p_data_fp32) + + # integrated look ahead... + # we do it at the param level instead of group level + if state["step"] % group["k"] == 0: + slow_p = state["slow_buffer"] # get access to slow param tensor + slow_p.add_( + self.alpha, p.data - slow_p + ) # (fast weights - slow weights) * alpha + p.data.copy_(slow_p) # copy interpolated weights to RAdam param tensor + + return loss diff --git a/projects/NeRF/optimizers/__init__.py b/projects/NeRF/optimizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c9a29ec89de4aa5e5446b11ec580e46c87a0c6dc --- /dev/null +++ b/projects/NeRF/optimizers/__init__.py @@ -0,0 +1,2 @@ +from .Radam import RAdam +from .Ranger import Ranger diff --git a/projects/NeRF/readme.md b/projects/NeRF/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..3bbd0e89e5e7616532d05ccaff2be6c509a5eaf9 --- /dev/null +++ b/projects/NeRF/readme.md @@ -0,0 +1,47 @@ +# NeRF + +Reproduce NeRF with OneFlow, which effect are equivalent to nerf_pytorch and nerf_pl [nerf_pytorch](https://github.com/yenchenlin/nerf-pytorch) and [nerf_pl](https://github.com/kwea123/nerf_pl). + +## Introduce +The NeRF is used for 3D view rendering and [NeRF](https://arxiv.org/abs/2003.08934). + +## Training NeRF +Training NeRF only on 1 GPUs. + + +### 1. Prepare the training data (blender and llff) +Prepare the training data by running: +Download `nerf_synthetic.zip` and `nerf_llff_data.zip` from [here](https://drive.google.com/drive/folders/128yBriW1IG_3NJ5Rp7APSTZsJqdJdfc1) and unzip them. + +NOTE: If your operating system without GUI is not convenient for pulling datasets, then please download the following script according +to (This script only downloads the blender dataset, to download the other dataset llff please change the filename and fileid): +```bash +#!/bin/bash +# Download zip dataset from Google Drive +filename='nerf_synthetic.zip' +fileid='18JxhpWD-4ZmuFKLzKlAw-w5PpzZxXOcG' +wget --load-cookies /tmp/cookies.txt "https://drive.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://drive.google.com/uc?export=download&id=${fileid}' -O- | sed -rn 's/.confirm=([0-9A-Za-z_]+)./\1\n/p')&id=${fileid}" -O ${filename} && rm -rf /tmp/cookies.txt + +# Unzip +unzip -q ${filename} +rm ${filename} +``` +### 2. Modify the dataset path and related hyperparameters in `projects/NeRF/configs/config_nerf.py` + +### 3. Install additional dependencies +```bash +pip install opencv-python +pip install imageio +``` +### 4. Run the following code to start training +```bash +# cd /path/to/libai + bash tools/train.sh tools/train_net.py projects/NeRF/configs/config_nerf.py 1 +``` +Note that we use PSNR to evaluate the performance of NeRF. PSNR (Peak Signal-to-Noise Ratio) is an engineering term that represents the ratio of the maximum possible power of a signal to the destructive noise power that affects its representation accuracy. + +### 5. Visual rendering results (Please modify the value of `train.load_weight` in `projects/NeRF/configs/config_nerf_for_rendering.py` first) +```bash +# cd /path/to/libai + bash tools/train.sh tools/train_net.py projects/NeRF/configs/config_nerf_for_rendering.py 1 --eval-only +``` diff --git a/projects/PaLM/README.md b/projects/PaLM/README.md new file mode 100644 index 0000000000000000000000000000000000000000..dc56eb9297644ed5c0182359e3701b38543074f6 --- /dev/null +++ b/projects/PaLM/README.md @@ -0,0 +1,39 @@ + + +# Pathways Language Model (PaLM) based on LiBai + +Implementation of the model architecture of [Pathways Language Model (PaLM): Scaling to 540 Billion Parameters for Breakthrough Performance](https://ai.googleblog.com/2022/04/pathways-language-model-palm-scaling-to.html) with OneFlow in less than 300 lines of a code. + +We take advantage of [LiBai](https://github.com/Oneflow-Inc/libai) to exploit multiple parallelism strategies, e.g. data parallelism, tensor parallelism and pipeline parallelism. Besides, some advanced features such as mixed precision and ZeRO come here to help scale the training to multiple GPUs. + +We welcome contributions, to help us enhance the usability of this project. + +## Install + +1. To install LiBai please follow install instruction. + +2. Download the gpt dataset as demo dataset. + +```shell +cd projects/PaLM +python tools/download_demo_dataset.py -o +``` + +## Usage + +1. Configure your settings in `CONFIG_FILE.py` like below. We also provide some examples in [./configs](./configs/). You can check the advanced features tutorial and distributed tutorial. + +```python +train.amp.enabled = True +train.activation_checkpoint.enabled = True +``` + +2. Run + +```shell +# single gpu +bash tools/train.sh tools/train_net.py projects/PaLM/configs/palm_pretrain.py 1 + +# 8 gpus +bash tools/train.sh tools/train_net.py projects/PaLM/configs/palm_pretrain.py 8 +``` diff --git a/projects/PaLM/assets/palm.gif b/projects/PaLM/assets/palm.gif new file mode 100644 index 0000000000000000000000000000000000000000..ba27d6228a6b8bed8cf5c5600aec017d0c83d4c3 Binary files /dev/null and b/projects/PaLM/assets/palm.gif differ diff --git a/projects/PaLM/configs/models/palm_16b.py b/projects/PaLM/configs/models/palm_16b.py new file mode 100644 index 0000000000000000000000000000000000000000..bf603d9110e0d69eb4d31ee8c461a418a208fe3d --- /dev/null +++ b/projects/PaLM/configs/models/palm_16b.py @@ -0,0 +1,6 @@ +from .palm_small import model + +model.cfg.dim = 4096 +model.cfg.depth = 64 +model.cfg.dim_head = 256 +model.cfg.num_heads = 16 diff --git a/projects/PaLM/configs/models/palm_30b.py b/projects/PaLM/configs/models/palm_30b.py new file mode 100644 index 0000000000000000000000000000000000000000..d4de53553834488bb0d892da194d33b4a5a4aec6 --- /dev/null +++ b/projects/PaLM/configs/models/palm_30b.py @@ -0,0 +1,6 @@ +from .palm_small import model + +model.cfg.dim = 6144 +model.cfg.depth = 48 +model.cfg.dim_head = 256 +model.cfg.num_heads = 24 diff --git a/projects/PaLM/configs/models/palm_62b.py b/projects/PaLM/configs/models/palm_62b.py new file mode 100644 index 0000000000000000000000000000000000000000..eacbfdd70cc0b1600c4f618fb113a0acd21afbb1 --- /dev/null +++ b/projects/PaLM/configs/models/palm_62b.py @@ -0,0 +1,6 @@ +from .palm_small import model + +model.cfg.dim = 8192 +model.cfg.depth = 64 +model.cfg.dim_head = 256 +model.cfg.num_heads = 32 diff --git a/projects/PaLM/configs/models/palm_8b.py b/projects/PaLM/configs/models/palm_8b.py new file mode 100644 index 0000000000000000000000000000000000000000..a48035fc81ac0a220d35e5cb8a2a01801f91681c --- /dev/null +++ b/projects/PaLM/configs/models/palm_8b.py @@ -0,0 +1,6 @@ +from .palm_small import model + +model.cfg.dim = 4096 +model.cfg.depth = 32 +model.cfg.dim_head = 256 +model.cfg.num_heads = 16 diff --git a/projects/PaLM/configs/models/palm_small.py b/projects/PaLM/configs/models/palm_small.py new file mode 100644 index 0000000000000000000000000000000000000000..ed0dc1842c10993972f048b587675eabb4f3f0db --- /dev/null +++ b/projects/PaLM/configs/models/palm_small.py @@ -0,0 +1,16 @@ +from libai.config import LazyCall +from projects.PaLM.palm_model import PaLM + +palm_cfg = dict( + vocab_size=50304, + dim=768, + depth=12, + dim_head=64, + num_heads=12, + ffn_mult=4, + initializer_range=0.02, + layernorm_eps=1e-12, + amp_enabled=False, +) + +model = LazyCall(PaLM)(cfg=palm_cfg) diff --git a/projects/PaLM/configs/palm_pretrain.py b/projects/PaLM/configs/palm_pretrain.py new file mode 100644 index 0000000000000000000000000000000000000000..f722496af521cef904cb46d4978debcf2cdb7ef7 --- /dev/null +++ b/projects/PaLM/configs/palm_pretrain.py @@ -0,0 +1,28 @@ +from libai.config import LazyCall, get_config +from .models.palm_small import model +from libai.evaluation import PPLEvaluator + +graph = get_config("common/models/graph.py").graph +train = get_config("common/train.py").train +optim = get_config("common/optim.py").optim +data = get_config("common/data/gpt_dataset.py") + +dataloader = data.dataloader +tokenization = data.tokenization + + +vocab_file = "./projects/PaLM/gpt_dataset/gpt2-vocab.json" +merge_files = "./projects/PaLM/gpt_dataset/gpt2-merges.txt" +data_prefix = "./projects/PaLM/gpt_dataset/loss_compara_content_sentence" + +tokenization.tokenizer.vocab_file = vocab_file +tokenization.tokenizer.merges_file = merge_files +dataloader.train.dataset[0].data_prefix = data_prefix +dataloader.train.dataset[0].indexed_dataset.data_prefix = data_prefix + +train.train_micro_batch_size = 4 +train.activation_checkpoint.enabled = True + +train.evaluation.evaluator = LazyCall(PPLEvaluator)() + +train.output_dir = "./output/palm_output" diff --git a/projects/PaLM/palm_model.py b/projects/PaLM/palm_model.py new file mode 100644 index 0000000000000000000000000000000000000000..8385b9ecd4fbb92c388c9673e90292e4d4446478 --- /dev/null +++ b/projects/PaLM/palm_model.py @@ -0,0 +1,323 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import oneflow as flow +import oneflow.nn.functional as F +from oneflow import einsum, nn + +from libai.config import configurable +from libai.layers import LayerNorm, Linear, LMLogits, ParallelCrossEntropyLoss, VocabEmbedding +from libai.models.utils import init_method_normal +from libai.utils import distributed as dist + +# rotary positional embedding +# https://arxiv.org/abs/2104.09864 + + +class RotaryEmbedding(nn.Module): + def __init__(self, dim, *, layer_idx=0): + super().__init__() + inv_freq = flow.tensor( + 1.0 / (10000 ** (np.arange(0, dim, 2, dtype=np.float32) / dim)), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(layer_idx), + ) + self.register_buffer("inv_freq", inv_freq) + + def forward(self, max_seq_len): + seq = flow.arange( + max_seq_len, + dtype=self.inv_freq.dtype, + sbp=self.inv_freq.sbp, + placement=self.inv_freq.placement, + ) + freqs = einsum("i , j -> i j", seq, self.inv_freq) + return flow.cat((freqs, freqs), dim=-1) + + +def rotate_half(x): + # x = rearrange(x, "... (j d) -> ... j d", j=2) + x = x.reshape(*list(x.shape[:-1]), 2, -1) + x1 = x[..., 0, :] + x2 = x[..., 1, :] + return flow.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb(pos, t): + return (t * pos.cos()) + (rotate_half(t) * pos.sin()) + + +# feedforward +# classic Noam Shazeer paper, except here they use SwiGLU instead of the more popular GEGLU +# https://arxiv.org/abs/2002.05202 + + +class SwiGLU(nn.Module): + def forward(self, x): + x, gate = x.chunk(2, dim=-1) + return F.silu(gate) * x + + +def FeedForward(dim, mult=4, *, layer_idx=0): + inner_dim = int(dim * mult) + return nn.Sequential( + LayerNorm(dim, bias=False, layer_idx=layer_idx), + Linear(dim, inner_dim * 2, bias=False, parallel="col", layer_idx=layer_idx), + SwiGLU(), + Linear(inner_dim, dim, bias=False, parallel="row", layer_idx=layer_idx), + ) + + +class PalmTransformerLayer(nn.Module): + def __init__( + self, + dim: int, + dim_head: int = 64, + num_heads: int = 8, + ffn_mult: int = 4, + layernorm_epsilon: float = 1e-5, + *, + layer_idx=0 + ): + """PaLM transformer block with hybrid parallelism""" + + super().__init__() + self.num_heads = num_heads + self.dim_head = dim_head + inner_dim = dim_head * num_heads + self.attn_inner_dim = num_heads * dim_head + self.ffn_inner_dim = int(ffn_mult * dim) + self.ffn_mult = ffn_mult + self.layer_idx = layer_idx + + # only query has multi head + # key and value remain as single head + self.to_q = Linear(dim, inner_dim, bias=False, parallel="col", layer_idx=layer_idx) + self.to_kv = Linear(dim, dim_head * 2, bias=False, parallel="col", layer_idx=layer_idx) + + self.to_out = Linear(inner_dim, dim, bias=False, parallel="row", layer_idx=layer_idx) + + self.rotary_emb = RotaryEmbedding(self.dim_head, layer_idx=layer_idx) + + self.ffwd = FeedForward(dim, ffn_mult, layer_idx=layer_idx) + self.norm = LayerNorm(dim, eps=layernorm_epsilon, bias=False, layer_idx=layer_idx) + self.scale = dim_head ** -0.5 + + def get_mask(self, seq): + if hasattr(self, "mask") and self.mask.shape[-1] >= seq: + return self.mask[:seq, :seq] + + mask = ( + 1 + - flow.ones( + (seq, seq), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(self.layer_idx), + dtype=flow.int8, + ).triu(1) + ) + self.register_buffer("mask", mask, persistent=False) + return mask + + def get_rotary_embedding(self, seq): + if hasattr(self, "pos_emb") and self.pos_emb.shape[-2] >= seq: + return self.pos_emb[:seq] + + pos_emb = self.rotary_emb(seq) + self.register_buffer("pos_emb", pos_emb, persistent=False) + return pos_emb + + def forward(self, x): + # move x to the stage with right placement + x = x.to_global(placement=dist.get_layer_placement(self.layer_idx)) + + bsz, seq_length = x.size()[0:2] + + # pre-layernorm + layernorm_output = self.norm(x) + + # fused input linear layer + query = self.to_q(layernorm_output) + query = query.view(bsz, -1, self.num_heads, self.dim_head) + query = query.permute(0, 2, 1, 3) + + key_value = self.to_kv(layernorm_output) + key, value = flow.chunk(key_value, chunks=2, dim=-1) + + # apply position embedding + positions = self.get_rotary_embedding(seq_length) + query, key = map(lambda t: apply_rotary_pos_emb(positions, t), (query, key)) + + # apply scale + query = query * self.scale + + # calculate similarity + attention_scores = einsum("b h s d, b j d -> b h s j", query, key) + + # apply casual mask + attention_mask = self.get_mask(seq_length) + + attention_scores = flow.mul(attention_scores, attention_mask) + attention_scores = attention_scores - 10000.0 * (1 - attention_mask) + + attention_weights = flow.softmax(attention_scores, dim=-1) + + # aggregate values + attn_out = einsum("b h i j, b j d -> b h i d", attention_weights, value) + + # merge heads + attn_out = attn_out.transpose(1, 2) + attn_out = attn_out.view(bsz, seq_length, -1) + # attn_out = rearrange(attn_out, "b h s d -> b s (h d)") + + attn_out = self.to_out(attn_out) + + # feedforward + out = self.ffwd(x) + attn_out + return out + x + + +class PalmHead(nn.Module): + def __init__(self, vocab_size, word_embedding_weight): + super().__init__() + self.lm_head = LMLogits(vocab_size, bias=False) + self.loss_func = ParallelCrossEntropyLoss() + + self.word_embedding_weight = word_embedding_weight + + def forward(self, x, lm_labels): + logits = self.lm_head(x, self.word_embedding_weight) + if lm_labels is not None: + lm_loss = self.loss_func(logits, lm_labels) + lm_loss = lm_loss.mean() + return {"lm_loss": lm_loss} + else: + return {"prediction_scores": logits} + + +class PaLM(nn.Module): + @configurable + def __init__( + self, + vocab_size, + dim, + depth, + dim_head=64, + num_heads=8, + ffn_mult=4, + initializer_range=0.02, + layernorm_eps=1e-12, + amp_enabled=False, + ): + super().__init__() + init_method = init_method_normal(initializer_range) + + word_embedding = VocabEmbedding( + vocab_size, + dim, + init_method=init_method, + amp_enabled=amp_enabled, + ) + + self.net = nn.Sequential( + word_embedding, + *[ + PalmTransformerLayer( + dim, + dim_head=dim_head, + num_heads=num_heads, + ffn_mult=ffn_mult, + layernorm_epsilon=layernorm_eps, + layer_idx=i, + ) + for i in range(depth) + ], + LayerNorm(dim, bias=False, eps=layernorm_eps, layer_idx=-1), + ) + + self.head = PalmHead(vocab_size, word_embedding.weight) + + def forward(self, input_ids, labels=None): + output = self.net(input_ids) + return self.head(output, labels) + + @classmethod + def from_config(cls, cfg): + return { + "vocab_size": cfg.vocab_size, + "dim": cfg.dim, + "depth": cfg.depth, + "dim_head": cfg.dim_head, + "num_heads": cfg.num_heads, + "ffn_mult": cfg.ffn_mult, + "initializer_range": cfg.initializer_range, + "layernorm_eps": cfg.layernorm_eps, + "amp_enabled": cfg.amp_enabled, + } + + @staticmethod + def set_activation_checkpoint(model): + for module_block in model.modules(): + if hasattr(module_block, "origin"): + if isinstance(module_block.origin, PalmTransformerLayer): + module_block.config.activation_checkpointing = True + else: + if isinstance(module_block.to(nn.Module), PalmTransformerLayer): + module_block.to(flow.nn.graph.GraphModule).activation_checkpointing = True + + @staticmethod + def set_pipeline_stage_id(model: nn.Module): + dist_utils = dist.get_dist_util() + + if hasattr(model.net[-1], "config"): + # Old API in OneFlow 0.8 + for module_block in model.modules(): + if isinstance(module_block.origin, VocabEmbedding): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, PalmTransformerLayer): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.origin, PalmHead): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + # final layernorm + model.net[-1].config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + else: + for module_block in model.modules(): + if isinstance(module_block.to(nn.Module), VocabEmbedding): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), PalmTransformerLayer): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.to(nn.Module), PalmHead): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + # final layernorm + model.net[-1].to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) diff --git a/projects/PaLM/tools/download_demo_dataset.py b/projects/PaLM/tools/download_demo_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..616ca18357732af5e31bd6a87d0290d496b48a8d --- /dev/null +++ b/projects/PaLM/tools/download_demo_dataset.py @@ -0,0 +1,31 @@ +import argparse + +from libai.utils.file_utils import get_data_from_cache + +VOCAB_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/gpt_dataset/gpt2-vocab.json" # noqa +MERGE_FILE_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/gpt_dataset/gpt2-merges.txt" # noqa +BIN_DATA_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/loss_compara_content_sentence.bin" # noqa +IDX_DATA_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/loss_compara_content_sentence.idx" # noqa + +VOCAB_MD5 = "dffec25a898b1f5e569bec4dffd7e5c0" +MERGE_FILE_MD5 = "75a37753dd7a28a2c5df80c28bf06e4e" +BIN_DATA_MD5 = "b842467bd5ea7e52f7a612ea6b4faecc" +IDX_DATA_MD5 = "cf5963b8543f0a7a867361eb980f0372" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "-o", "--output", default="./gpt_dataset", type=str, help="The output path to store data" + ) + args = parser.parse_args() + cache_dir = args.output + + get_data_from_cache(VOCAB_URL, cache_dir, md5=VOCAB_MD5) + get_data_from_cache(MERGE_FILE_URL, cache_dir, md5=MERGE_FILE_MD5) + get_data_from_cache(BIN_DATA_URL, cache_dir, md5=BIN_DATA_MD5) + get_data_from_cache(IDX_DATA_URL, cache_dir, md5=IDX_DATA_MD5) + + +if __name__ == "__main__": + main() diff --git a/projects/QQP/README.md b/projects/QQP/README.md new file mode 100644 index 0000000000000000000000000000000000000000..00fe7f6d06c661621e0874e573c23322dac19937 --- /dev/null +++ b/projects/QQP/README.md @@ -0,0 +1,34 @@ +# QQP projects + +QQP is a fine-tuning task for sequence classification, it belongs to GLUE: [General Language Understanding Evaluation](https://gluebenchmark.com/). + +We provide code for finetuning on qqp dataset based on pretrained [Bert](https://arxiv.org/pdf/1810.04805.pdf) model. Differently, the language model is fine-tuned and pretrained from **Chinese** vocabulary. + +## Download Data + +Run command: +```bash +cd /path/to/libai_root +python3 projects/QQP/dataset/download_qqp_data.py +``` +Data and Vocabulary will be downloaded in `projects/QQP/QQP_DATA/` +> NOTE: pretrained model will not be provided due to privacy reasons, you can download pretrained bert model from [Megatron](https://github.com/NVIDIA/Megatron-LM) +``` +projects/QQP/QQP_DATA/ +│---bert-base-chinese-vocab.txt +│---train.tsv +│---dev.tsv +``` + +## Start Training +Run command: +```shell +cd /path/to/libai_root +bash tools/train.sh tools/train_net.py projects/QQP/configs/config_qqp.py $num_gpus +``` +where `$num_gpus` indicates the number of GPUs. For example, if you want to run the distributed program on 4 GPUs, you can set to 4.: +```bash +bash tools/train.sh tools/train_net.py projects/text_classification/configs/config.py 4 +``` + +Before running the program, you should modify the `projects/QQP/configs/config_qqp.py` depend on your own needs. Modification fields include but are not limited to task_name, data_dir, vocab_file, model hyperparameter, learning rate, and batch size, and so on. \ No newline at end of file diff --git a/projects/QQP/configs/config_qqp.py b/projects/QQP/configs/config_qqp.py new file mode 100644 index 0000000000000000000000000000000000000000..6daec049a10457470e3be1913506629240f2d8b7 --- /dev/null +++ b/projects/QQP/configs/config_qqp.py @@ -0,0 +1,83 @@ +from omegaconf import OmegaConf + +from configs.common.data.bert_dataset import tokenization +from configs.common.models.bert import cfg as qqp_cfg +from configs.common.optim import optim +from configs.common.train import train +from configs.common.models.graph import graph +from libai.config import LazyCall +from libai.data.build import build_nlp_test_loader, build_nlp_train_loader +from projects.QQP.dataset.qqp_dataset import QQPDataset +from projects.QQP.modeling.model import Classification +from projects.QQP.tokenizer.tokenizer import _BertCNWWMTokenizer + +tokenization.tokenizer = LazyCall(_BertCNWWMTokenizer)( + vocab_file="projects/QQP/QQP_DATA/bert-base-chinese-vocab.txt", + lower_case=True, +) +tokenization.append_eod = False +tokenization.make_vocab_size_divisible_by = 128 + +dataloader = OmegaConf.create() +dataloader.train = LazyCall(build_nlp_train_loader)( + dataset=[ + LazyCall(QQPDataset)( + dataset_name="QQP_TRAIN", + data_paths=[ + "projects/QQP/QQP_DATA/train.tsv", + ], + max_seq_length=512, + ), + ], + num_workers=4, +) +dataloader.test = [ + LazyCall(build_nlp_test_loader)( + dataset=LazyCall(QQPDataset)( + dataset_name="QQP_TEST", + data_paths=[ + "projects/QQP/QQP_DATA/dev.tsv", + ], + max_seq_length=512, + ), + num_workers=4, + ), +] + +qqp_cfg.update( + dict( + # exist key + vocab_size=21248, + hidden_size=1024, + hidden_layers=24, + num_attention_heads=16, + # new key + num_classes=2, + pretrain_megatron_weight=None, # "path/to/model_optim_rng.pt", + ) +) +model = LazyCall(Classification)(cfg=qqp_cfg) + +optim.lr = 1e-6 +optim.weight_decay = 0.1 + +train.update( + dict( + activation_checkpoint=dict(enabled=True), + amp=dict(enabled=True), + output_dir="output/finetune_qqp/", + train_micro_batch_size=16, + test_micro_batch_size=4, + train_epoch=1, + train_iter=0, + eval_period=100, + log_period=10, + warmup_ratio=0.01, + topk=(1,), + dist=dict( + data_parallel_size=1, + tensor_parallel_size=1, + pipeline_parallel_size=1, + ), + ) +) diff --git a/projects/QQP/dataset/data.py b/projects/QQP/dataset/data.py new file mode 100644 index 0000000000000000000000000000000000000000..0b7ed9ace520a86c59b4eeee759b3dbad266225d --- /dev/null +++ b/projects/QQP/dataset/data.py @@ -0,0 +1,62 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from abc import ABC, abstractmethod + +from oneflow.utils.data import Dataset + +from .data_utils import build_sample, build_tokens_types_paddings_from_text + +logger = logging.getLogger("libai." + __name__) + + +class GLUEAbstractDataset(ABC, Dataset): + """GLUE base dataset class.""" + + def __init__(self, task_name, dataset_name, datapaths, tokenizer, max_seq_length): + # Store inputs. + self.task_name = task_name + self.dataset_name = dataset_name + self.tokenizer = tokenizer + self.max_seq_length = max_seq_length + logger.info(" > building {} dataset for {}:".format(self.task_name, self.dataset_name)) + # Process the files. + string = " > paths:" + for path in datapaths: + string += " " + path + logger.info(string) + self.samples = [] + for datapath in datapaths: + self.samples.extend(self.process_samples_from_single_path(datapath)) + logger.info(" >> total number of samples: {}".format(len(self.samples))) + + def __len__(self): + return len(self.samples) + + def __getitem__(self, idx): + raw_sample = self.samples[idx] + ids, types, paddings = build_tokens_types_paddings_from_text( + raw_sample["text_a"], raw_sample["text_b"], self.tokenizer, self.max_seq_length + ) + sample = build_sample(ids, types, paddings, raw_sample["label"], raw_sample["uid"]) + return sample + + @abstractmethod + def process_samples_from_single_path(self, datapath): + """Abstract method that takes a single path / filename and + returns a list of dataset samples, each sample being a dict of + {'text_a': string, 'text_b': string, 'label': int, 'uid': int} + """ diff --git a/projects/QQP/dataset/data_utils.py b/projects/QQP/dataset/data_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8d592facbbaeecc5c30ed4c7458cb0c4257896b7 --- /dev/null +++ b/projects/QQP/dataset/data_utils.py @@ -0,0 +1,121 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import numpy as np +import oneflow as flow + +from libai.data.structures import DistTensorData, Instance + + +def clean_text(text): + """Remove new lines and multiple spaces and adjust end of sentence dot.""" + + text = text.replace("\n", " ") + text = re.sub(r"\s+", " ", text) + for _ in range(3): + text = text.replace(" . ", ". ") + + return text + + +def build_sample(ids, types, paddings, label, unique_id): + """Convert to numpy and return a sample consumed by the batch producer.""" + + ids_np = np.array(ids, dtype=np.int64) + types_np = np.array(types, dtype=np.int64) + paddings_np = np.array(paddings, dtype=np.int64) + sample = Instance( + model_input=DistTensorData(flow.tensor(ids_np, dtype=flow.long), placement_idx=0), + attention_mask=DistTensorData(flow.tensor(paddings_np, dtype=flow.long), placement_idx=0), + tokentype_ids=DistTensorData(flow.tensor(types_np, dtype=flow.long), placement_idx=0), + labels=DistTensorData(flow.tensor(label, dtype=flow.long), placement_idx=-1), + ) + + return sample + + +def build_tokens_types_paddings_from_text(text_a, text_b, tokenizer, max_seq_length): + """Build token types and paddings, trim if needed, and pad if needed.""" + + text_a_ids = tokenizer.tokenize(text_a) + text_b_ids = None + if text_b is not None: + text_b_ids = tokenizer.tokenize(text_b) + + return build_tokens_types_paddings_from_ids( + text_a_ids, text_b_ids, max_seq_length, tokenizer.cls, tokenizer.sep, tokenizer.pad + ) + + +def build_tokens_types_paddings_from_ids( + text_a_ids, text_b_ids, max_seq_length, cls_id, sep_id, pad_id +): + """Build token types and paddings, trim if needed, and pad if needed.""" + + ids = [] + types = [] + paddings = [] + + # [CLS]. + ids.append(cls_id) + types.append(0) + paddings.append(1) + + # A. + len_text_a = len(text_a_ids) + ids.extend(text_a_ids) + types.extend([0] * len_text_a) + paddings.extend([1] * len_text_a) + + # [SEP]. + ids.append(sep_id) + types.append(0) + paddings.append(1) + + # B. + if text_b_ids is not None: + len_text_b = len(text_b_ids) + ids.extend(text_b_ids) + types.extend([1] * len_text_b) + paddings.extend([1] * len_text_b) + + # Cap the size. + trimmed = False + if len(ids) >= max_seq_length: + max_seq_length_m1 = max_seq_length - 1 + ids = ids[0:max_seq_length_m1] + types = types[0:max_seq_length_m1] + paddings = paddings[0:max_seq_length_m1] + trimmed = True + + # [SEP]. + if (text_b_ids is not None) or trimmed: + ids.append(sep_id) + if text_b_ids is None: + types.append(0) + else: + types.append(1) + paddings.append(1) + + # Padding. + padding_length = max_seq_length - len(ids) + if padding_length > 0: + ids.extend([pad_id] * padding_length) + types.extend([pad_id] * padding_length) + paddings.extend([0] * padding_length) + + return ids, types, paddings diff --git a/projects/QQP/dataset/download_qqp_data.py b/projects/QQP/dataset/download_qqp_data.py new file mode 100644 index 0000000000000000000000000000000000000000..79e709799885741544178aed73b33f05d8c69d04 --- /dev/null +++ b/projects/QQP/dataset/download_qqp_data.py @@ -0,0 +1,41 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +sys.path.append(".") +from libai.utils.file_utils import get_data_from_cache # noqa + +# fmt:off +VOCAB_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/bert-base-chinese-vocab.txt" # noqa +QQP_TRAIN_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/QQP/train.tsv" # noqa +QQP_TEST_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/QQP/dev.tsv" # noqa +# fmt:on + + +VOCAB_MD5 = "3b5b76c4aef48ecf8cb3abaafe960f09" +QQP_TRAIN_MD5 = "f65950abb9499d8e3e33da7d68d61c4e" +QQP_TEST_MD5 = "35ca3d547003266660a77c6031069548" + +cache_dir = "projects/QQP/QQP_DATA/" + +if __name__ == "__main__": + print("downloading vocab...") + get_data_from_cache(VOCAB_URL, cache_dir, md5=VOCAB_MD5) + print("downloading training data...") + get_data_from_cache(QQP_TRAIN_URL, cache_dir, md5=QQP_TRAIN_MD5) + print("downloading testing data...") + get_data_from_cache(QQP_TEST_URL, cache_dir, md5=QQP_TEST_MD5) + print("downloading complete") diff --git a/projects/QQP/dataset/qqp_dataset.py b/projects/QQP/dataset/qqp_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..f946121c528a8aaf64f0e3e5c35b19ae36053aae --- /dev/null +++ b/projects/QQP/dataset/qqp_dataset.py @@ -0,0 +1,97 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from .data import GLUEAbstractDataset +from .data_utils import clean_text + +logger = logging.getLogger("libai." + __name__) + +LABELS = [0, 1] + + +class QQPDataset(GLUEAbstractDataset): + def __init__(self, dataset_name, data_paths, tokenizer, max_seq_length, test_label=0): + self.test_label = test_label + self.dataset_name = dataset_name + super().__init__("QQP", dataset_name, data_paths, tokenizer, max_seq_length) + + def process_samples_from_single_path(self, filename): + """ "Implement abstract method.""" + logger.info(" > Processing {} ...".format(filename)) + + samples = [] + total = 0 + first = True + is_test = False + with open(filename, "r") as f: + for line in f: + row = line.strip().split("\t") + if first: + first = False + if len(row) == 3: + is_test = True + logger.info( + " reading {}, {}, and {} columns and " + "setting labels to {}".format( + row[0].strip(), row[1].strip(), row[2].strip(), self.test_label + ) + ) + else: + assert len(row) == 6 + logger.info( + " reading {}, {}, {}, and {} columns" + " ...".format( + row[0].strip(), row[3].strip(), row[4].strip(), row[5].strip() + ) + ) + continue + + if is_test: + assert len(row) == 3, "expected length 3: {}".format(row) + uid = int(row[0].strip()) + text_a = clean_text(row[1].strip()) + text_b = clean_text(row[2].strip()) + label = self.test_label + assert len(text_a) > 0 + assert len(text_b) > 0 + else: + if len(row) == 6: + uid = int(row[0].strip()) + text_a = clean_text(row[3].strip()) + text_b = clean_text(row[4].strip()) + label = int(row[5].strip()) + else: + logger.info("***WARNING*** index error, " "skipping: {}".format(row)) + continue + if len(text_a) == 0: + logger.info("***WARNING*** zero length a, " "skipping: {}".format(row)) + continue + if len(text_b) == 0: + logger.info("***WARNING*** zero length b, " "skipping: {}".format(row)) + continue + assert label in LABELS + assert uid >= 0 + + sample = {"uid": uid, "text_a": text_a, "text_b": text_b, "label": label} + total += 1 + samples.append(sample) + + if total % 50000 == 0: + logger.info(" > processed {} so far ...".format(total)) + + logger.info(" >> processed {} samples.".format(len(samples))) + return samples diff --git a/projects/QQP/modeling/load_megatron_weight.py b/projects/QQP/modeling/load_megatron_weight.py new file mode 100644 index 0000000000000000000000000000000000000000..6abbab24e4e1dde9be8b943580675eddd2f0a182 --- /dev/null +++ b/projects/QQP/modeling/load_megatron_weight.py @@ -0,0 +1,134 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import oneflow as flow +import torch + +import libai.utils.distributed as dist +from libai.utils.checkpoint import get_missing_parameters_message, get_unexpected_parameters_message + +logger = logging.getLogger("libai." + __name__) + + +def convert_tensor(tensor: torch.Tensor): + tensor = tensor.float() + return flow.Tensor(tensor.cpu().numpy()) + + +def change_megatron_key(state_dict): + of_state_dict = {} + + # Language model. + language_model = state_dict["language_model"] + + # Embedding. + embedding = language_model["embedding"] + of_state_dict["embeddings.vocab_embeddings.weight"] = convert_tensor( + embedding["word_embeddings"]["weight"] + ) + of_state_dict["embeddings.position_embeddings.weight"] = convert_tensor( + embedding["position_embeddings"]["weight"] + ) + of_state_dict["embeddings.tokentype_embeddings.weight"] = convert_tensor( + embedding["tokentype_embeddings"]["weight"] + ) + + # Encoder. + encoder = language_model["encoder"] + for key, value in encoder.items(): + # Change layers.0.input_layernorm.weight -> encoder.layers_0.input_layernorm.weight + key = "encoders." + key.replace("layers.", "") + if key.startswith("encoders.final_layernorm"): + key = key.replace("encoders.", "") + of_state_dict[key] = convert_tensor(value) + + # Pooler. + pooler = language_model["pooler"] + of_state_dict["pooler.dense.weight"] = convert_tensor(pooler["dense.weight"]) + of_state_dict["pooler.dense.bias"] = convert_tensor(pooler["dense.bias"]) + + # LM head. + lm_head = state_dict["lm_head"] + of_state_dict["cls.predictions.dense.weight"] = convert_tensor(lm_head["dense.weight"]) + of_state_dict["cls.predictions.dense.bias"] = convert_tensor(lm_head["dense.bias"]) + + of_state_dict["cls.predictions.layernorm.weight"] = convert_tensor(lm_head["layernorm.weight"]) + of_state_dict["cls.predictions.layernorm.bias"] = convert_tensor(lm_head["layernorm.bias"]) + + of_state_dict["lm_logits.bias"] = convert_tensor(lm_head["bias"]) + + # Binary head. + binary_head = state_dict["binary_head"] + of_state_dict["cls.seq_relationship.weight"] = convert_tensor(binary_head["weight"]) + of_state_dict["cls.seq_relationship.bias"] = convert_tensor((binary_head["bias"])) + + return of_state_dict + + +def load_tensor(tensor_lhs, tensor_rhs): + tensor_rhs = flow.to_global( + tensor_rhs, + placement=tensor_lhs.placement, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + tensor_rhs = tensor_rhs.to_global(sbp=tensor_lhs.sbp) + tensor_lhs.copy_(tensor_rhs) + + +def load_model(model: flow.nn.Module, state_dict): + model_state_dict = model.state_dict() + + # Decide shape + incorrect_shapes = [] + for k in list(state_dict.keys()): + if k in model_state_dict: + shape_model = tuple(model_state_dict[k].shape) + shape_ckpt = tuple(state_dict[k].shape) + if shape_model != shape_ckpt: + incorrect_shapes.append((k, shape_ckpt, shape_model)) + state_dict.pop(k) + + unexpected_keys = [] + for key, value in state_dict.items(): + if key not in model_state_dict: + unexpected_keys.append(key) + continue + model_state_dict.pop(key) + load_tensor(model.state_dict()[key], value) + + missing_keys = list(model_state_dict.keys()) + + for k, shape_checkpoint, shape_model in incorrect_shapes: + logger.warning( + "Skip loading parameter '{}' to the model due to incompatible " + "shapes: {} in the checkpoint but {} in the " + "model! You might want to double check if this is expected.".format( + k, shape_checkpoint, shape_model + ) + ) + if missing_keys: + logger.info(get_missing_parameters_message(missing_keys)) + if unexpected_keys: + logger.info(get_unexpected_parameters_message(unexpected_keys)) + + +def load_megatron_bert(model: flow.nn.Module, model_weight_path: str): + import torch + + megatron_state_dict = torch.load(model_weight_path, map_location="cpu")["model"] + of_state_dict = change_megatron_key(megatron_state_dict) + load_model(model, of_state_dict) diff --git a/projects/QQP/modeling/model.py b/projects/QQP/modeling/model.py new file mode 100644 index 0000000000000000000000000000000000000000..6661dbd3df0b9c56ce0e0110538155c2b97da388 --- /dev/null +++ b/projects/QQP/modeling/model.py @@ -0,0 +1,80 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import oneflow as flow +from oneflow import nn + +from libai.layers import Linear +from libai.models.bert_model import BertModel +from libai.models.utils import init_method_normal +from libai.utils import distributed as dist + +logger = logging.getLogger("libai." + __name__) + + +class ClassificationLoss(nn.Module): + def __init__(self): + super().__init__() + + def forward(self, classification_logits, label): + loss = nn.CrossEntropyLoss()(classification_logits, label) + # NOTE: Change loss sbp sign [P, P] -> [P, B] to add with sop loss + # whose sbp sign: [P, B] + loss = loss.to_global(sbp=dist.get_nd_sbp([flow.sbp.partial_sum, flow.sbp.broadcast])) + return loss + + +class Classification(nn.Module): + def __init__(self, cfg): + super().__init__() + self.num_classes = cfg.num_classes + self.language_model = BertModel(cfg) + if cfg.pretrain_megatron_weight is not None: + from .load_megatron_weight import load_megatron_bert + + logger.info(f"loading pretraining: {cfg.pretrain_megatron_weight}") + load_megatron_bert(self.language_model, cfg.pretrain_megatron_weight) + logger.info("load succeed") + + init_method = init_method_normal(cfg.initializer_range) + self.classification_dropout = nn.Dropout(cfg.hidden_dropout_prob) + self.classification_head = Linear( + cfg.hidden_size, + self.num_classes, + bias=True, + parallel="row", + init_method=init_method, + layer_idx=-1, + ) + self.loss_func = ClassificationLoss() + + def forward(self, model_input, attention_mask, tokentype_ids=None, labels=None): + + encoder_output, pooled_output = self.language_model( + model_input, attention_mask, tokentype_ids + ) + classification_output = self.classification_dropout(pooled_output) + classification_logits = self.classification_head(classification_output) + + # reshape + classification_logits = classification_logits.view(-1, self.num_classes) + + if self.training and labels is not None: + loss = self.loss_func(classification_logits, labels) + return {"total_loss": loss} + + return {"prediction_scores": classification_logits} diff --git a/projects/QQP/tokenizer/bert_tokenization.py b/projects/QQP/tokenizer/bert_tokenization.py new file mode 100644 index 0000000000000000000000000000000000000000..75d479b2e671da56a4ab026019d9566aaadd7f37 --- /dev/null +++ b/projects/QQP/tokenizer/bert_tokenization.py @@ -0,0 +1,623 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tokenization classes.""" + + +import collections +import re +import sys +import unicodedata + +import six + +is_python3 = sys.version_info.major == 3 +if is_python3: + unicode = str + + +def validate_case_matches_checkpoint(do_lower_case, init_checkpoint): + """Checks whether the casing config is consistent with the checkpoint name.""" + + # The casing has to be passed in by the user and there is no explicit check + # as to whether it matches the checkpoint. The casing information probably + # should have been stored in the bert_config.json file, but it's not, so + # we have to heuristically detect it to validate. + + if not init_checkpoint: + return + + m = re.match("^.*?([A-Za-z0-9_-]+)/bert_model.ckpt", init_checkpoint) + if m is None: + return + + model_name = m.group(1) + + lower_models = [ + "uncased_L-24_H-1024_A-16", + "uncased_L-12_H-768_A-12", + "multilingual_L-12_H-768_A-12", + "chinese_L-12_H-768_A-12", + ] + + cased_models = [ + "cased_L-12_H-768_A-12", + "cased_L-24_H-1024_A-16", + "multi_cased_L-12_H-768_A-12", + ] + + is_bad_config = False + if model_name in lower_models and not do_lower_case: + is_bad_config = True + actual_flag = "False" + case_name = "lowercased" + opposite_flag = "True" + + if model_name in cased_models and do_lower_case: + is_bad_config = True + actual_flag = "True" + case_name = "cased" + opposite_flag = "False" + + if is_bad_config: + raise ValueError( + "You passed in `--do_lower_case=%s` with `--init_checkpoint=%s`. " + "However, `%s` seems to be a %s model, so you " + "should pass in `--do_lower_case=%s` so that the fine-tuning matches " + "how the model was pre-training. If this error is wrong, please " + "just comment out this check." + % (actual_flag, init_checkpoint, model_name, case_name, opposite_flag) + ) + + +def convert_to_unicode(text): + """Converts `text` to Unicode (if it's not already), assuming utf-8 input.""" + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text.decode("utf-8", "ignore") + elif isinstance(text, unicode): + return text + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def printable_text(text): + """Returns text encoded in a way suitable for print or `tf.logging`.""" + + # These functions want `str` for both Python2 and Python3, but in one case + # it's a Unicode string and in the other it's a byte string. + if six.PY3: + if isinstance(text, str): + return text + elif isinstance(text, bytes): + return text.decode("utf-8", "ignore") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + elif six.PY2: + if isinstance(text, str): + return text + elif isinstance(text, unicode): + return text.encode("utf-8") + else: + raise ValueError("Unsupported string type: %s" % (type(text))) + else: + raise ValueError("Not running on Python2 or Python 3?") + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + index = 0 + with open(vocab_file, "r", encoding="utf-8") as reader: + while True: + token = convert_to_unicode(reader.readline()) + if not token: + break + token = token.strip() + vocab[token] = index + index += 1 + return vocab + + +def convert_by_vocab(vocab, items, inv=False): + """Converts a sequence of [tokens|ids] using the vocab.。""" + output = [] + for item in items: + if not inv: + if _is_chinese_substr(item): + output.append(vocab[item[2:]] + len(vocab)) + else: + output.append(vocab[item]) + else: + if item > len(vocab): + output.append("##" + vocab[item - len(vocab)]) + else: + output.append(vocab[item]) + return output + + +def convert_tokens_to_ids(vocab, tokens): + return convert_by_vocab(vocab, tokens) + + +def convert_ids_to_tokens(inv_vocab, ids): + return convert_by_vocab(inv_vocab, ids, inv=True) + + +def whitespace_tokenize(text): + """Runs basic whitespace cleaning and splitting on a piece of text.""" + text = text.strip() + if not text: + return [] + tokens = text.split() + return tokens + + +class FullTokenizer(object): + """Runs end-to-end tokenziation.""" + + def __init__(self, vocab_file, do_lower_case=True, do_chinese_wwm=False): + self.vocab = load_vocab(vocab_file) # _BertTokenizer会增加[BOS]和[EOS] + self.inv_vocab = {v: k for k, v in self.vocab.items()} + if not do_chinese_wwm: + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + else: + self.basic_tokenizer = BasicTokenizerWithChineseWWM(do_lower_case) + self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) + + def tokenize(self, text): + split_tokens = [] + for token in self.basic_tokenizer.tokenize(text): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + + return split_tokens + + def convert_tokens_to_ids(self, tokens): + return convert_by_vocab(self.vocab, tokens) + + def convert_ids_to_tokens(self, ids): + return convert_by_vocab(self.inv_vocab, ids, inv=True) + + @staticmethod + def convert_tokens_to_string(tokens, clean_up_tokenization_spaces=True): + """Converts a sequence of tokens (string) to a single string.""" + + def clean_up_tokenization(out_string): + """Clean up a list of simple English tokenization artifacts + like spaces before punctuations and abreviated forms. + """ + out_string = ( + out_string.replace(" .", ".") + .replace(" ?", "?") + .replace(" !", "!") + .replace(" ,", ",") + .replace(" ' ", "'") + .replace(" n't", "n't") + .replace(" 'm", "'m") + .replace(" 's", "'s") + .replace(" 've", "'ve") + .replace(" 're", "'re") + ) + return out_string + + text = " ".join(tokens).replace(" ##", "").strip() + if clean_up_tokenization_spaces: + clean_text = clean_up_tokenization(text) + return clean_text + else: + return text + + def vocab_size(self): + return len(self.vocab) + + +class BasicTokenizer(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" + + def __init__(self, do_lower_case=True): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + """ + self.do_lower_case = do_lower_case + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = convert_to_unicode(text) + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character.""" + output = [] + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + output.append(" ") + output.append(char) + output.append(" ") + else: + output.append(char) + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ( + (cp >= 0x4E00 and cp <= 0x9FFF) + or (cp >= 0x3400 and cp <= 0x4DBF) # + or (cp >= 0x20000 and cp <= 0x2A6DF) # + or (cp >= 0x2A700 and cp <= 0x2B73F) # + or (cp >= 0x2B740 and cp <= 0x2B81F) # + or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0xF900 and cp <= 0xFAFF) + or (cp >= 0x2F800 and cp <= 0x2FA1F) # + ): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xFFFD or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +class WordpieceTokenizer(object): + """Runs WordPiece tokenziation.""" + + def __init__(self, vocab, unk_token="[UNK]", max_input_chars_per_word=200): + self.vocab = vocab + self.unk_token = unk_token + self.max_input_chars_per_word = max_input_chars_per_word + + def tokenize(self, text): + """Tokenizes a piece of text into its word pieces. + + This uses a greedy longest-match-first algorithm to perform tokenization + using the given vocabulary. + + For example: + input = "unaffable" + output = ["un", "##aff", "##able"] + input = "有没有" + output = ["有", "##没", "##有"] + + （NickPan）对中文的特殊处理： + 中文的substr和原字不应该用不同的embedding，因此"有"和"##有"应该要用同样的id， + 考虑到有些中文词表中本身带有中文substr，有些则没有（如bert4kreas），为了兼容 + 两种情况，这里统一的处理方式是将中文substr的id设置为vocab_size+id(substr.remove(##))， + 然后在构建样本mask的时候再考虑。因此在wordpiece_tokenize时，不需要考虑"##有"是否在词表中， + 只需要考虑"有"在词表中即可。 + + Args: + text: A single token or whitespace separated tokens. This should have + already been passed through `BasicTokenizer. + + Returns: + A list of wordpiece tokens. + """ + + text = convert_to_unicode(text) + + output_tokens = [] + for token in whitespace_tokenize(text): + chars = list(token) + if len(chars) > self.max_input_chars_per_word: + output_tokens.append(self.unk_token) + continue + + is_bad = False + start = 0 + sub_tokens = [] + while start < len(chars): + end = len(chars) + cur_substr = None + + while start < end: + substr = "".join(chars[start:end]) + if start > 0: + substr = "##" + substr + + if substr.startswith("##"): + if _is_chinese_substr(substr): + if substr[2:] in self.vocab: # 中文substr + cur_substr = substr + break + else: + if substr in self.vocab: + cur_substr = substr # 英文substr + break + else: + if substr in self.vocab: # 非substr，可以是字，也可以是整个中文词 + cur_substr = substr + break + end -= 1 + + if cur_substr is None: + is_bad = True + break + + sub_tokens.append(cur_substr) + start = end + + if is_bad: + output_tokens.append(self.unk_token) + else: + output_tokens.extend(sub_tokens) + return output_tokens + + +def _is_whitespace(char): + """Checks whether `chars` is a whitespace character.""" + # \t, \n, and \r are technically contorl characters but we treat them + # as whitespace since they are generally considered as such. + if char == " " or char == "\t" or char == "\n" or char == "\r": + return True + cat = unicodedata.category(char) + if cat == "Zs": + return True + return False + + +def _is_control(char): + """Checks whether `chars` is a control character.""" + # These are technically control characters but we count them as whitespace + # characters. + if char == "\t" or char == "\n" or char == "\r": + return False + cat = unicodedata.category(char) + if cat in ("Cc", "Cf"): + return True + return False + + +def _is_punctuation(char): + """Checks whether `chars` is a punctuation character.""" + cp = ord(char) + # We treat all non-letter/number ASCII as punctuation. + # Characters such as "^", "$", and "`" are not in the Unicode + # Punctuation class but we treat them as punctuation anyways, for + # consistency. + if ( + (cp >= 33 and cp <= 47) + or (cp >= 58 and cp <= 64) + or (cp >= 91 and cp <= 96) + or (cp >= 123 and cp <= 126) + ): + return True + cat = unicodedata.category(char) + if cat.startswith("P"): + return True + return False + + +class BasicTokenizerWithChineseWWM(object): + """Runs basic tokenization (punctuation splitting, lower casing, etc.). + (nickpan)对中英混杂做了特殊处理，见_tokenize_chinese_chars + """ + + def __init__(self, do_lower_case=True): + """Constructs a BasicTokenizer. + + Args: + do_lower_case: Whether to lower case the input. + """ + try: + import jieba + + self.pre_tokenizer = lambda x: jieba.lcut(x, HMM=False) + except ImportError: + raise (ImportError("Chinese WWM need jieba")) + + self.do_lower_case = do_lower_case + + def tokenize(self, text): + """Tokenizes a piece of text.""" + text = convert_to_unicode(text) + text = self._clean_text(text) + + # This was added on November 1st, 2018 for the multilingual and Chinese + # models. This is also applied to the English models now, but it doesn't + # matter since the English models were not trained on any Chinese data + # and generally don't have any Chinese data in them (there are Chinese + # characters in the vocabulary because Wikipedia does have some Chinese + # words in the English Wikipedia.). + text = self._tokenize_chinese_chars(text) + orig_tokens = whitespace_tokenize(text) + split_tokens = [] + for token in orig_tokens: + if self.do_lower_case: + token = token.lower() + token = self._run_strip_accents(token) + split_tokens.extend(self._run_split_on_punc(token)) + + output_tokens = whitespace_tokenize(" ".join(split_tokens)) + return output_tokens + + def _run_strip_accents(self, text): + """Strips accents from a piece of text.""" + text = unicodedata.normalize("NFD", text) + output = [] + for char in text: + cat = unicodedata.category(char) + if cat == "Mn": + continue + output.append(char) + return "".join(output) + + def _run_split_on_punc(self, text): + """Splits punctuation on a piece of text.""" + chars = list(text) + i = 0 + start_new_word = True + output = [] + while i < len(chars): + char = chars[i] + if _is_punctuation(char): + output.append([char]) + start_new_word = True + else: + if start_new_word: + output.append([]) + start_new_word = False + output[-1].append(char) + i += 1 + + return ["".join(x) for x in output] + + def _tokenize_chinese_chars(self, text): + """Adds whitespace around any CJK character. + (nickpan)并且如果是纯中文片段，则用jieba分词，否则则保留 + 两边加空格的操作。 + """ + output = [] + piece = "" + for char in text: + cp = ord(char) + if self._is_chinese_char(cp): + piece += char + else: + chinese_words = self.pre_tokenizer(piece) + for word in chinese_words: + output.append(" ") + output.append(word) + output.append(" ") + output.append(char) + piece = "" + + chinese_words = self.pre_tokenizer(piece) + for word in chinese_words: + output.append(" ") + output.append(word) + output.append(" ") + + return "".join(output) + + def _is_chinese_char(self, cp): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ( + (cp >= 0x4E00 and cp <= 0x9FFF) + or (cp >= 0x3400 and cp <= 0x4DBF) # + or (cp >= 0x20000 and cp <= 0x2A6DF) # + or (cp >= 0x2A700 and cp <= 0x2B73F) # + or (cp >= 0x2B740 and cp <= 0x2B81F) # + or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0xF900 and cp <= 0xFAFF) + or (cp >= 0x2F800 and cp <= 0x2FA1F) # + ): # + return True + + return False + + def _clean_text(self, text): + """Performs invalid character removal and whitespace cleanup on text.""" + output = [] + for char in text: + cp = ord(char) + if cp == 0 or cp == 0xFFFD or _is_control(char): + continue + if _is_whitespace(char): + output.append(" ") + else: + output.append(char) + return "".join(output) + + +def _is_chinese_substr(char): + return re.findall("##[\u4E00-\u9FA5]", char) diff --git a/projects/QQP/tokenizer/tokenizer.py b/projects/QQP/tokenizer/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..d2376025850b58712fabdef65cf905bd89209b8c --- /dev/null +++ b/projects/QQP/tokenizer/tokenizer.py @@ -0,0 +1,187 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import re + +import jieba + +from libai.tokenizer.tokenization_base import PreTrainedTokenizer + +from .bert_tokenization import FullTokenizer as FullBertTokenizer + +logger = logging.getLogger("libai." + __name__) + + +class _BertCNWWMTokenizer(PreTrainedTokenizer): + """Chinese whole word BERT tokenizer.""" + + def __init__(self, vocab_file, lower_case=True, vocab_extra_ids=0): + if lower_case: + name = "BERT Lower Case" + else: + name = "BERT Upper Case" + super().__init__(name) + self.tokenizer = FullBertTokenizer(vocab_file, do_lower_case=lower_case) + self.cls_id = self.tokenizer.vocab["[CLS]"] + self.sep_id = self.tokenizer.vocab["[SEP]"] + self.pad_id = self.tokenizer.vocab["[PAD]"] + self.mask_id = self.tokenizer.vocab["[MASK]"] + self.unk_id = self.tokenizer.vocab["[UNK]"] + self._additional_special_tokens = [] + + # (dsachan) Add BOS and EOS tokens + # SPECIAL_TOKENS = {"eos_token": "[EOS]", "bos_token": "[BOS]"} + self._bos_token = "[BOS]" + self.add_token(self._bos_token) + self._bos_token_id = self.vocab.get(self._bos_token) + + self._eos_token = "[EOS]" + self.add_token(self._eos_token) + self._eos_token_id = self.vocab.get(self._eos_token) + + # (dsachan) Add additional special tokens + # These can be used as sentinel tokens in T5 model inputs + additional_special_tokens = [] + additional_special_tokens.extend( + ["".format(i) for i in range(vocab_extra_ids)] + ) + self.add_additional_special_tokens(additional_special_tokens) + + def add_token(self, token): + if token not in self.vocab: + self.inv_vocab[self.vocab_size] = token + # self.vocab_size comes from len(vocab) + # and it will increase as we add elements + self.vocab[token] = self.vocab_size + + def add_additional_special_tokens(self, tokens_list): + setattr(self, "additional_special_tokens", tokens_list) + for value in tokens_list: + self.add_token(value) + + @property + def vocab_size(self): + return self.tokenizer.vocab_size() + + @property + def vocab(self): + return self.tokenizer.vocab + + @property + def inv_vocab(self): + return self.tokenizer.inv_vocab + + def tokenize(self, text): + text_tokens = self.tokenizer.tokenize(text) + # 使用jieba分词 + text_tokens = get_new_segment(text_tokens) + return self.tokenizer.convert_tokens_to_ids(text_tokens) + + def decode(self, ids): + tokens = self.tokenizer.convert_ids_to_tokens(ids) + return self.tokenizer.convert_tokens_to_string(tokens) + + def decode_token_ids(self, token_ids): + tokens = self.tokenizer.convert_ids_to_tokens(token_ids) + exclude_list = ["[PAD]", "[CLS]"] + non_pads = [t for t in tokens if t not in exclude_list] + + result = "" + for s in non_pads: + if s.startswith("##"): + result += s[2:] + else: + result += " " + s + + return result + + @property + def cls(self): + return self.cls_id + + @property + def sep(self): + return self.sep_id + + @property + def pad(self): + return self.pad_id + + @property + def mask(self): + return self.mask_id + + @property + def bos_token(self): + """Beginning of sentence token id""" + return self._bos_token + + @property + def eos_token(self): + """End of sentence token id""" + return self._eos_token + + @property + def additional_special_tokens(self): + """All the additional special tokens you may want to use (list of strings).""" + return self._additional_special_tokens + + @property + def bos_token_id(self): + """Id of the beginning of sentence token in the vocabulary.""" + return self._bos_token_id + + @property + def eos_token_id(self): + """Id of the end of sentence token in the vocabulary.""" + return self._eos_token_id + + @property + def additional_special_tokens_ids(self): + """Ids of all the additional special tokens in the vocabulary (list of integers).""" + return [self.vocab.get(token) for token in self._additional_special_tokens] + + @additional_special_tokens.setter + def additional_special_tokens(self, value): + self._additional_special_tokens = value + + +def get_new_segment(segment): + seq_cws = jieba.cut("".join(segment) if isinstance(segment, list) else segment) + seq_cws_dict = {x: 1 for x in seq_cws} + new_segment = [] + i = 0 + while i < len(segment): + if len(re.findall("[\u4E00-\u9FA5]", segment[i])) == 0: + new_segment.append(segment[i]) + i += 1 + continue + + has_add = False + for length in range(3, 0, -1): + if i + length > len(segment): + continue + if "".join(segment[i : i + length]) in seq_cws_dict: + new_segment.append(segment[i]) + for l in range(1, length): + new_segment.append("##" + segment[i + l]) + i += length + has_add = True + break + if not has_add: + new_segment.append(segment[i]) + i += 1 + return new_segment diff --git a/projects/README.md b/projects/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8d6a936ee1df2f199379f7ff1201e06366e6f51c --- /dev/null +++ b/projects/README.md @@ -0,0 +1,18 @@ +## Introduction +Here are projects built based on LiBai, which show you how to use LiBai as a library, to make your own projects more maintainable. + +## Projects on LiBai +- [CLIP: Contrastive Language-Image Pre-Training](./CLIP) +- [Couplets](./Couplets/) +- [DALLE2](./DALLE2/) +- [GLM: General Language Model Pretraining with Autoregressive Blank Infilling](./GLM/) +- [MAE: Masked Autoencoders Are Scalable Vision Learners](./MAE) +- [MagicPrompt-Stable-Diffusion](./MagicPrompt/) +- [MOCOV3: An Empirical Study of Training Self-Supervised Vision Transformers](./MOCOV3) +- [MT5](./MT5/) +- [NeRF](./NeRF/) +- [PaLM: Pathways Language Model](./PaLM) +- [QQP: Quora Question Pairs](./QQP) +- [SimCSE: Simple Contrastive Learning of Sentence Embeddings](./SimCSE) +- [T5: Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](./T5) +- [Text Classification](./text_classification) diff --git a/projects/SimCSE/config/config_simcse_sup.py b/projects/SimCSE/config/config_simcse_sup.py new file mode 100644 index 0000000000000000000000000000000000000000..8a0bd9d789df039ca9465b0132bb7e83a3a53db5 --- /dev/null +++ b/projects/SimCSE/config/config_simcse_sup.py @@ -0,0 +1,101 @@ +from omegaconf import OmegaConf + +from configs.common.data.bert_dataset import tokenization +from configs.common.models.bert import cfg as simcse_cfg +from configs.common.models.graph import graph +from configs.common.optim import optim +from configs.common.train import train +from libai.config import LazyCall +from libai.data.build import build_nlp_test_loader, build_nlp_train_loader +from libai.scheduler import WarmupExponentialLR +from libai.tokenizer import BertTokenizer +from projects.SimCSE.dataset.dataset import TestDataset_sup, TrainDataset_sup +from projects.SimCSE.evaluator import SimcseEvaluator +from projects.SimCSE.modeling.simcse_sup import Simcse_sup + +optim["lr"] = 1e-5 +graph["enabled"] = True + +tokenization.tokenizer = LazyCall(BertTokenizer)( + vocab_file="./data/vocab.txt", +) +tokenization.make_vocab_size_divisible_by = 1 + + +dataloader = OmegaConf.create() +dataloader.train = LazyCall(build_nlp_train_loader)( + dataset=[ + LazyCall(TrainDataset_sup)( + name="snli-sup", + path="./data/SNLI/train.txt", + tokenizer=LazyCall(BertTokenizer)(vocab_file="./data/vocab.txt"), + max_len=64, + ) + ], +) + +dataloader.test = [ + LazyCall(build_nlp_test_loader)( + dataset=LazyCall(TestDataset_sup)( + name="cnsd_sts", + path="./data/STS/cnsd-sts-test.txt", + tokenizer=LazyCall(BertTokenizer)(vocab_file="./data/vocab.txt"), + ), + ), + LazyCall(build_nlp_test_loader)( + dataset=LazyCall(TestDataset_sup)( + name="cnsd_sts", + path="./data/STS/cnsd-sts-dev.txt", + tokenizer=LazyCall(BertTokenizer)(vocab_file="./data/vocab.txt"), + ) + ), +] + +simcse_cfg.update( + dict( + vocab_size=21128, + hidden_size=768, + hidden_layers=12, + layernorm_eps=1e-12, + intermediate_size=3072, + pretrained_model_weight="./data/pytorch_model.bin", + temp=0.05, + pooler_type="cls", + bias_gelu_fusion=False, + bias_dropout_fusion=False, + apply_query_key_layer_scaling=False, + apply_residual_post_layernorm=True, + ) +) + +model = LazyCall(Simcse_sup)(cfg=simcse_cfg) + +train.update( + dict( + output_dir="./result", + train_micro_batch_size=8, + test_micro_batch_size=8, + train_epoch=1, + train_iter=1000, + log_period=10, + dist=dict( + data_parallel_size=8, + tensor_parallel_size=1, + pipeline_parallel_size=1, + ), + evaluation=dict( + enabled=True, + evaluator=LazyCall(SimcseEvaluator)(), + eval_period=10, + eval_metric="Spearman", + eval_mode="max", + eval_iter=100, + ), + scheduler=LazyCall(WarmupExponentialLR)( + warmup_factor=0.0, + gamma=1.0, + warmup_method="linear", + warmup_iter=0.0, + ), + ) +) diff --git a/projects/SimCSE/config/config_simcse_unsup.py b/projects/SimCSE/config/config_simcse_unsup.py new file mode 100644 index 0000000000000000000000000000000000000000..0ecec1a30cbfb94218ccf309adcc47adb5218050 --- /dev/null +++ b/projects/SimCSE/config/config_simcse_unsup.py @@ -0,0 +1,99 @@ +from omegaconf import OmegaConf + +from configs.common.data.bert_dataset import tokenization +from configs.common.models.bert import cfg as simcse_cfg +from configs.common.models.graph import graph +from configs.common.optim import optim +from configs.common.train import train +from libai.config import LazyCall +from libai.data.build import build_nlp_test_loader, build_nlp_train_loader +from libai.scheduler import WarmupExponentialLR +from libai.tokenizer import BertTokenizer +from projects.SimCSE.dataset.dataset import TestDataset_unsup, TrainDataset_unsup +from projects.SimCSE.evaluator import SimcseEvaluator +from projects.SimCSE.modeling.simcse_unsup import Simcse_unsup + +optim["lr"] = 3e-5 +graph["enabled"] = True + +tokenization.tokenizer = LazyCall(BertTokenizer)( + vocab_file="./data/vocab.txt", +) +tokenization.make_vocab_size_divisible_by = 1 + +dataloader = OmegaConf.create() +dataloader.train = LazyCall(build_nlp_train_loader)( + dataset=[ + LazyCall(TrainDataset_unsup)( + name="snli-unsup", + path="./data/SNLI/train.txt", + tokenizer=LazyCall(BertTokenizer)(vocab_file="./data/vocab.txt"), + max_len=64, + path2="./data/STS/cnsd-sts-train.txt", + ) + ], +) + +dataloader.test = [ + LazyCall(build_nlp_test_loader)( + dataset=LazyCall(TestDataset_unsup)( + name="cnsd_sts", + path="./data/STS/cnsd-sts-test.txt", + tokenizer=LazyCall(BertTokenizer)(vocab_file="./data/vocab.txt"), + ), + ), + LazyCall(build_nlp_test_loader)( + dataset=LazyCall(TestDataset_unsup)( + name="cnsd_sts", + path="./data/STS/cnsd-sts-dev.txt", + tokenizer=LazyCall(BertTokenizer)(vocab_file="./data/vocab.txt"), + ) + ), +] + + +simcse_cfg.update( + dict( + vocab_size=21128, + hidden_size=768, + hidden_layers=12, + layernorm_eps=1e-12, + intermediate_size=3072, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + apply_query_key_layer_scaling=False, + apply_residual_post_layernorm=True, + pretrained_model_weight="./data/pytorch_model.bin", + pooler_type="cls", + temp=0.05, + ) +) + +model = LazyCall(Simcse_unsup)(cfg=simcse_cfg) + +train.update( + dict( + output_dir="./result", + train_micro_batch_size=8, + test_micro_batch_size=8, + train_epoch=1, + train_iter=2500, + log_period=10, + dist=dict( + data_parallel_size=8, + tensor_parallel_size=1, + pipeline_parallel_size=1, + ), + evaluation=dict( + enabled=True, + evaluator=LazyCall(SimcseEvaluator)(), + eval_period=10, + eval_iter=1e5, + eval_metric="Spearman", + eval_mode="max", + ), + scheduler=LazyCall(WarmupExponentialLR)( + warmup_factor=0.000, gamma=1.0, warmup_method="linear", warmup_iter=0 + ), + ) +) diff --git a/projects/SimCSE/dataset/dataset.py b/projects/SimCSE/dataset/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..3881b883e5a02199a1bf6efc1436974227d75368 --- /dev/null +++ b/projects/SimCSE/dataset/dataset.py @@ -0,0 +1,230 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random + +import jsonlines +import oneflow as flow +from oneflow.utils.data import Dataset + +from libai.data.structures import DistTensorData, Instance + + +def load_data(name, path): + assert name in ["snli-sup", "snli-unsup", "lqcmc", "eng_sts", "cnsd_sts", "wiki", "add"] + + def load_snli_data_unsup(path): + with jsonlines.open(path, "r") as f: + return [line.get("origin") for line in f] + + def load_snli_data_sup(path): + with jsonlines.open(path, "r") as f: + return [(line["origin"], line["entailment"], line["contradiction"]) for line in f] + + def load_lqcmc_data(path): + with open(path, "r", encoding="utf8") as f: + return [line.strip().split("\t")[0] for line in f] + + def load_cnsd_sts_data(path): + with open(path, "r", encoding="utf8") as f: + return [(line.split("||")[1], line.split("||")[2], line.split("||")[3]) for line in f] + + def load_wiki_data(path): + data = [] + with open(path, "r", encoding="utf8") as file: + for line in file.readlines(): + line = " ".join(line.strip().split()) + data.append(line) + return data + + def load_eng_sts_data(path): + data = [] + with open(path, "r", encoding="utf8") as file: + for line in file.readlines(): + line = line.strip().split("\t") + data.append(line) + return data + + def load_sts_to_train(path): + if path is None: + return [] + with open( + path, + "r", + encoding="utf8", + ) as f: + data = [line.split("||")[1] for line in f] + return data + + if name == "snli-unsup": + return load_snli_data_unsup(path) + elif name == "snli-sup": + return load_snli_data_sup(path) + elif name == "wiki": + return load_wiki_data(path) + elif name == "cnsd_sts": + return load_cnsd_sts_data(path) + elif name == "eng_sts": + return load_eng_sts_data(path) + elif name == "lqcmc": + return load_lqcmc_data(path) + else: + return load_sts_to_train(path) + + +def padding_for_ids(data, pad_id=0, max_len=64): + data["input_ids"] = data["input_ids"] + [pad_id] * (max_len - len(data["input_ids"])) + data["attention_mask"] = data["attention_mask"] + [0] * (max_len - len(data["attention_mask"])) + + data["input_ids"] = [data["input_ids"], data["input_ids"]] + data["attention_mask"] = [data["attention_mask"], data["attention_mask"]] + + return Instance( + input_ids=DistTensorData(flow.tensor(data["input_ids"], dtype=flow.long)), + attention_mask=DistTensorData(flow.tensor(data["attention_mask"], dtype=flow.long)), + ) + + +class TrainDataset_unsup(Dataset): + # unsup + def __init__(self, name, path, tokenizer, max_len, path2=None): + self.name = name + self.data = load_data(name, path) + load_data("add", path2) + random.shuffle(self.data) + self.tokenizer = tokenizer + self.max_len = max_len + self.pad_id = self.tokenizer.pad_token_id + self.cls_id = self.tokenizer.cls_token_id + self.sep_id = self.tokenizer.sep_token_id + + def __len__(self): + return len(self.data) + + def text2id(self, text): + tokens = self.tokenizer.tokenize(text) + ids = self.tokenizer.convert_tokens_to_ids(tokens) + + ids = ids[: self.max_len - 2] + ids = [self.cls_id] + ids + [self.sep_id] + + attention_mask = [1] * len(ids) + + return padding_for_ids( + data={ + "input_ids": ids, + "attention_mask": attention_mask, + }, + pad_id=self.pad_id, + max_len=self.max_len, + ) + + def __getitem__(self, index): + return self.text2id(self.data[index]) + + +class TestDataset_unsup(Dataset): + # sts datasets + def __init__(self, name, path, tokenizer): + self.data = load_data(name, path) + self.tokenizer = tokenizer + self.max_len = 64 + self.pad_id = self.tokenizer.pad_token_id + self.cls_id = self.tokenizer.cls_token_id + self.sep_id = self.tokenizer.sep_token_id + + def __len__(self): + return len(self.data) + + def text2id(self, text): + tokens = self.tokenizer.tokenize(text) + ids = self.tokenizer.convert_tokens_to_ids(tokens) + ids = ids[: self.max_len - 2] + + ids = [self.cls_id] + ids + [self.sep_id] + length = len(ids) + + ids = ids + [self.pad_id] * (self.max_len - length) + attention_mask = [1] * length + [0] * (self.max_len - length) + + return { + "input_ids": ids, + "attention_mask": attention_mask, + } + + def __getitem__(self, index): + # sent1, sent2, laebl + sample = self.data[index] + + sent1 = self.text2id(sample[0]) + sent2 = self.text2id(sample[1]) + score = int(sample[2]) + + return Instance( + input_ids=DistTensorData( + flow.tensor([sent1["input_ids"], sent2["input_ids"]], dtype=flow.long) + ), + attention_mask=DistTensorData( + flow.tensor([sent1["attention_mask"], sent2["attention_mask"]], dtype=flow.long) + ), + labels=DistTensorData(flow.tensor(score, dtype=flow.int)), + ) + + +class TrainDataset_sup(Dataset): + def __init__(self, name, path, tokenizer, max_len=64): + self.data = load_data(name, path) + self.tokenizer = tokenizer + self.max_len = max_len + self.pad_id = self.tokenizer.pad_token_id + self.cls_id = self.tokenizer.cls_token_id + self.sep_id = self.tokenizer.sep_token_id + + def __len__(self): + return len(self.data) + + def pad_text(self, ids): + attention_mask = [1] * len(ids) + ids = ids + [self.pad_id] * (self.max_len - len(ids)) + attention_mask = attention_mask + [0] * (self.max_len - len(attention_mask)) + return ids, attention_mask + + def text2id(self, text): + tokens = self.tokenizer.tokenize(text) + ids = self.tokenizer.convert_tokens_to_ids(tokens) + ids = ids[: self.max_len - 2] + ids = [self.cls_id] + ids + [self.sep_id] + ids, attention_mask = self.pad_text(ids) + return ids, attention_mask + + def __getitem__(self, index): + ids0, mask0 = self.text2id(self.data[index][0]) + ids1, mask1 = self.text2id(self.data[index][1]) + ids2, mask2 = self.text2id(self.data[index][2]) + return Instance( + input_ids=DistTensorData(flow.tensor([ids0, ids1, ids2], dtype=flow.long)), + attention_mask=DistTensorData(flow.tensor([mask0, mask1, mask2], dtype=flow.long)), + ) + + +class TestDataset_sup(TrainDataset_sup): + def __getitem__(self, index): + label = int(self.data[index][2]) + ids0, mask0 = self.text2id(self.data[index][0]) + ids1, mask1 = self.text2id(self.data[index][1]) + return Instance( + input_ids=DistTensorData(flow.tensor([ids0, ids1], dtype=flow.long)), + attention_mask=DistTensorData(flow.tensor([mask0, mask1], dtype=flow.long)), + labels=DistTensorData(flow.tensor(label, dtype=flow.int)), + ) diff --git a/projects/SimCSE/evaluator.py b/projects/SimCSE/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..d4e9404293e05536bb22d572feae42e8f233f6d6 --- /dev/null +++ b/projects/SimCSE/evaluator.py @@ -0,0 +1,50 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +from scipy.stats import spearmanr + +from libai.evaluation import DatasetEvaluator +from libai.utils import distributed as dist + + +def spearman_target(cos_sim, labels): + return spearmanr(cos_sim, labels).correlation + + +class SimcseEvaluator(DatasetEvaluator): + def __init__(self): + self._predictions = [] + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + sim = outputs["sim"] + labels = inputs["labels"] + self._predictions.append({"sim": sim, "labels": labels}) + + def evaluate(self): + if not dist.is_main_process(): + return {} + else: + predictions = self._predictions + sim_array = np.array([]) + label_array = np.array([]) + for prediction in predictions: + sim_array = np.append(sim_array, dist.tton(prediction["sim"])) + label_array = np.append(label_array, dist.tton(prediction["labels"])) + self._results = spearman_target(sim_array, label_array) + return {"Spearman": self._results} diff --git a/projects/SimCSE/modeling/bert_for_simcse.py b/projects/SimCSE/modeling/bert_for_simcse.py new file mode 100644 index 0000000000000000000000000000000000000000..f9b6ae84b5d44a940b1685297e59630838091dc0 --- /dev/null +++ b/projects/SimCSE/modeling/bert_for_simcse.py @@ -0,0 +1,33 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libai.models import BertModel + + +class BertForSimCSE(BertModel): + def __init__(self, cfg): + super().__init__(cfg) + + def forward(self, input_ids, attention_mask, tokentype_ids=None): + extended_attention_mask = self.extended_attn_mask(attention_mask) + embedding_output = self.embeddings(input_ids, tokentype_ids) + total_hidden = [] + hidden_states = embedding_output + for layer in self.encoders: + hidden_states = layer(hidden_states, extended_attention_mask) + total_hidden.append(hidden_states) + encoder_output = self.final_layernorm(hidden_states) + pooled_output = self.pooler(encoder_output) if self.pooler is not None else None + return encoder_output, pooled_output, total_hidden diff --git a/projects/SimCSE/modeling/model_utils.py b/projects/SimCSE/modeling/model_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1c994c7a0ebbdef6203179a75889409292d7f2e7 --- /dev/null +++ b/projects/SimCSE/modeling/model_utils.py @@ -0,0 +1,22 @@ +import oneflow as flow +from oneflow import nn + +import libai + + +def cosine_similarity(x, y, dim=-1): + return flow.sum(x * y, dim=dim) / (flow.linalg.norm(x, dim=dim) * flow.linalg.norm(y, dim=dim)) + + +class MLPLayer(nn.Module): + def __init__(self, cfg): + super().__init__() + self.dense = libai.layers.Linear( + cfg.hidden_size, cfg.hidden_size, bias=True, parallel="row", layer_idx=-1 + ) + self.activation = libai.layers.build_activation("tanh") + + def forward(self, features): + x = self.dense(features) + x = self.activation(x) + return x diff --git a/projects/SimCSE/modeling/simcse_sup.py b/projects/SimCSE/modeling/simcse_sup.py new file mode 100644 index 0000000000000000000000000000000000000000..1a3e75615984e6ce1360046da51f42bdcd12649d --- /dev/null +++ b/projects/SimCSE/modeling/simcse_sup.py @@ -0,0 +1,115 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +from oneflow import nn + +from libai.utils import distributed as dist +from projects.SimCSE.modeling.model_utils import MLPLayer, cosine_similarity +from projects.SimCSE.utils.load_huggingface_weight import load_huggingface_bert + +from .bert_for_simcse import BertForSimCSE + + +class Simcse_sup(nn.Module): + def __init__(self, cfg): + super().__init__() + self.bert = BertForSimCSE(cfg) + self.mlp = MLPLayer(cfg) + self.pooler_type = cfg.pooler_type + + if cfg.pretrained_model_weight is not None: + load_huggingface_bert( + self.bert, + cfg.pretrained_model_weight, + cfg["hidden_size"], + cfg["num_attention_heads"], + cfg["hidden_layers"], + ) + + def pooler(self, inputs, attention_mask): + if self.pooler_type == "cls": + return inputs[0][:, 0] + + elif self.pooler_type == "pooled": + return inputs[1] + + elif self.pooler_type == "last-avg": + last_hidden = inputs[0] + return (last_hidden * attention_mask.unsqueeze(-1)).sum(1) / attention_mask.sum( + -1 + ).unsqueeze(-1) + + elif self.pooler_type == "first-last-avg": + first_hidden = inputs[2][1] + last_hidden = inputs[0] + res = ((first_hidden + last_hidden) / 2.0 * attention_mask.unsqueeze(-1)).sum( + 1 + ) / attention_mask.sum(-1).unsqueeze(-1) + return res + + def create_use_row(self, labels): + count = 0 + use_row = [] + for row in range(labels.size(0)): + if count % 2 == 0 and count != 0: + count = 0 + continue + use_row.append(row) + count += 1 + return flow.tensor(use_row, sbp=labels.sbp, placement=labels.placement) + + def forward(self, input_ids, attention_mask, token_type_ids=None, labels=None): + if self.training: + bs = input_ids.size(0) + input_ids = input_ids.view(bs * 3, -1) + attention_mask = attention_mask.view(bs * 3, -1) + out = self.bert(input_ids, attention_mask) + out = self.pooler(out, attention_mask) + out = self.mlp(out) + labels = flow.arange( + out.size(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=out.placement, + ) + use_row = self.create_use_row(labels) + labels = (use_row - use_row % 3 * 2) + 1 + sim = cosine_similarity(out.unsqueeze(1), out.unsqueeze(0)) + sim = ( + sim + - flow.eye( + out.size(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=out.placement, + ) + * 1e12 + ) + sim = flow.index_select(sim, dim=0, index=use_row) + sim = sim / 0.05 + loss = nn.CrossEntropyLoss()(sim, labels) + return {"loss": loss} + else: + bs = input_ids.size(0) + input_ids = input_ids.view(bs * 2, -1) + attention_mask = attention_mask.view(bs * 2, -1) + out = self.bert(input_ids, attention_mask) + out = self.pooler(out, attention_mask) + self.mlp(out) + out = out.view(bs, 2, -1) + sent1 = out[:, 0] + sent2 = out[:, 1] + sim = cosine_similarity(sent1, sent2) + sim = sim.to_global(sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast])) + return {"sim": sim.unsqueeze(1), "labels": labels} diff --git a/projects/SimCSE/modeling/simcse_unsup.py b/projects/SimCSE/modeling/simcse_unsup.py new file mode 100644 index 0000000000000000000000000000000000000000..55613150902f68e79b26f6cb927b6c7e7a5b9941 --- /dev/null +++ b/projects/SimCSE/modeling/simcse_unsup.py @@ -0,0 +1,102 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +from oneflow import nn + +from libai.utils import distributed as dist +from projects.SimCSE.modeling.model_utils import MLPLayer, cosine_similarity +from projects.SimCSE.utils.load_huggingface_weight import load_huggingface_bert + +from .bert_for_simcse import BertForSimCSE + + +class Simcse_unsup(nn.Module): + def __init__(self, cfg): + super().__init__() + self.bert = BertForSimCSE(cfg) + self.mlp = MLPLayer(cfg) + self.pooler_type = cfg.pooler_type + + if cfg.pretrained_model_weight is not None: + load_huggingface_bert( + self.bert, + cfg.pretrained_model_weight, + cfg["hidden_size"], + cfg["num_attention_heads"], + cfg["hidden_layers"], + ) + + def pooler(self, inputs, attention_mask): + if self.pooler_type == "cls": + return inputs[0][:, 0] + + elif self.pooler_type == "pooled": + return inputs[1] + + elif self.pooler_type == "last-avg": + last_hidden = inputs[0] + return (last_hidden * attention_mask.unsqueeze(-1)).sum(1) / attention_mask.sum( + -1 + ).unsqueeze(-1) + + elif self.pooler_type == "first-last-avg": + first_hidden = inputs[2][1] + last_hidden = inputs[0] + res = ((first_hidden + last_hidden) / 2.0 * attention_mask.unsqueeze(-1)).sum( + 1 + ) / attention_mask.sum(-1).unsqueeze(-1) + return res + + def forward(self, input_ids, attention_mask, token_type_ids=None, labels=None): + if self.training: + bs, num_sent = input_ids.size(0), input_ids.size(1) + input_ids = input_ids.view(bs * num_sent, -1) + attention_mask = attention_mask.view(bs * num_sent, -1) + out = self.bert(input_ids, attention_mask) + out = self.pooler(out, attention_mask) + out = self.mlp(out) + labels = flow.arange( + out.size(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=out.placement, + ) + labels = (labels - labels % 2 * 2) + 1 + sim = cosine_similarity(out.unsqueeze(1), out.unsqueeze(0)) + sim = ( + sim + - flow.eye( + out.size(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=out.placement, + ) + * 1e12 + ) + sim = sim / 0.05 + loss = nn.CrossEntropyLoss()(sim, labels) + return {"loss": loss} + else: + bs, num_sent = input_ids.size(0), input_ids.size(1) + input_ids = input_ids.view(bs * num_sent, -1) + attention_mask = attention_mask.view(bs * num_sent, -1) + out = self.bert(input_ids, attention_mask) + out = self.pooler(out, attention_mask) + self.mlp(out) + out = out.view(bs, num_sent, -1) + sent1 = out[:, 0] + sent2 = out[:, 1] + sim = cosine_similarity(sent1, sent2) + sim = sim.to_global(sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast])) + return {"sim": sim.unsqueeze(1), "labels": labels} diff --git a/projects/SimCSE/readme.md b/projects/SimCSE/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..5d916b59a328a0fbaea6a280db303ed165f67782 --- /dev/null +++ b/projects/SimCSE/readme.md @@ -0,0 +1,40 @@ +# SimCSE +Contributor{Xie Zipeng: xzpaiks@163.com} + +Reproduce Supervised-simcse and Unsupervised-simcse with OneFlow. + +## Introduce +SimCSE is a sentence representation learning method, in which there are two training methods: supervised learning and unsupervised learning. The unsupervised learning method is to input sentences and predict itself in the comparison target, and only use the standard dropout as noise; The supervised learning method uses the NLI data set, taking 'entry' as a positive sample and 'contrast' as a negative sample for supervised learning. This task uses Spearman to evaluate the model's performance on STS dataset, and uses Alignment and Uniformity to measure the effect of contrastive learning. +- 《SimCSE: Simple Contrastive Learning of Sentence Embeddings》: https://arxiv.org/pdf/2104.08821.pdf +- Official GitHub: https://github.com/princeton-nlp/SimCSE + +## Modle List(single GPU) +learning_rate=3e-5, batch_size=64 +| Unsupervised-Model |STS-B dev |STS-B test|Pool type | +|:-------------------------------|:--------:|:--------:|:--------:| +|[unsup-simcse-bert-base-chinese](http://oneflow-public.oss-cn-beijing.aliyuncs.com/model_zoo/LiBai/SimCSE/unsup-simcse-bert-base-chinese.zip) |74.64 |68.67 |cls | +|unsup-simcse-bert-base-chinese |74.86 |68.71 |last-avg | +|unsup-simcse-bert-base-chinese |64.33 |54.82 |pooled | +|unsup-simcse-bert-base-chinese |74.32 |67.55 |first-last-avg| + +learning_rate=1e-5, batch_size=64 +| Supervised-Model |STS-B dev |STS-B test|Pool type | +|:-------------------------------|:--------:|:--------:|:--------:| +|[sup-simcse-bert-base-chinese](http://oneflow-public.oss-cn-beijing.aliyuncs.com/model_zoo/LiBai/SimCSE/sup-simcse-bert-base-chinese.zip) |80.93 |77.32 |cls | +|sup-simcse-bert-base-chinese |81.20 |77.09 |last-avg | +|sup-simcse-bert-base-chinese |76.61 |75.00 |pooled | +|sup-simcse-bert-base-chinese |80.64 |76.33 |first-last-avg| + +## Training +Training SimCSE on 8 GPUs using data parallelism. +```bash +cd /path/to/libai +bash projects/SimCSE/train.sh tools/train_net.py projects/SimCSE/config/config_simcse_unsup.py 8 +``` + +## Evaluation +Evaluate SimCSE on 8 GPUs using data parallelism: +```bash +cd /path/to/libai +bash projects/SimCSE/train.sh tools/train_net.py projects/SimCSE/config/config_simcse_unsup.py 8 --eval-only +``` diff --git a/projects/SimCSE/train.sh b/projects/SimCSE/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..6fa5181522b51a3934a7b78571001f7891c773a5 --- /dev/null +++ b/projects/SimCSE/train.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +FILE=$1 +CONFIG=$2 +GPUS=$3 +NODE=${NODE:-1} +NODE_RANK=${NODE_RANK:-0} +ADDR=${ADDR:-127.0.0.1} +PORT=${PORT:-12345} +DATA_PATH="data" + +if [ ! -d "$DATA_PATH" ]; then + wget http://oneflow-public.oss-cn-beijing.aliyuncs.com/model_zoo/LiBai/SimCSE/data.zip + unzip data.zip + wget https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt -P ./data/ + wget https://huggingface.co/bert-base-chinese/resolve/main/pytorch_model.bin -P ./data/ +fi + +python3 -m oneflow.distributed.launch \ +--nproc_per_node $GPUS --nnodes $NODE --node_rank $NODE_RANK --master_addr $ADDR --master_port $PORT \ +$FILE --config-file $CONFIG ${@:4} \ No newline at end of file diff --git a/projects/SimCSE/utils/load_huggingface_weight.py b/projects/SimCSE/utils/load_huggingface_weight.py new file mode 100644 index 0000000000000000000000000000000000000000..3329a173ea76f98f51bd5eb8995908b068925941 --- /dev/null +++ b/projects/SimCSE/utils/load_huggingface_weight.py @@ -0,0 +1,160 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import OrderedDict + +import oneflow as flow +import torch + +import libai.utils.distributed as dist + + +def convert_tensor(tensor): + tensor = tensor.float() + return flow.Tensor(tensor.cpu().numpy()) + + +def conver_state(state, layers, hidden_size, num_heads, head_size): + save = OrderedDict() + not_saved = [] + Layers = layers + for name, tensor in state.items(): + if "embeddings" in name: + if "word_embeddings" in name: + save["embeddings.vocab_embeddings.weight"] = convert_tensor(tensor) + elif "position_embeddings" in name: + save["embeddings.position_embeddings.weight"] = convert_tensor(tensor) + elif "token_type_embeddings" in name: + save["embeddings.tokentype_embeddings.weight"] = convert_tensor(tensor) + elif "LayerNorm.gamma" in name: + save["encoders.0.input_layernorm.weight"] = convert_tensor(tensor) + elif "LayerNorm.beta" in name: + save["encoders.0.input_layernorm.bias"] = convert_tensor(tensor) + + elif "attention" in name: + if "self" in name: + index = name.split(".")[3] + if "encoders." + index + ".self_attention.query_key_value.weight" in save.keys(): + continue + q_w = name.replace(name.split(".")[6], "query").replace( + name.split(".")[7], "weight" + ) + k_w = name.replace(name.split(".")[6], "key").replace(name.split(".")[7], "weight") + v_w = name.replace(name.split(".")[6], "value").replace( + name.split(".")[7], "weight" + ) + q_b = name.replace(name.split(".")[6], "query").replace(name.split(".")[7], "bias") + k_b = name.replace(name.split(".")[6], "key").replace(name.split(".")[7], "bias") + v_b = name.replace(name.split(".")[6], "value").replace(name.split(".")[7], "bias") + + qkv_w = torch.cat((state[q_w], state[k_w], state[v_w]), dim=0) # 【768*3， 768】 + # function for weight----------------------------------- + qkv_w = qkv_w.view([3, num_heads, head_size, hidden_size]) + qkv_w = qkv_w.permute(1, 0, 2, 3).contiguous().view(3 * hidden_size, hidden_size) + # --------------------------------------------------------- + + qkv_b = torch.cat((state[q_b], state[k_b], state[v_b]), dim=-1) + # function for bias-------------------------------------- + qkv_b = qkv_b.view(3, num_heads, head_size) + qkv_b = qkv_b.permute(1, 0, 2).contiguous().view(-1) + # --------------------------------------------------------- + + target_w = "encoders." + index + ".self_attention.query_key_value.weight" + save[target_w] = convert_tensor(qkv_w) + target_b = "encoders." + index + ".self_attention.query_key_value.bias" + save[target_b] = convert_tensor(qkv_b) + elif "output" in name: + index = name.split(".")[3] + if "dense" in name: + if "weight" in name: + target = "encoders." + index + ".self_attention.dense.weight" + save[target] = convert_tensor(tensor) + elif "bias" in name: + target = "encoders." + index + ".self_attention.dense.bias" + save[target] = convert_tensor(tensor) + elif "LayerNorm" in name: + if "gamma" in name: + target = "encoders." + index + ".post_attention_layernorm.weight" + save[target] = convert_tensor(tensor) + elif "beta" in name: + target = "encoders." + index + ".post_attention_layernorm.bias" + save[target] = convert_tensor(tensor) + + elif "intermediate" in name: + index = name.split(".")[3] + if "encoders." + index + ".mlp.dense_h_to_4h.weight" in save.keys(): + continue + w = "bert.encoder.layer." + index + ".intermediate.dense.weight" + b = "bert.encoder.layer." + index + ".intermediate.dense.bias" + t_w = "encoders." + index + ".mlp.dense_h_to_4h.weight" + t_b = "encoders." + index + ".mlp.dense_h_to_4h.bias" + save[t_w] = convert_tensor(state[w]) + save[t_b] = convert_tensor(state[b]) + + elif "output" in name: + index = name.split(".")[3] + if "dense.weight" in name: + target = "encoders." + index + ".mlp.dense_4h_to_h.weight" + save[target] = convert_tensor(tensor) + elif "dense.bias" in name: + target = "encoders." + index + ".mlp.dense_4h_to_h.bias" + save[target] = convert_tensor(tensor) + elif "LayerNorm.gamma" in name: + if index == str(Layers - 1): + target = "final_layernorm.weight" + save[target] = convert_tensor(tensor) + continue + target = "encoders." + str(int(index) + 1) + ".input_layernorm.weight" + save[target] = convert_tensor(tensor) + elif "LayerNorm.beta" in name: + if index == str(Layers - 1): + target = "final_layernorm.bias" + save[target] = convert_tensor(tensor) + continue + target = "encoders." + str(int(index) + 1) + ".input_layernorm.bias" + save[target] = convert_tensor(tensor) + + elif "pooler" in name: + if "weight" in name: + save["pooler.dense.weight"] = convert_tensor(tensor) + elif "bias" in name: + save["pooler.dense.bias"] = convert_tensor(tensor) + else: + not_saved.append(name) + return save, not_saved + + +def load_tensor(tensor_lhs, tensor_rhs): + tensor_rhs = flow.to_global( + tensor_rhs, + placement=tensor_lhs.placement, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + tensor_rhs = tensor_rhs.to_global(sbp=tensor_lhs.sbp) + tensor_lhs.copy_(tensor_rhs) + + +def load_huggingface_bert(model, path, hidden_size, num_heads, layers=12): + head_size = hidden_size // num_heads + huggingface_state_dict = torch.load(path) + of_state_dict, _ = conver_state( + huggingface_state_dict, + layers=layers, + hidden_size=hidden_size, + num_heads=num_heads, + head_size=head_size, + ) + for key, value in of_state_dict.items(): + load_tensor(model.state_dict()[key], value) diff --git a/projects/T5/configs/mt5_pretrain.py b/projects/T5/configs/mt5_pretrain.py new file mode 100644 index 0000000000000000000000000000000000000000..bd9fd3bd8c37b97c9dd2e95d550daa6b2f27fedd --- /dev/null +++ b/projects/T5/configs/mt5_pretrain.py @@ -0,0 +1,91 @@ +from libai import evaluation +from libai.data.build import build_nlp_train_loader +from omegaconf import OmegaConf + +from libai.config import LazyCall +from libai.evaluation import PPLEvaluator, evaluator +from libai.scheduler import WarmupExponentialLR + +from configs.common.train import train +from configs.common.models.graph import graph + +from projects.T5.configs.optim import optim +from projects.T5.configs.t5_model_config import cfg +from projects.T5.datasets.dataset import UnsuperviseT5Dataset, collate_fn +from projects.T5.models.t5_model import T5ForPreTraining + + +train_data_path = "projects/T5/data/training_data/part_0" +pretrained_model_path = None + +micro_batch_size = 64 +optim["lr"] = 1e-4 + +# dataloader +dataloader = OmegaConf.create() +dataloader.train = LazyCall(build_nlp_train_loader)( + dataset=[ + LazyCall(UnsuperviseT5Dataset)( + data_path=train_data_path, + ) + ], + collate_fn=collate_fn( + vocab_size=12902, + max_seq_length=512, + noise_density=0.15, + mean_noise_span_length=3, + eos_token_id=12801, + pad_token_id=0, + decoder_start_token_id=12800, + ), +) + +model = LazyCall(T5ForPreTraining)(cfg=cfg) + +# model config +model.cfg.vocab_size = 12902 +model.cfg.hidden_size = 512 +model.cfg.hidden_layers = 8 +model.cfg.num_attention_heads = 6 +model.cfg.head_size = 64 +model.cfg.intermediate_size = 1024 +model.cfg.hidden_dropout_prob = 0.0 +model.cfg.attention_probs_dropout_prob = 0.0 +model.cfg.embedding_dropout_prob = 0.0 +model.cfg.layernorm_eps = 1e-6 +model.cfg.model_type = "mt5" +model.cfg.pretrained_model_path = pretrained_model_path + +train.update( + dict( + output_dir="projects/T5/output/mt5_output", + train_micro_batch_size=micro_batch_size, + train_epoch=1, + train_iter=24000, + log_period=10, + amp=dict(enabled=False), + warmup_ratio=1 / 24, + # checkpointer=dict(period=10, max_to_keep=20), + dist=dict( + data_parallel_size=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + pipeline_num_layers=2 * model.cfg.hidden_layers, + ), + scheduler=LazyCall(WarmupExponentialLR)( + warmup_factor=0.001, + gamma=1.0, + warmup_method="linear", + warmup_iter=0.0, + ), + evaluation=dict( + evaluator=LazyCall(PPLEvaluator)(), + enabled=True, + eval_iter=1e5, + eval_period=5000, + ), + ) +) + +train.zero_optimization.enabled = True +train.zero_optimization.stage = 2 diff --git a/projects/T5/configs/optim.py b/projects/T5/configs/optim.py new file mode 100644 index 0000000000000000000000000000000000000000..7cd5f2cf05ef469aa80cf64715e4940736613731 --- /dev/null +++ b/projects/T5/configs/optim.py @@ -0,0 +1,20 @@ +import oneflow as flow + +from libai.optim import get_default_optimizer_params +from libai.config import LazyCall + +optim = LazyCall(flow.optim.AdamW)( + params=LazyCall(get_default_optimizer_params)( + # params.model is meant to be set to the model object, + # before instantiating the optimizer. + clip_grad_max_norm=1.0, + clip_grad_norm_type=2.0, + weight_decay_norm=0.0, + weight_decay_bias=0.0, + ), + lr=1e-4, + weight_decay=0.01, + betas=(0.9, 0.999), + eps=1e-8, + do_bias_correction=True, +) diff --git a/projects/T5/configs/t5_model_config.py b/projects/T5/configs/t5_model_config.py new file mode 100644 index 0000000000000000000000000000000000000000..50523f75696647f5d19c1821208a8271c3ea96d1 --- /dev/null +++ b/projects/T5/configs/t5_model_config.py @@ -0,0 +1,21 @@ +from omegaconf import DictConfig + + +cfg = dict( + vocab_size=30522, + hidden_size=768, + hidden_layers=6, + num_attention_heads=12, + head_size=64, + intermediate_size=1536, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + relative_attention_num_buckets=32, + embedding_dropout_prob=0.1, + initializer_range=0.02, + layernorm_eps=1e-5, + amp_enabled=False, + model_type="t5", +) + +cfg = DictConfig(cfg) diff --git a/projects/T5/datasets/dataset.py b/projects/T5/datasets/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..bb426e59834e6c1c423cb65db977904653ccd02c --- /dev/null +++ b/projects/T5/datasets/dataset.py @@ -0,0 +1,252 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings + +try: + import datasets +except: # noqa + warnings.warn("datasets library is needed") +import numpy as np +import oneflow as flow +from oneflow.utils.data import Dataset + +from libai.data.structures import DistTensorData, Instance + + +def get_data(path): + total_data = [] + for i in range(10): + path = path[:-1] + str(i) + dataset = datasets.load_from_disk(path) + for i in dataset: + total_data.append(i) + return total_data + + +def compute_input_and_target_lengths(inputs_length, noise_density, mean_noise_span_length): + """This function is copy of `random_spans_helper `__ . + Training parameters to avoid padding with random_spans_noise_mask. + When training a model with random_spans_noise_mask, we would like to set the other + training hyperparmeters in a way that avoids padding. + This function helps us compute these hyperparameters. + We assume that each noise span in the input is replaced by extra_tokens_per_span_inputs + sentinel tokens, and each non-noise span in the targets is replaced by + extra_tokens_per_span_targets sentinel tokens. + This function tells us the required number of tokens in the raw example (for split_tokens()) + as well as the length of the encoded targets. Note that this function assumes + the inputs and targets will have EOS appended and includes that in the reported length. + Args: + inputs_length: an integer - desired length of the tokenized inputs sequence + noise_density: a float + mean_noise_span_length: a float + Returns: + tokens_length: length of original text in tokens + targets_length: an integer - length in tokens of encoded targets sequence + """ + + def _tokens_length_to_inputs_length_targets_length(tokens_length): + num_noise_tokens = int(round(tokens_length * noise_density)) + num_nonnoise_tokens = tokens_length - num_noise_tokens + num_noise_spans = int(round(num_noise_tokens / mean_noise_span_length)) + # inputs contain all nonnoise tokens, sentinels for all noise spans + # and one EOS token. + _input_length = num_nonnoise_tokens + num_noise_spans + 1 + _output_length = num_noise_tokens + num_noise_spans + 1 + return _input_length, _output_length + + tokens_length = inputs_length + + while _tokens_length_to_inputs_length_targets_length(tokens_length + 1)[0] <= inputs_length: + tokens_length += 1 + + inputs_length, targets_length = _tokens_length_to_inputs_length_targets_length(tokens_length) + + # minor hack to get the targets length to be equal to inputs length + # which is more likely to have been set to a nice round number. + if noise_density == 0.5 and targets_length > inputs_length: + tokens_length -= 1 + targets_length -= 1 + return tokens_length, targets_length + + +class UnsuperviseT5Dataset(Dataset): + """This function is copy of https://github.com/IDEA-CCNL/Fengshenbang-LM/blob/ + ec13aeb8689cfafaa6a7a9e9595d110edbe34123/fengshen/data/t5_dataloader/t5_datasets.py#L61. + """ + + def __init__(self, data_path): + # [{input_ids: ...}, {input_ids: ...}, ...] + self.data = get_data(data_path) + + def __len__(self): + return len(self.data) + + def __getitem__(self, index): + x = self.data[index] + return x + + +class collate_fn: + def __init__( + self, + vocab_size, + max_seq_length, + noise_density, + mean_noise_span_length, + eos_token_id=1, + pad_token_id=0, + decoder_start_token_id=0, + ): + self.max_seq_length = max_seq_length + self.eos_token_id = eos_token_id + self.pad_token_id = pad_token_id + self.decoder_start_token_id = decoder_start_token_id + self.vocab_size = vocab_size + self.noise_density = noise_density + self.mean_noise_span_length = mean_noise_span_length + self.expanded_inputs_length, self.targets_length = compute_input_and_target_lengths( + inputs_length=self.max_seq_length, + noise_density=self.noise_density, + mean_noise_span_length=self.mean_noise_span_length, + ) + + def __call__(self, examples): + batch = { + k: np.array([examples[i][k] for i in range(len(examples))]) + for k, v in examples[0].items() + } + input_ids = np.array(batch["input_ids"]) + batch_size, expanded_input_length = input_ids.shape + mask_indices = np.asarray( + [self.random_spans_noise_mask(expanded_input_length) for i in range(batch_size)] + ) + labels_mask = ~mask_indices + + input_ids_sentinel = self.create_sentinel_ids(mask_indices.astype(np.int8)) + labels_sentinel = self.create_sentinel_ids(labels_mask.astype(np.int8)) + + batch["input_ids"] = self.filter_input_ids(input_ids, input_ids_sentinel) + batch["labels"] = self.filter_input_ids(input_ids, labels_sentinel) + + if batch["input_ids"].shape[-1] != self.max_seq_length: + raise ValueError( + f"`input_ids` are incorrectly preprocessed. `input_ids` length is \ + {batch['input_ids'].shape[-1]}, but should be {self.targets_length}." + ) + + if batch["labels"].shape[-1] != self.targets_length: + raise ValueError( + f"`labels` are incorrectly preprocessed. `labels` length is \ + {batch['labels'].shape[-1]}, but should be {self.targets_length}." + ) + + batch["decoder_input_ids"] = self.shift_tokens_right( + batch["labels"], self.pad_token_id, self.decoder_start_token_id + ) + + return Instance( + encoder_input_ids=DistTensorData(flow.tensor(batch["input_ids"])), + decoder_input_ids=DistTensorData(flow.tensor(batch["decoder_input_ids"])), + encoder_attn_mask=DistTensorData( + flow.ones(len(batch["input_ids"]), len(batch["input_ids"][0])).to(flow.bool) + ), + decoder_attn_mask=DistTensorData( + flow.ones(len(batch["decoder_input_ids"]), len(batch["decoder_input_ids"][0])).to( + flow.bool + ) + ), + encoder_decoder_attn_mask=DistTensorData( + flow.ones( + len(batch["input_ids"]), + len(batch["decoder_input_ids"][0]), + len(batch["input_ids"][0]), + ).to(flow.bool) + ), + lm_labels=DistTensorData(flow.tensor(batch["labels"])), + loss_mask=DistTensorData(flow.tensor(batch["labels"])), + ) + + def filter_input_ids(self, input_ids, sentinel_ids): + batch_size = input_ids.shape[0] + + input_ids_full = np.where(sentinel_ids != 0, sentinel_ids, input_ids) + input_ids = input_ids_full[input_ids_full >= 0].reshape((batch_size, -1)) + input_ids = np.concatenate( + [input_ids, np.full((batch_size, 1), self.eos_token_id, dtype=np.int32)], axis=-1 + ) + return input_ids + + def create_sentinel_ids(self, mask_indices): + start_indices = mask_indices - np.roll(mask_indices, 1, axis=-1) * mask_indices + start_indices[:, 0] = mask_indices[:, 0] + + sentinel_ids = np.where( + start_indices != 0, np.cumsum(start_indices, axis=-1), start_indices + ) + sentinel_ids = np.where(sentinel_ids != 0, (self.vocab_size - sentinel_ids), 0) + sentinel_ids -= mask_indices - start_indices + + return sentinel_ids + + def random_spans_noise_mask(self, length): + orig_length = length + num_noise_tokens = int(np.round(length * self.noise_density)) + # avoid degeneracy by ensuring positive numbers of noise and nonnoise tokens. + num_noise_tokens = min(max(num_noise_tokens, 1), length - 1) + num_noise_spans = int(np.round(num_noise_tokens / self.mean_noise_span_length)) + + # avoid degeneracy by ensuring positive number of noise spans + num_noise_spans = max(num_noise_spans, 1) + num_nonnoise_tokens = length - num_noise_tokens + + # pick the lengths of the noise spans and the non-noise spans + def _random_segmentation(num_items, num_segments): + mask_indices = np.arange(num_items - 1) < (num_segments - 1) + np.random.shuffle(mask_indices) + first_in_segment = np.pad(mask_indices, [[1, 0]]) + segment_id = np.cumsum(first_in_segment) + # count length of sub segments assuming that list is sorted + _, segment_length = np.unique(segment_id, return_counts=True) + return segment_length + + noise_span_lengths = _random_segmentation(num_noise_tokens, num_noise_spans) + nonnoise_span_lengths = _random_segmentation(num_nonnoise_tokens, num_noise_spans) + + interleaved_span_lengths = np.reshape( + np.stack([nonnoise_span_lengths, noise_span_lengths], axis=1), [num_noise_spans * 2] + ) + span_starts = np.cumsum(interleaved_span_lengths)[:-1] + span_start_indicator = np.zeros((length,), dtype=np.int8) + span_start_indicator[span_starts] = True + span_num = np.cumsum(span_start_indicator) + is_noise = np.equal(span_num % 2, 1) + + return is_noise[:orig_length] + + def shift_tokens_right( + self, input_ids: np.array, pad_token_id: int, decoder_start_token_id: int + ) -> np.ndarray: + """ + Shift input ids one token to the right. + """ + shifted_input_ids = np.zeros_like(input_ids) + shifted_input_ids[:, 1:] = input_ids[:, :-1] + shifted_input_ids[:, 0] = decoder_start_token_id + + shifted_input_ids = np.where(shifted_input_ids == -100, pad_token_id, shifted_input_ids) + return shifted_input_ids diff --git a/projects/T5/models/attention.py b/projects/T5/models/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..a825f681a0112e39d3543fd845e20aeb4653b25a --- /dev/null +++ b/projects/T5/models/attention.py @@ -0,0 +1,350 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import Tuple + +import oneflow as flow +from oneflow import nn + +from libai.layers.linear import Linear +from libai.utils import distributed as dist +from projects.T5.models.embedding import Embedding + + +class MultiheadAttention(nn.Module): + """Multi-head attention layer, support self attention and cross attention. + + Args: + hidden_size: size of hidden state. + num_attention_heads: number of attention heads. + is_cross_attention: used to specify whether it is self attention or cross attention. + Defaults to False. + attention_dropout_prob: dropout probability of attention weights. + Defaults to 0.0. + output_dropout_prob: dropout probability of output. Defaults to 0.0. + init_method: method to initialize the input layer weights. + Defaults to ``init.xavier_normal_``. + output_layer_init_method: method to initialize the output layer weights. + If None, use ``init_method``. + layer_idx: a layer_idx sign which determines the placements. + It will be used in pipeline parallelism. Defaults to 0. + """ + + def __init__( + self, + hidden_size, + num_attention_heads, + head_size, + relative_attention_num_buckets, + is_cross_attention=False, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + *, + layer_idx=0, + has_relative_attention_bias=False, + is_decoder=False, + ): + super().__init__() + self.hidden_size = hidden_size + self.relative_attention_num_buckets = relative_attention_num_buckets + self.has_relative_attention_bias = has_relative_attention_bias + self.is_decoder = is_decoder + self.attention_dropout_prob = attention_dropout_prob + + if output_layer_init_method is None: + output_layer_init_method = init_method + self.num_heads = num_attention_heads + self.head_size = head_size + + self.dropout = nn.Dropout(p=attention_dropout_prob) + self.norm_factor = 1.0 / math.sqrt(float(self.head_size)) + + self.is_cross_attention = is_cross_attention + + self.output_dropout = nn.Dropout(p=output_dropout_prob) + + if self.is_cross_attention: + self.query = Linear( + self.hidden_size, + self.num_heads * self.head_size, + bias=False, + parallel="col", + init_method=init_method, + layer_idx=layer_idx, + ) + self.key_value = Linear( + self.hidden_size, + self.num_heads * self.head_size * 2, + bias=False, + parallel="col", + init_method=init_method, + layer_idx=layer_idx, + ) + else: + self.query_key_value = Linear( + self.hidden_size, + self.num_heads * self.head_size * 3, + bias=False, + parallel="col", + init_method=init_method, + layer_idx=layer_idx, + ) + + self.dense = Linear( + self.num_heads * self.head_size, + self.hidden_size, + bias=False, + parallel="row", + init_method=output_layer_init_method, + skip_bias_add=False, + layer_idx=layer_idx, + ) + if self.has_relative_attention_bias: + self.relative_attention_bias = Embedding( + self.relative_attention_num_buckets, self.num_heads, layer_idx=layer_idx + ) + + def forward( + self, + hidden_states: flow.Tensor, + encoder_states: flow.Tensor = None, + attention_mask: flow.Tensor = None, + past_key_value: Tuple[flow.Tensor, flow.Tensor] = None, + use_cache: bool = False, + position_bias=None, + query_length=None, + ): + """ + + Args: + hidden_states (flow.Tensor): shape is [bsz, tgt_len, hidden_size]. + encoder_states (flow.Tensor, optional): shape is [bsz, src_len, hidden_size]. + Defaults to None. + attention_mask (flow.Tensor, optional): shape is [bsz, 1, tgt_len, src_len]. + It should be the combination of padding mask and casual mask. + It is the padding mask of source input when used with self-attention in encoder. + And it is the combination of padding mask of target input and casual mask when + used with self-attention in decoder. It is the padding mask of source input when + used with cross-attention in decoder. + Defaults to None. + past_key_value (Tuple[flow.Tensor, flow.Tensor], optional): tuple of key and value, + each shape is [bsz, num_heads, src_len, head_size]. Defaults to None. + use_cache (bool, optional): it will be set to True, when the model is in the inference + phase and used for incremental decoding. Defaults to False. + """ + + # hidden_states, encoder_states: [S(0), B] + # attention_mask: [S(0), B] + + if encoder_states is not None: + encoder_states = encoder_states.to_global(placement=hidden_states.placement) + + if attention_mask is not None: + attention_mask = attention_mask.to_global(placement=hidden_states.placement) + + bsz, real_seq_length = hidden_states.size()[:2] + + if past_key_value is not None: + assert ( + len(past_key_value) == 2 + ), "past_key_value should have 2 past states: keys and values." + f"Got {len(past_key_value)} past states.\n" + real_seq_length += past_key_value[0].shape[2] if query_length is None else query_length + + key_length = real_seq_length if encoder_states is None else encoder_states.shape[1] + + if self.is_cross_attention: + # if it is cross attention, key and value should be calculated only once, and the + # result can be reused. + query = self.query(hidden_states) + query = query.view(bsz, -1, self.num_heads, self.head_size) + query = query.permute(0, 2, 1, 3) + if past_key_value is not None: + key, value = past_key_value + elif encoder_states is not None: + key_value = self.key_value(encoder_states) + key_value = key_value.view(bsz, -1, self.num_heads, 2 * self.head_size) + key_value = key_value.permute(0, 2, 1, 3) + key, value = flow.chunk(key_value, chunks=2, dim=-1) + else: + raise ValueError( + "past_key_value and encoder_states cannot be None at the same time." + ) + else: + # if it is self attention, query, key, and value are all obtained from hidden_states. + # when in the inference phase of an incremental decoder, + # hidden_states is the last-added state, + # the full key and value could be obtained by concatenating with past_key_value. + query_key_value = self.query_key_value(hidden_states) + query_key_value = query_key_value.view(bsz, -1, self.num_heads, 3 * self.head_size) + query_key_value = query_key_value.permute( + 0, 2, 1, 3 + ) # [bsz, num_heads, src_len, 3 * head_size] + query, key, value = flow.chunk(query_key_value, chunks=3, dim=-1) + if past_key_value is not None: + past_key, past_value = past_key_value + key = flow.cat((past_key.type_as(key), key), dim=2) + value = flow.cat((past_value.type_as(value), value), dim=2) + + # query, key, value: [S(0), S(1)], shape: [bsz, num_heads, seq_length, head_size] + if use_cache: + past_key_value = (key, value) + + # [bsz, num_heads, tgt_len, src_len] with [S(0), S(1)] + attention_scores = flow.matmul(query, key, transpose_b=True) + + if position_bias is None: + if not self.has_relative_attention_bias: + position_bias = flow.zeros( + (1, self.num_heads, real_seq_length, key_length), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=attention_scores.placement, + ) + else: + position_bias = self.compute_bias( + real_seq_length, key_length, placement=attention_mask.placement + ) + + if past_key_value is not None: + position_bias = position_bias[:, :, -hidden_states.size(1) :, :] + + position_bias = position_bias + (1 - attention_mask) * -1000 + position_bias = position_bias.to_global(placement=attention_scores.placement) + + attention_scores = attention_scores + position_bias + + # [S(0), S(1)] x [S(0), B] = [S(0), S(1)] + if attention_mask is not None: + attention_scores = flow.mul(attention_scores, attention_mask) + attention_scores = attention_scores - 10000.0 * (1 - attention_mask) + # TODO(xingyu.liao): graph will occur `where_scalar` errors + # when using `masked_fill` + # attention_scores = attention_scores.masked_fill(1 - attention_mask, -10000.0) + attention_weights = flow.softmax(attention_scores, dim=-1) + # [bsz, num_heads, tgt_len, src_len] + attention_weights = self.dropout(attention_weights) + else: + attention_weights = flow.softmax(attention_scores, dim=-1) + # [bsz, num_heads, tgt_len, src_len] + attention_weights = self.dropout(attention_weights) + + # Context shape: [bsz, num_heads, tgt_len, head_size] with [S(0), S(1)] + context = flow.matmul(attention_weights, value) + # Change shape: [bsz, num_heads, tgt_len, head_size] -> [bsz, tgt_len, num_heads, head_size] + context = context.transpose(1, 2) + + # Concat multi-head results from + # [bsz, tgt_len, num_heads, head_size] -> [bsz, tgt_len, num_heads * head_size] + # SBP sign: [S(0), S(2)] + # [S(0), S(2)] x [B, S(0)] = [S(0), P] -> [S(0), B] + output = self.dense(context.flatten(2)) + + output = self.output_dropout(output) + + if use_cache: + output = (output, past_key_value) + + output = (output,) + (position_bias,) + return output + + def extra_repr(self) -> str: + return "hidden_size={}, num_heads={}, is_cross_attention={}".format( + self.hidden_size, + self.num_heads, + self.is_cross_attention, + ) + + def _relative_position_bucket( + self, relative_position, bidirectional=True, num_buckets=32, max_distance=128 + ): + # relative_position: (seq_len, seq_len) + relative_buckets = 0 + if bidirectional: + num_buckets //= 2 + relative_buckets = ( + relative_buckets + (relative_position > 0).to(flow.long) * num_buckets + ) + relative_position = flow.abs(relative_position) + else: + relative_position = ( + -1 + * flow.min( + relative_position, + flow.zeros( + relative_position.size(), + sbp=relative_position.sbp, + placement=relative_position.placement, + ), + ).to(flow.long) + ) + + max_exact = num_buckets // 2 + is_small = relative_position < max_exact + + relative_postion_if_large = max_exact + ( + flow.log(relative_position.float() / max_exact) + / math.log(max_distance / max_exact) + * (num_buckets - max_exact) + ).to(flow.long) + + relative_postion_if_large = flow.min( + relative_postion_if_large, + flow.zeros( + relative_postion_if_large.size(), + dtype=relative_postion_if_large.dtype, + sbp=relative_postion_if_large.sbp, + placement=relative_postion_if_large.placement, + ).fill_(num_buckets - 1), + ) + + relative_buckets = relative_buckets + flow.where( + is_small, relative_position, relative_postion_if_large + ) + return relative_buckets + + def compute_bias(self, query_length, key_length, placement=None): + """Compute binned relative position bias""" + context_position = flow.arange( + query_length, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=placement, + ) + memory_position = flow.arange( + key_length, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=placement, + ) + relative_position = ( + memory_position[None, :] - context_position[:, None] + ) # shape (query_length, key_length) + + relative_position_bucket = self._relative_position_bucket( + relative_position, + bidirectional=(not self.is_decoder), + num_buckets=self.relative_attention_num_buckets, + ) # shape (query_length, key_length) + + values = self.relative_attention_bias( + relative_position_bucket + ) # shape (query_length, key_length, num_heads) + values = values.permute([2, 0, 1]).unsqueeze( + 0 + ) # shape (1, num_heads, query_length, key_length) + return values diff --git a/projects/T5/models/embedding.py b/projects/T5/models/embedding.py new file mode 100644 index 0000000000000000000000000000000000000000..3d51a16b0a3dc3c6ab2efd86824997eb904a7a8f --- /dev/null +++ b/projects/T5/models/embedding.py @@ -0,0 +1,125 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +import oneflow.nn as nn +from oneflow.nn import init + +import libai.utils.distributed as dist +from libai.layers.embedding import VocabEmbedding + + +class T5Embedding(flow.nn.Module): + def __init__( + self, + hidden_size, + vocab_size, + embedding_dropout_prob, + init_method=flow.nn.init.xavier_normal_, + amp_enabled=False, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + self.vocab_size = vocab_size + + self.word_embeddings = VocabEmbedding( + num_embeddings=vocab_size, + embedding_dim=hidden_size, + init_method=init_method, + amp_enabled=amp_enabled, + ) + + self.embedding_dropout = flow.nn.Dropout(embedding_dropout_prob) + + def forward(self, input_ids): + word_embeddings = self.word_embeddings(input_ids) + embeddings = self.embedding_dropout(word_embeddings) + return embeddings + + +class Embedding(nn.Module): + """Construct the trainable embedding module, which does not support parallelization. + This can be used for positional embedding and token type embedding. + + Arguments: + num_embeddings: size of vocabulary. + embedding_dim: dimension of embeddings. + padding_idx: pad index. Defaults to None. + init_method: method to initialize weights. Defaults to ``flow.nn.init.xavier_normal_``. + amp_enabled: fp16 option for embedding weight. Defaults to False. + """ + + def __init__( + self, + num_embeddings, + embedding_dim, + padding_idx=None, + init_method=init.xavier_normal_, + amp_enabled=False, + layer_idx=0, + ): + super().__init__() + self.num_embeddings = num_embeddings + self.embedding_dim = embedding_dim + if padding_idx is not None: + if padding_idx > 0: + assert ( + padding_idx < self.num_embeddings + ), "Padding_idx must be within num_embeddings" + elif padding_idx < 0: + assert ( + padding_idx >= -self.num_embeddings + ), "Padding_idx must be within num_embeddings" + padding_idx = self.num_embeddings + padding_idx + self.padding_idx = padding_idx + self.init_method = init_method + self.amp_enabled = amp_enabled + + assert num_embeddings > 0 + self.weight = nn.Parameter( + flow.empty( + (num_embeddings, embedding_dim), + dtype=flow.float32, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + ) + self.init_method(self.weight) + # FIXME(lxy): Fill padding_idx is not supported in nd_sbp right now. + # self._fill_padding_idx_with_zero() + + def forward(self, input_ids): + weight = flow._C.amp_white_identity(self.weight) if self.amp_enabled else self.weight + # embeddings with sbp sign: [B, B] + # [B, B] x [S(0), B] --> [S(0), B] + # ↑ ↑ ↑ + # embed pos_ids pos_embed + input_embeds = flow._C.gather(weight, input_ids, axis=0) + return input_embeds + + def _fill_padding_idx_with_zero(self) -> None: + if self.padding_idx is not None: + with flow.no_grad(): + self.weight[self.padding_idx] = flow.zeros( + self.embedding_dim, + placement=dist.get_layer_placement(0), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + + def extra_repr(self) -> str: + s = "num_embeddings={num_embeddings}, embedding_dim={embedding_dim}" + if self.padding_idx is not None: + s += ", padding_idx={padding_idx}" + return s.format(**self.__dict__) diff --git a/projects/T5/models/layer_norm.py b/projects/T5/models/layer_norm.py new file mode 100644 index 0000000000000000000000000000000000000000..b7e9864a5f23b68a67e2236b56a991212e6572cf --- /dev/null +++ b/projects/T5/models/layer_norm.py @@ -0,0 +1,36 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow + +from libai.utils import distributed as dist + + +class LayerNorm(flow.nn.Module): + def __init__(self, normalized_shape, eps=1e-6, layer_idx=0): + super().__init__() + self.layer_idx = layer_idx + self.weight = flow.nn.Parameter( + flow.ones( + normalized_shape, + dtype=flow.float32, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + ) + ) + self.l2norm_epsilon = eps + + def forward(self, hidden_states): + return flow._C.rms_layer_norm(hidden_states, self.weight, self.l2norm_epsilon) diff --git a/projects/T5/models/logits.py b/projects/T5/models/logits.py new file mode 100644 index 0000000000000000000000000000000000000000..27fd2ae018484b254ea216bd18625de142ceb562 --- /dev/null +++ b/projects/T5/models/logits.py @@ -0,0 +1,52 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +from oneflow import nn + +from libai.layers import Linear +from libai.utils import distributed as dist + + +class LMLogits(nn.Module): + def __init__(self, vocab_size, hidden_size=None, bias=False, model_type="t5", layer_idx=-1): + super().__init__() + self.model_type = model_type + if model_type == "t5": + self.bias = ( + nn.Parameter( + flow.zeros( + (vocab_size,), + dtype=flow.float32, + placement=dist.get_layer_placement(layer_idx), + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.split(0)]), + ) + ) + if bias + else None + ) + elif model_type == "mt5": + self.linear = Linear(hidden_size, vocab_size, bias=False, layer_idx=layer_idx) + + def forward(self, input, word_embeddings=None): + if self.model_type == "t5": + w = word_embeddings.to_global(placement=input.placement) + input = input.to_global(grad_sbp=input.sbp) + logits = flow._C.matmul(input, w, transpose_b=True) + if self.bias is not None: + logits = logits + self.bias + else: + logits = self.linear(input) + return logits diff --git a/projects/T5/models/mlp.py b/projects/T5/models/mlp.py new file mode 100644 index 0000000000000000000000000000000000000000..3a69d5816eabf61b1353e5567057ad0514dd443a --- /dev/null +++ b/projects/T5/models/mlp.py @@ -0,0 +1,128 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oneflow import nn + +from libai.layers import Linear, build_activation + + +class T5MLP(nn.Module): + def __init__( + self, + hidden_size, + ffn_hidden_size, + output_dropout_prob=0.0, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + *, + layer_idx=0, + ): + super().__init__() + self.output_dropout_prob = output_dropout_prob + + if output_layer_init_method is None: + output_layer_init_method = init_method + + self.dense_h_to_4h = Linear( + hidden_size, + ffn_hidden_size, + bias=False, + parallel="col", + skip_bias_add=False, + init_method=init_method, + layer_idx=layer_idx, + ) + + self.activation_func = build_activation("relu") + + self.dense_4h_to_h = Linear( + ffn_hidden_size, + hidden_size, + bias=False, + parallel="row", + skip_bias_add=False, + init_method=output_layer_init_method, + layer_idx=layer_idx, + ) + + self.dropout = nn.Dropout(self.output_dropout_prob) + + def forward(self, hidden_states): + intermediate = self.dense_h_to_4h(hidden_states) + intermediate = self.activation_func(intermediate) + output = self.dense_4h_to_h(intermediate) + output = self.dropout(output) + return output + + +class MT5MLP(nn.Module): + def __init__( + self, + hidden_size, + ffn_hidden_size, + output_dropout_prob=0.0, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + *, + layer_idx=0, + ): + super().__init__() + self.output_dropout_prob = output_dropout_prob + + if output_layer_init_method is None: + output_layer_init_method = init_method + + self.wi_0 = Linear( + hidden_size, + ffn_hidden_size, + bias=False, + parallel="col", + skip_bias_add=False, + init_method=init_method, + layer_idx=layer_idx, + ) + + self.wi_1 = Linear( + hidden_size, + ffn_hidden_size, + bias=False, + parallel="col", + skip_bias_add=False, + init_method=init_method, + layer_idx=layer_idx, + ) + + self.activation_func = build_activation("gelu_tanh") + + self.wo = Linear( + ffn_hidden_size, + hidden_size, + bias=False, + parallel="row", + skip_bias_add=False, + init_method=output_layer_init_method, + layer_idx=layer_idx, + ) + + self.dropout = nn.Dropout(self.output_dropout_prob) + + def forward(self, hidden_states): + wi_0_out = self.wi_0(hidden_states) + hidden_gelu = self.activation_func(wi_0_out) + hidden_linear = self.wi_1(hidden_states) + hidden_states = hidden_gelu * hidden_linear + output = self.wo(hidden_states) + output = self.dropout(output) + return output diff --git a/projects/T5/models/t5_model.py b/projects/T5/models/t5_model.py new file mode 100644 index 0000000000000000000000000000000000000000..2551d6c54903629fcf264843516977be4684c2b9 --- /dev/null +++ b/projects/T5/models/t5_model.py @@ -0,0 +1,344 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +import oneflow.nn as nn + +from libai.config import configurable +from libai.layers import Linear, LMLogits +from libai.models.t5_model import T5Loss +from libai.models.utils import init_method_normal, scaled_init_method_normal +from libai.utils import distributed as dist +from projects.MT5.utils.mt5_loader import T5LoaderHuggerFace +from projects.T5.models.embedding import T5Embedding +from projects.T5.models.layer_norm import LayerNorm +from projects.T5.models.transformer_layer import TransformerLayer +from projects.T5.utils.mask import ExtendedMask + + +class T5Model(flow.nn.Module): + @configurable + def __init__( + self, + vocab_size, + hidden_size, + hidden_layers, + num_attention_heads, + head_size, + intermediate_size, + embedding_dropout_prob, + hidden_dropout_prob, + attention_probs_dropout_prob, + relative_attention_num_buckets, + initializer_range=0.02, + layernorm_eps=1e-12, + amp_enabled=False, + model_type="t5", + ) -> None: + super().__init__() + self.model_type = model_type + init_method = init_method_normal(initializer_range) + scaled_init_method = scaled_init_method_normal(initializer_range, hidden_layers) + self.embedding = T5Embedding( + hidden_size=hidden_size, + vocab_size=vocab_size, + embedding_dropout_prob=embedding_dropout_prob, + init_method=init_method, + amp_enabled=amp_enabled, + ) + self.extended_attn_mask = ExtendedMask() + + encoder_layers = flow.nn.ModuleList( + [ + TransformerLayer( + hidden_size=hidden_size, + ffn_hidden_size=intermediate_size, + num_attention_heads=num_attention_heads, + head_size=head_size, + relative_attention_num_buckets=relative_attention_num_buckets, + is_decoder=False, + attention_dropout_prob=attention_probs_dropout_prob, + output_dropout_prob=hidden_dropout_prob, + layernorm_epsilon=layernorm_eps, + init_method=init_method, + output_layer_init_method=scaled_init_method, + layer_idx=i, + model_type=model_type, + has_relative_attention_bias=bool(i == 0), + ) + for i in range(hidden_layers) + ] + ) + + encoder_final_layernorm = LayerNorm( + (hidden_size,), + eps=layernorm_eps, + layer_idx=hidden_layers - 1, + ) + + self.encoder = flow.nn.Sequential() + self.encoder.add_module("layers", encoder_layers) + self.encoder.add_module("final_layernorm", encoder_final_layernorm) + + decoder_layers = flow.nn.ModuleList( + [ + TransformerLayer( + hidden_size=hidden_size, + ffn_hidden_size=intermediate_size, + num_attention_heads=num_attention_heads, + head_size=head_size, + relative_attention_num_buckets=relative_attention_num_buckets, + is_decoder=True, + attention_dropout_prob=attention_probs_dropout_prob, + output_dropout_prob=hidden_dropout_prob, + layernorm_epsilon=layernorm_eps, + init_method=init_method, + output_layer_init_method=scaled_init_method, + layer_idx=i, + model_type=model_type, + has_relative_attention_bias=bool(i - hidden_layers == 0), + ) + for i in range(hidden_layers, 2 * hidden_layers) + ] + ) + + decoder_final_layernorm = LayerNorm( + (hidden_size,), + eps=layernorm_eps, + layer_idx=2 * hidden_layers - 1, + ) + + self.decoder = flow.nn.Sequential() + self.decoder.add_module("layers", decoder_layers) + self.decoder.add_module("final_layernorm", decoder_final_layernorm) + self.past_key_values = [None] * len(self.decoder.layers) + self.encoder_states = None + self.past_length = 0 + + if model_type == "mt5": + self.lm_head = Linear( + hidden_size, vocab_size, bias=False, layer_idx=2 * hidden_layers - 1 + ) + else: + self.lm_head = LMLogits(vocab_size, bias=False) + + @classmethod + def from_config(cls, cfg): + return { + "vocab_size": cfg.vocab_size, + "hidden_size": cfg.hidden_size, + "hidden_layers": cfg.hidden_layers, + "num_attention_heads": cfg.num_attention_heads, + "head_size": cfg.head_size, + "intermediate_size": cfg.intermediate_size, + "embedding_dropout_prob": cfg.embedding_dropout_prob, + "hidden_dropout_prob": cfg.hidden_dropout_prob, + "attention_probs_dropout_prob": cfg.attention_probs_dropout_prob, + "relative_attention_num_buckets": cfg.relative_attention_num_buckets, + "initializer_range": cfg.initializer_range, + "layernorm_eps": cfg.layernorm_eps, + "amp_enabled": cfg.amp_enabled, + "model_type": cfg.model_type, + } + + def forward( + self, + encoder_input_ids, + decoder_input_ids, + encoder_attn_mask, + decoder_attn_mask, + encoder_decoder_attn_mask, + use_cache=False, + ): + encoder_input_ids = encoder_input_ids.to_global(placement=dist.get_layer_placement(0)) + decoder_input_ids = decoder_input_ids.to_global(placement=dist.get_layer_placement(0)) + encoder_attn_mask = encoder_attn_mask.to_global(placement=dist.get_layer_placement(0)) + decoder_attn_mask = decoder_attn_mask.to_global(placement=dist.get_layer_placement(0)) + encoder_decoder_attn_mask = encoder_decoder_attn_mask.to_global( + placement=dist.get_layer_placement(0) + ) + + if use_cache and self.encoder_states is not None: + encoder_states = self.encoder_states + else: + position_bias = None + encoder_decoder_position_bias = None + self.set_cache(encoder_states=None, past_key_values=None) + encoder_attn_mask = self.extended_attn_mask(encoder_attn_mask) + enc_embedding_output = self.embedding(encoder_input_ids) + enc_hidden_states = enc_embedding_output + + for layer in self.encoder.layers: + enc_hidden_states, position_bias = layer( + enc_hidden_states, + encoder_attn_mask, + position_bias=position_bias, + ) + encoder_states = self.encoder.final_layernorm(enc_hidden_states) + + decoder_attn_mask = self.extended_attn_mask( + decoder_attn_mask, decoder_input_ids, is_decoder=True + ) + encoder_decoder_attn_mask = self.extended_attn_mask(encoder_decoder_attn_mask) + + dec_embedding_output = self.embedding(decoder_input_ids) + dec_hidden_states = dec_embedding_output + if use_cache: + presents = [] + + position_bias = None + encoder_decoder_position_bias = None + for layer, past_key_value in zip(self.decoder.layers, self.past_key_values): + dec_hidden_states, position_bias, encoder_decoder_position_bias = layer( + dec_hidden_states, + decoder_attn_mask, + encoder_states, + encoder_decoder_attn_mask, + past_key_value=past_key_value, + position_bias=position_bias, + encoder_decoder_position_bias=encoder_decoder_position_bias, + use_cache=use_cache, + ) + if use_cache: + dec_hidden_states, present = dec_hidden_states + presents.append(present) + if use_cache: + self.set_cache(encoder_states, past_key_values=presents) + + decoder_states = self.decoder.final_layernorm(dec_hidden_states) + + if self.model_type == "mt5": + logits = self.lm_head(decoder_states) + else: + logits = self.lm_head(decoder_states, self.embedding.word_embeddings.weight) + + return logits + + def set_cache(self, encoder_states, past_key_values): + self.encoder_states = encoder_states + self.past_length = 0 if past_key_values is None else past_key_values[0][0].shape[2] + + if past_key_values is None: + past_key_values = [None] * len(self.decoder.layers) + assert len(past_key_values) == len(self.decoder.layers), ( + f"past_key_values's length {len(past_key_values)} doesn't match " + f"decoder num_layers' length {self.decoder.layers}" + ) + self.past_key_values = past_key_values + + +class T5ForPreTraining(flow.nn.Module): + def __init__(self, cfg) -> None: + super().__init__() + if cfg.pretrained_model_path is not None: + loader = T5LoaderHuggerFace(T5Model, cfg, cfg.pretrained_model_path) + self.t5_model = loader.load() + else: + self.t5_model = T5Model(cfg) + self.loss_func = T5Loss() + + def set_cache(self, encoder_states, past_key_values): + self.t5_model.set_cache(encoder_states, past_key_values) + + def forward( + self, + encoder_input_ids, + decoder_input_ids, + encoder_attn_mask, + decoder_attn_mask, + encoder_decoder_attn_mask, + lm_labels=None, + loss_mask=None, + use_cache=False, + ): + logits = self.t5_model( + encoder_input_ids, + decoder_input_ids, + encoder_attn_mask, + decoder_attn_mask, + encoder_decoder_attn_mask, + use_cache=use_cache, + ) + + if lm_labels is not None: + lm_loss = self.loss_func(logits, lm_labels, loss_mask) + return lm_loss + else: + return { + "prediction_scores": logits, + } + + @staticmethod + def set_pipeline_stage_id(model): + dist_utils = dist.get_dist_util() + + # Set pipeline parallelism stage_id + if hasattr(model.t5_model.encoder.final_layernorm, "config"): + # Old API in OneFlow 0.8 + for module_block in model.modules(): + if isinstance(module_block.origin, T5Embedding): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, ExtendedMask): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.origin, TransformerLayer): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.origin, T5Loss): + module_block.config.set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + model.t5_model.encoder.final_layernorm.config.set_stage( + dist_utils.get_layer_stage_id(model.t5_model.encoder.final_layernorm.layer_idx), + dist.get_layer_placement(model.t5_model.encoder.final_layernorm.layer_idx), + ) + model.t5_model.decoder.final_layernorm.config.set_stage( + dist_utils.get_layer_stage_id(model.t5_model.decoder.final_layernorm.layer_idx), + dist.get_layer_placement(model.t5_model.decoder.final_layernorm.layer_idx), + ) + else: + for module_block in model.modules(): + if isinstance(module_block.to(nn.Module), T5Embedding): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), ExtendedMask): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(0), dist.get_layer_placement(0) + ) + elif isinstance(module_block.to(nn.Module), TransformerLayer): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(module_block.layer_idx), + dist.get_layer_placement(module_block.layer_idx), + ) + elif isinstance(module_block.to(nn.Module), T5Loss): + module_block.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(-1), dist.get_layer_placement(-1) + ) + + model.t5_model.encoder.final_layernorm.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(model.t5_model.encoder.final_layernorm.layer_idx), + dist.get_layer_placement(model.t5_model.encoder.final_layernorm.layer_idx), + ) + model.t5_model.decoder.final_layernorm.to(flow.nn.graph.GraphModule).set_stage( + dist_utils.get_layer_stage_id(model.t5_model.decoder.final_layernorm.layer_idx), + dist.get_layer_placement(model.t5_model.decoder.final_layernorm.layer_idx), + ) diff --git a/projects/T5/models/transformer_layer.py b/projects/T5/models/transformer_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..c23cb903ddd48faafc310380850701df8bd1b003 --- /dev/null +++ b/projects/T5/models/transformer_layer.py @@ -0,0 +1,252 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow.nn as nn + +from libai.layers.droppath import DropPath +from libai.utils import distributed as dist +from projects.T5.models.attention import MultiheadAttention +from projects.T5.models.layer_norm import LayerNorm +from projects.T5.models.mlp import MT5MLP, T5MLP + + +class TransformerLayer(nn.Module): + """A single transformer layer. + + Transformer layer takes input with size [bsz, seq_length, hidden size] and returns an + output of the same size. + The input and output has same sbp sign, (S(0), B). + + Arguments: + hidden_size: size of hidden state. + ffn_hidden_size: size of feed forword neural network. + num_attention_heads: number of attention heads. + is_decoder: used to specify whether this is transformer encoder layer or transformer + decoder layer. Default: ``False``. + attention_dropout_prob: dropout probability of attention weights. + output_dropout_prob: dropout probability of output. + layernorm_epsilon: epsilon used in layernorm layer. Default: `1e-5`. + init_method: method to initialize the input layer weights. + output_layer_init_method: method to initialize the output layer weights. + If None, use `init_method`. + layer_idx: the layer index, which determines the placement. + """ + + def __init__( + self, + hidden_size, + ffn_hidden_size, + num_attention_heads, + head_size, + relative_attention_num_buckets, + is_decoder=False, + attention_dropout_prob=0.0, + output_dropout_prob=0.0, + drop_path_prob=0.0, + layernorm_epsilon=1e-5, + init_method=nn.init.xavier_normal_, + output_layer_init_method=None, + *, + layer_idx=0, + model_type="t5", + has_relative_attention_bias=False + ): + super().__init__() + self.hidden_size = hidden_size + self.ffn_hidden_size = ffn_hidden_size + self.num_attention_heads = num_attention_heads + self.head_size = head_size + self.attention_dropout_prob = attention_dropout_prob + self.output_dropout_prob = output_dropout_prob + self.layernorm_epsilon = layernorm_epsilon + self.layer_idx = layer_idx + self.is_decoder = is_decoder + + self.init_method = init_method + if output_layer_init_method is None: + output_layer_init_method = init_method + self.output_layer_init_method = output_layer_init_method + + self.drop_path = DropPath(drop_path_prob) if drop_path_prob > 0.0 else nn.Identity() + + self.input_layernorm = LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon, layer_idx=self.layer_idx + ) + + self.self_attention = self.build_attention( + is_cross_attention=False, + relative_attention_num_buckets=relative_attention_num_buckets, + has_relative_attention_bias=has_relative_attention_bias, + is_decoder=self.is_decoder, + ) + self.post_attention_layernorm = LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon, layer_idx=self.layer_idx + ) + + if self.is_decoder: + self.cross_attention = self.build_attention( + is_cross_attention=True, + relative_attention_num_buckets=relative_attention_num_buckets, + is_decoder=self.is_decoder, + ) + self.post_cross_attention_layernorm = LayerNorm( + self.hidden_size, eps=self.layernorm_epsilon, layer_idx=self.layer_idx + ) + if model_type == "mt5": + self.mlp = MT5MLP( + self.hidden_size, + self.ffn_hidden_size, + self.output_dropout_prob, + self.init_method, + output_layer_init_method=self.output_layer_init_method, + layer_idx=self.layer_idx, + ) + elif model_type == "t5": + self.mlp = T5MLP( + self.hidden_size, + self.ffn_hidden_size, + self.output_dropout_prob, + self.init_method, + output_layer_init_method=self.output_layer_init_method, + layer_idx=self.layer_idx, + ) + + def forward( + self, + hidden_states, + attention_mask=None, + encoder_states=None, + encoder_attention_mask=None, + past_key_value=None, + use_cache=False, + position_bias=None, + encoder_decoder_position_bias=None, + ): + """ + Args: + hidden_states: shape is (batch_size, seq_length, hidden_size), + sbp signature is (S(0), B). + attention_mask: the combination of key padding mask and casual mask of hidden states + with shape (batch_size, 1, seq_length, seq_length) and the sbp + signature is (S(0), B), + encoder_states: encoder output with shape (batch_size, seq_length, hidden_size) + and the sbp signature is (S(0), B), which will be used in cross attention. + encoder_attention_mask: key padding mask of encoder states with shape + (batch_size, 1, seq_length, seq_length) and the sbp signature is (S(0), B). + past_key_value: tuple of key and value, each shape is + (seq_length, bsz, num_heads, head_size), For decoder layer, + the past_key_value contains the states both from self attention + and cross attention. + use_cache: it will be set to `True` when the model is in the inference phase and + used for incremental decoding. + """ + # Change placement for pipeline parallelsim + hidden_states = hidden_states.to_global(placement=dist.get_layer_placement(self.layer_idx)) + + # hidden_states shape: (batch_size, seq_length, hidden_size) + if attention_mask is not None: + attention_mask = attention_mask.to_global( + placement=dist.get_layer_placement(self.layer_idx) + ) + + if past_key_value is not None: + if self.is_decoder: + assert len(past_key_value) == 4 + self_attn_past_key_value = past_key_value[:2] + cross_attn_past_key_value = past_key_value[2:] + else: + self_attn_past_key_value = past_key_value + cross_attn_past_key_value = None + else: + self_attn_past_key_value, cross_attn_past_key_value = None, None + + layernorm_output = self.input_layernorm(hidden_states) + + attention_output, position_bias = self.self_attention( + layernorm_output, + attention_mask=attention_mask, + past_key_value=self_attn_past_key_value, + position_bias=position_bias, + use_cache=use_cache, + ) + + attention_output = self.drop_path(attention_output) + + if use_cache: + attention_output, presents = attention_output + else: + presents = None + + hidden_states = hidden_states + attention_output + + layernorm_output = self.post_attention_layernorm(hidden_states) + + if self.is_decoder: + if presents is not None: + query_length = presents[0].shape[2] + else: + query_length = None + + attention_output, encoder_decoder_position_bias = self.cross_attention( + layernorm_output, + encoder_states, + attention_mask=encoder_attention_mask, + past_key_value=cross_attn_past_key_value, + position_bias=encoder_decoder_position_bias, + use_cache=use_cache, + query_length=query_length, + ) + if use_cache: + attention_output, decoder_presents = attention_output + presents = presents + decoder_presents + + attention_output = self.drop_path(attention_output) + + hidden_states = hidden_states + attention_output + layernorm_output = self.post_cross_attention_layernorm(hidden_states) + + mlp_output = self.mlp(layernorm_output) + mlp_output = self.drop_path(mlp_output) + + output = hidden_states + mlp_output + + if use_cache: + output = (output, presents) + output = (output,) + (position_bias,) + if self.is_decoder: + output = output + (encoder_decoder_position_bias,) + return output + + def build_attention( + self, + is_cross_attention=False, + relative_attention_num_buckets=None, + has_relative_attention_bias=False, + is_decoder=False, + ): + return MultiheadAttention( + self.hidden_size, + self.num_attention_heads, + head_size=self.head_size, + relative_attention_num_buckets=relative_attention_num_buckets, + is_cross_attention=is_cross_attention, + attention_dropout_prob=self.attention_dropout_prob, + output_dropout_prob=self.output_dropout_prob, + init_method=self.init_method, + output_layer_init_method=self.output_layer_init_method, + layer_idx=self.layer_idx, + has_relative_attention_bias=has_relative_attention_bias, + is_decoder=is_decoder, + ) diff --git a/projects/T5/readme.md b/projects/T5/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..915405f246906b612307d005bc3d3ed0b42fd482 --- /dev/null +++ b/projects/T5/readme.md @@ -0,0 +1,60 @@ +# T5 + +Note: this project uses a special Dataset, which is an **unofficial** version, if you want to use your own dataset and experience complete mt5, please refer to [projects/MT5](../MT5/) + +Reproduce T5Model and MT5Model with OneFlow, which effect is equivalent to HuggingFace's [T5](https://huggingface.co/docs/transformers/v4.19.4/en/model_doc/t5#overview) and [T5v1.1](https://github.com/google-research/text-to-text-transfer-transformer/blob/main/released_checkpoints.md#t511). + +## Introduce +The T5 pretraining project can support 3D parallel and [ZERO](https://arxiv.org/abs/2202.10435). + +## Training MT5 +Training MT5 on 8 GPUs using 3D parallelism and ZERO. + +### 1. Prepare your training config file + +set the pretrain parameters in `T5/configs/mt5_pretrain.py`, such as `train_data_path` and `pretrained_model_path`. +> If the `pretrained_model_path` is set, the path should contain `pytorch_model.bin` and `config.json`, +> and the `T5/configs/mt5_pretrain.py` set default `pretrained_model_path` for consistent the same initialization as huggingface. + +### 2. Prepare the default init model and config + +```bash +# /path/to/projects/T5/data/init_mt5/ +wget http://oneflow-static.oss-cn-beijing.aliyuncs.com/libai/mt5_init/config.json +wget http://oneflow-static.oss-cn-beijing.aliyuncs.com/libai/mt5_init/pytorch_model.bin +``` + +### 3.Prepare the training data + +```bash +# /path/to/projects/T5/data/training_data/ +wget http://oneflow-static.oss-cn-beijing.aliyuncs.com/libai/mt5_init/wudao_180g_test_bert_tokenized_512_demo.zip +unzip wudao_180g_test_bert_tokenized_512_demo.zip +``` + +The structure of the training data folder should be like: +``` +$ tree training_data_dir +path/to/projects/T5/data/training_data/ + ├──part_0 + ├──part_1 + ├──part_2 + ├──part_3 + ... + ├──part_8 + └──part_9 +``` + +### 4. Run the following code +```bash +# cd /path/to/libai +bash tools/train.sh tools/train_net.py projects/T5/configs/mt5_pretrain.py 8 +``` + +### 5. Convert OneFlow Checkpoint to PyTorch +Suppose you need to put the final checkpoint to huggingface, run the following code, and ensure the parameters are set correctly in `weight_convert.sh`: + +```bash +# cd /path/to/libai +bash projects/T5/utils/weight_convert.sh +``` \ No newline at end of file diff --git a/projects/T5/utils/mask.py b/projects/T5/utils/mask.py new file mode 100644 index 0000000000000000000000000000000000000000..3fb0b90dacaa3267907a77bb07a3ea9893380dad --- /dev/null +++ b/projects/T5/utils/mask.py @@ -0,0 +1,63 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow + +from libai.utils import distributed as dist + + +class ExtendedMask(flow.nn.Module): + def forward(self, x, input_tensor=None, is_decoder=False): + if x.dim() == 3: + extended_mask = x[:, None, :, :] + elif x.dim() == 2: + if is_decoder: + extended_mask = self.create_extended_mask_for_decoder(x, input_tensor) + else: + extended_mask = x[:, None, None, :] + + return extended_mask + + def create_extended_mask_for_decoder(self, x, input_tensor): + batch_size, seq_len = input_tensor.size() + seq_ids = flow.arange( + seq_len, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=x.placement, + ) + causal_mask = ( + seq_ids[None, None, :].repeat(batch_size, seq_len, 1) <= seq_ids[None, :, None] + ) + + causal_mask = causal_mask.to(x.dtype) + causal_mask = causal_mask.to_global(sbp=x.sbp) + if causal_mask.shape[1] < x.shape[1]: + prefix_seq_len = x.shape[1] - causal_mask.shape[1] + ones = flow.ones( + (batch_size, seq_len, prefix_seq_len), + dtype=causal_mask.dtype, + sbp=causal_mask.sbp, + placement=causal_mask.placement, + ) + causal_mask = flow.cat( + [ + ones, + causal_mask, + ], + dim=-1, + ) + + extended_mask = causal_mask[:, None, :, :] * x[:, None, None, :] + return extended_mask diff --git a/projects/T5/utils/weight_convert.py b/projects/T5/utils/weight_convert.py new file mode 100644 index 0000000000000000000000000000000000000000..4b3b321fd577d8b3e5aebfa55f4ac35cb6186790 --- /dev/null +++ b/projects/T5/utils/weight_convert.py @@ -0,0 +1,217 @@ +import argparse + +import oneflow as flow +import torch + +from libai.config import LazyConfig + + +def parse_args(): + parser = argparse.ArgumentParser(description="MT5 Weight Convertor") + parser.add_argument( + "--oneflow_state_dict_path", type=str, help="The path of mt5's checkpoint in LiBai" + ) + parser.add_argument( + "--config_path", + type=str, + default="projects/T5/configs/mt5_pretrain.py", + help="The path of the training config", + ) + parser.add_argument("--save_path", type=str, default="projects/T5/pytorch_model.bin") + return parser.parse_args() + + +def fix_qkv_ordering(qkv, head_size, num_heads, hidden_size=None): + hidden_size = (head_size * num_heads) if hidden_size is None else hidden_size + num_of_qkv = qkv.shape[0] // (head_size * num_heads) + + qkv = qkv.view(-1) + qkv = qkv.view(num_heads, num_of_qkv, head_size, hidden_size) + qkv = qkv.permute(1, 0, 2, 3).contiguous() + qkv = qkv.view(num_of_qkv * head_size * num_heads, hidden_size) + return qkv + + +def convert_tensor(tensor): + return torch.tensor(tensor.detach().to_numpy(), dtype=torch.float32) + + +def convert_state_dict(oneflow_state_dict_path, libai_cfg, prefix="t5_model."): + oneflow_state_dict = flow.load(oneflow_state_dict_path) + torch_state_dict = {} + # Get configs + num_heads = libai_cfg.get("num_attention_heads") + hidden_size = libai_cfg.get("hidden_size") + head_size = libai_cfg.get("head_size", None) + + if head_size is None: + head_size = int(hidden_size / num_heads) + + layer_idx = 3 if len(prefix) > 1 else 2 + enc_dec_idx = 1 if len(prefix) > 1 else 0 + op_idx = 4 if len(prefix) > 1 else 3 + + # Convert T5's Embedding layers. + x = convert_tensor(oneflow_state_dict.pop(prefix + "embedding.word_embeddings.weight")) + new_key = "shared.weight" + torch_state_dict[new_key] = x + new_key = "encoder.embed_tokens.weight" + torch_state_dict[new_key] = x + new_key = "decoder.embed_tokens.weight" + torch_state_dict[new_key] = x + + # Convert T5's final_layer_norm + new_key = "encoder.final_layer_norm.weight" + torch_state_dict[new_key] = convert_tensor( + oneflow_state_dict.pop(prefix + "encoder.final_layernorm.weight") + ) + new_key = "decoder.final_layer_norm.weight" + torch_state_dict[new_key] = convert_tensor( + oneflow_state_dict.pop(prefix + "decoder.final_layernorm.weight") + ) + + old_keys = list(oneflow_state_dict.keys()) + + # Convert T5's lm_head + new_key = "lm_head.weight" + if prefix + new_key in old_keys: + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(prefix + new_key)) + + for key in old_keys: + keys = key.split(".") + if op_idx > len(keys): + continue + layers = keys[layer_idx] + enc_dec = keys[enc_dec_idx] + op_name = keys[op_idx] + + if keys[op_idx + 1] == "relative_attention_bias": + new_key = enc_dec + ".block.0.layer.0.SelfAttention.relative_attention_bias.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + + # Convert T5's Encoder layers. + if enc_dec == "encoder": + if op_name == "self_attention": + if keys[op_idx + 1] == "query_key_value": + x = oneflow_state_dict.pop(key) + x = fix_qkv_ordering(x, head_size, num_heads, hidden_size) + q, k, v = flow.chunk(x, chunks=3, dim=0) + + new_key = "encoder.block." + layers + ".layer.0.SelfAttention.q.weight" + torch_state_dict[new_key] = convert_tensor(q) + new_key = new_key.replace("q", "k") + torch_state_dict[new_key] = convert_tensor(k) + new_key = new_key.replace("k", "v") + torch_state_dict[new_key] = convert_tensor(v) + if keys[op_idx + 1] == "dense": + new_key = "encoder.block." + layers + ".layer.0.SelfAttention.o.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + + elif op_name == "input_layernorm": + new_key = "encoder.block." + layers + ".layer.0.layer_norm.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + + elif op_name == "post_attention_layernorm": + new_key = "encoder.block." + layers + ".layer.1.layer_norm.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + + elif op_name == "mlp": + if libai_cfg.get("model_type") == "mt5": + if keys[op_idx + 1] == "wi_0": + new_key = "encoder.block." + layers + ".layer.1.DenseReluDense.wi_0.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + if keys[op_idx + 1] == "wi_1": + new_key = "encoder.block." + layers + ".layer.1.DenseReluDense.wi_1.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + if keys[op_idx + 1] == "wo": + new_key = "encoder.block." + layers + ".layer.1.DenseReluDense.wo.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + elif libai_cfg.get("model_type") == "t5": + if keys[op_idx + 1] == "dense_h_to_4h": + new_key = "encoder.block." + layers + ".layer.1.DenseReluDense.wi.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + if keys[op_idx + 1] == "dense_4h_to_h": + new_key = "encoder.block." + layers + ".layer.1.DenseReluDense.wo.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + + # Convert T5's decoder Layers. + elif enc_dec == "decoder": + if op_name == "self_attention": + if keys[op_idx + 1] == "query_key_value": + x = oneflow_state_dict.pop(key) + x = fix_qkv_ordering(x, head_size, num_heads, hidden_size) + q, k, v = flow.chunk(x, chunks=3, dim=0) + + new_key = "decoder.block." + layers + ".layer.0.SelfAttention.q.weight" + torch_state_dict[new_key] = convert_tensor(q) + new_key = new_key.replace("q", "k") + torch_state_dict[new_key] = convert_tensor(k) + new_key = new_key.replace("k", "v") + torch_state_dict[new_key] = convert_tensor(v) + if keys[op_idx + 1] == "dense": + new_key = "decoder.block." + layers + ".layer.0.SelfAttention.o.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + + elif op_name == "input_layernorm": + new_key = "decoder.block." + layers + ".layer.0.layer_norm.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + + elif op_name == "post_attention_layernorm": + new_key = "decoder.block." + layers + ".layer.1.layer_norm.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + + elif op_name == "post_cross_attention_layernorm": + new_key = "decoder.block." + layers + ".layer.2.layer_norm.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + + elif op_name == "cross_attention": + if keys[op_idx + 1] == "query": + x = oneflow_state_dict.pop(key) + x = fix_qkv_ordering(x, head_size, num_heads, hidden_size) + new_key = "decoder.block." + layers + ".layer.1.EncDecAttention.q.weight" + torch_state_dict[new_key] = convert_tensor(x) + if keys[op_idx + 1] == "key_value": + x = oneflow_state_dict.pop(key) + x = fix_qkv_ordering(x, head_size, num_heads, hidden_size) + k, v = flow.chunk(x, chunks=2, dim=0) + new_key = "decoder.block." + layers + ".layer.1.EncDecAttention.k.weight" + torch_state_dict[new_key] = convert_tensor(k) + new_key = new_key.replace("k", "v") + torch_state_dict[new_key] = convert_tensor(v) + if keys[op_idx + 1] == "dense": + new_key = "decoder.block." + layers + ".layer.1.EncDecAttention.o.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + + elif op_name == "mlp": + if libai_cfg.get("model_type") == "mt5": + if keys[op_idx + 1] == "wi_0": + new_key = "decoder.block." + layers + ".layer.1.DenseReluDense.wi_0.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + if keys[op_idx + 1] == "wi_1": + new_key = "decoder.block." + layers + ".layer.1.DenseReluDense.wi_1.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + if keys[op_idx + 1] == "wo": + new_key = "decoder.block." + layers + ".layer.1.DenseReluDense.wo.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + elif libai_cfg.get("model_type") == "t5": + if keys[op_idx + 1] == "dense_h_to_4h": + new_key = "decoder.block." + layers + ".layer.1.DenseReluDense.wi.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + if keys[op_idx + 1] == "dense_4h_to_h": + new_key = "decoder.block." + layers + ".layer.1.DenseReluDense.wo.weight" + torch_state_dict[new_key] = convert_tensor(oneflow_state_dict.pop(key)) + + return torch_state_dict + + +if __name__ == "__main__": + args = parse_args() + oneflow_state_dict_path = args.oneflow_state_dict_path + config_path = args.config_path + save_path = args.save_path + + training_config = LazyConfig.load(config_path) + model_config = training_config.model.cfg + torch_state_dict = convert_state_dict(oneflow_state_dict_path, model_config) + + torch.save(torch_state_dict, save_path) diff --git a/projects/T5/utils/weight_convert.sh b/projects/T5/utils/weight_convert.sh new file mode 100644 index 0000000000000000000000000000000000000000..8d55bb257926c504e6283c805972cd8d717bbfe4 --- /dev/null +++ b/projects/T5/utils/weight_convert.sh @@ -0,0 +1,10 @@ +set -aux + +ONEFLOW_STATE_DICT_PATH="projects/T5/output/mt5_output/model_final/model" +CONFIG_PATH="projects/T5/configs/mt5_pretrain.py" +SAVE_PATH="projects/T5/pytorch_model.bin" + +python3 projects/T5/utils/weight_convert.py \ + --oneflow_state_dict_path $ONEFLOW_STATE_DICT_PATH \ + --config_path $CONFIG_PATH \ + --save_path $SAVE_PATH \ No newline at end of file diff --git a/projects/text_classification/README.md b/projects/text_classification/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fcfabd4a7a54e9d8ed107807348abdcbf4dae9c8 --- /dev/null +++ b/projects/text_classification/README.md @@ -0,0 +1,32 @@ +# Text classification projects +**English** | [简体中文](/projects/text_classification/README_zh-CN.md) +## GLUE tasks + +Fine-tuning the pretrained language models for sequence classification on the GLUE benchmark: [General Language Understanding Evaluation](https://gluebenchmark.com/). This is a common benchmark for *English* pretrained language models. + +GLUE is made up of a total of 9 different tasks. The tasks contains cola, sst2, mrpc, stsb, qqp, mnli, qnli, rte, wnli. You can perform the following command to download and extract the dataset: +```bash +cd projects/text_classification +python3 dataset/download_glue_data.py +``` + +Here is how to finetune the task on one of them: +```bash +bash tools/train.sh tools/train_net.py projects/text_classification/configs/config.py $num_gpus train.train_iter=10 +``` +where `$num_gpus` indicates the number of GPUs. If you want to run the distributed program, you can change it, for example: +```bash +bash tools/train.sh tools/train_net.py projects/text_classification/configs/config.py 2 train.train_iter=10 +``` + +Before running the program, you should modify the `config.py` file. Modification fields include but are not limited to task_name, data_dir, vocab_file, model hyperparameter, learning rate, and batch size, and so on. + +## CLUE tasks + +For *Chinese* pretrained language model, it can be evaluated by [CLUE benchmark](https://github.com/CLUEbenchmark/CLUE). This is a sequence classification task just like GLUE, so you can finetune your model with the same command. Don't forget to modify `config.py` file. + +You can download and extract the datasets by: +```bash +cd projects/text_classification +python3 dataset/download_clue_data.py +``` \ No newline at end of file diff --git a/projects/text_classification/README_zh-CN.md b/projects/text_classification/README_zh-CN.md new file mode 100644 index 0000000000000000000000000000000000000000..eb3fc3760f63457393bf512e1e04320890d05799 --- /dev/null +++ b/projects/text_classification/README_zh-CN.md @@ -0,0 +1,32 @@ +# 文本分类任务 +[English](/projects/text_classification/README.md) | **简体中文** + +## GLUE 任务 + +在文本分类任务上微调预训练语言模型。[GLUE](https://gluebenchmark.com/) 是一个英文文本分类常见的基准任务，通常用它评估预训练语言模型的理解能力。 + +GLUE 由9个子任务组成，包括cola、sst2、mrpc、stsb、qqp、mnli、qnli、rte、wnli。你可以执行通过命令实现一键下载并提取数据： +```bash +cd projects/text_classification +python3 dataset/download_glue_data.py +``` + +可以通过以下命令在此任务上进行finetune: +```bash +bash tools/train.sh tools/train_net.py projects/text_classification/configs/config.py $num_gpus train.train_iter=10 +``` +其中`$num_gpus`表示运行程序的GPU数量。如果你想执行分布式训练，可以这样修改： +```bash +bash tools/train.sh tools/train_net.py projects/text_classification/configs/config.py 2 train.train_iter=10 +``` + +在运行程序之前，你应当修改`config.py`配置文件。待修改字段包括但不限于task_name、data_dir、vocab_file、模型超参数、学习率、批次大小等等。 + +## CLUE 任务 + +对于中文预训练语言模型，适用 CLUE 基准任务进行评估。和 GLUE 基准任务相同，CLUE 是一个文本分类任务，训练方法与上面相同。注意，记得修改`config.py`配置文件。 + +你可以下载并提取 CLUE 完整数据集通过以下命令: +```bash +python3 dataset/download_clue_data.py +``` diff --git a/projects/text_classification/configs/config.py b/projects/text_classification/configs/config.py new file mode 100644 index 0000000000000000000000000000000000000000..c4d6fde2ccae4ad3ed4be1d5cd31585154a36c5b --- /dev/null +++ b/projects/text_classification/configs/config.py @@ -0,0 +1,80 @@ +from omegaconf import OmegaConf + +from libai.config import get_config +from libai.config import LazyCall +from libai.data.build import build_nlp_test_loader, build_nlp_train_loader +from libai.tokenizer import BertTokenizer +from projects.text_classification.modeling.model import ModelForSequenceClassification +from projects.text_classification.dataset import ClueDataset + +tokenization = get_config("common/data/bert_dataset.py").tokenization +optim = get_config("common/optim.py").optim +model_cfg = get_config("common/models/bert.py").cfg +graph = get_config("common/models/graph.py").graph +train = get_config("common/train.py").train + +tokenization.tokenizer = LazyCall(BertTokenizer)( + vocab_file="/DATA/disk1/liuchi/work/bert-base-chinese-vocab.txt", + do_lower_case=True, + do_chinese_wwm=False, +) +tokenization.append_eod = False +tokenization.make_vocab_size_divisible_by = 128 + +dataloader = OmegaConf.create() +dataloader.train = LazyCall(build_nlp_train_loader)( + dataset=[ + LazyCall(ClueDataset)( + task_name="afqmc", + data_dir="./projects/text_classification/dataset/clue_data/afqmc", + tokenizer=tokenization.tokenizer, + max_seq_length=128, + mode="train", + ), + ], + num_workers=4, +) +dataloader.test = [ + LazyCall(build_nlp_test_loader)( + dataset=LazyCall(ClueDataset)( + task_name="afqmc", + data_dir="./projects/text_classification/dataset/clue_data/afqmc", + tokenizer=tokenization.tokenizer, + max_seq_length=512, + mode="dev", + ), + num_workers=4, + ), +] + +model_cfg.update( + dict( + # exist key + vocab_size=21248, + hidden_size=1024, + hidden_layers=24, + num_attention_heads=16, + # new key + num_classes=2, + pretrain_megatron_weight=None, + ) +) +model = LazyCall(ModelForSequenceClassification)(cfg=model_cfg) + +train.update( + dict( + recompute_grad=dict(enabled=True), + output_dir="output/benchmark/", + train_micro_batch_size=4, + test_micro_batch_size=4, + train_epoch=1, + train_iter=0, + eval_period=500, + log_period=50, + dist=dict( + data_parallel_size=1, + tensor_parallel_size=1, + pipeline_parallel_size=1, + ), + ) +) diff --git a/projects/text_classification/dataset/__init__.py b/projects/text_classification/dataset/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9b360dba6b13a4a4db1e760069d0c002cac993a9 --- /dev/null +++ b/projects/text_classification/dataset/__init__.py @@ -0,0 +1,17 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .clue_dataset import ClueDataset +from .glue_dataset import GlueDataset diff --git a/projects/text_classification/dataset/clue_dataset.py b/projects/text_classification/dataset/clue_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..2aba31d7a697b69d43daab1fd6bd132195b7b811 --- /dev/null +++ b/projects/text_classification/dataset/clue_dataset.py @@ -0,0 +1,123 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import time +from enum import Enum +from typing import Optional, Union + +import oneflow as flow +from filelock import FileLock +from oneflow.utils.data import Dataset + +from libai.data.structures import DistTensorData, Instance + +from .utils import EncodePattern +from .utils_clue import clue_convert_examples_to_features, clue_output_modes, clue_processors + +logger = logging.getLogger(__name__) + + +class Split(Enum): + train = "train" + dev = "dev" + test = "test" + + +class ClueDataset(Dataset): + def __init__( + self, + task_name, + data_dir, + tokenizer, + max_seq_length: int = 128, + mode: Union[str, Split] = Split.train, + pattern: Union[str, EncodePattern] = EncodePattern.bert_pattern, + cache_dir: Optional[str] = None, + overwrite_cache: bool = True, + ): + self.processor = clue_processors[task_name]() + self.output_mode = clue_output_modes[task_name] + if isinstance(mode, str): + try: + mode = Split[mode] + except KeyError: + raise KeyError("mode is not a valid split name") + # Load data features from cache or dataset file + cached_features_file = os.path.join( + cache_dir if cache_dir is not None else data_dir, + f"cached_{mode.value}_{tokenizer.__class__.__name__}_{max_seq_length}_{task_name}", + ) + label_list = self.processor.get_labels() + self.label_list = label_list + + # Make sure only the first process in distributed training processes the dataset, + # and the others will use the cache. + lock_path = cached_features_file + ".lock" + with FileLock(lock_path): + + if os.path.exists(cached_features_file) and not overwrite_cache: + start = time.time() + self.features = flow.load(cached_features_file) + logger.info( + f"Loading features from cached file {cached_features_file} [took %.3f s]", + time.time() - start, + ) + else: + logger.info(f"Creating features from dataset file at {data_dir}") + + if mode == Split.dev: + examples = self.processor.get_dev_examples(data_dir) + elif mode == Split.test: + examples = self.processor.get_test_examples(data_dir) + else: + examples = self.processor.get_train_examples(data_dir) + + self.features = clue_convert_examples_to_features( + examples, + tokenizer, + max_length=max_seq_length, + pattern=pattern, + label_list=label_list, + output_mode=self.output_mode, + ) + start = time.time() + flow.save(self.features, cached_features_file) + logger.info( + f"Saving features into cached file {cached_features_file} " + f"[took {time.time() - start:.3f} s]" + ) + + def __len__(self): + return len(self.features) + + def __getitem__(self, i): + feature = self.features[i] + tensors = {} + for k, v in feature.__dict__.items(): + if v is not None: + if k == "labels": + dtype = flow.long if isinstance(v, int) else flow.float + t = flow.tensor(v, dtype=dtype) + tensors[k] = DistTensorData(t, placement_idx=-1) + else: + t = flow.tensor(v, dtype=flow.long) + tensors[k] = DistTensorData(t) + sample = Instance(**tensors) + return sample + + def get_labels(self): + return self.label_list diff --git a/projects/text_classification/dataset/download_clue_data.py b/projects/text_classification/dataset/download_clue_data.py new file mode 100644 index 0000000000000000000000000000000000000000..8d9ecdc1c448cbd05ec9e0edca275bd29cfb7cf2 --- /dev/null +++ b/projects/text_classification/dataset/download_clue_data.py @@ -0,0 +1,96 @@ +# flake8: noqa +""" Script for downloading all CLUE data. +The original dataset information links +available from: https://www.cluebenchmarks.com/ +Example usage: + python download_clue_data.py --data_dir data --tasks all +""" + +import argparse +import os +import sys +import urllib.request +import zipfile + +TASKS = [ + "afqmc", + "cmnli", + "copa", + "csl", + "iflytek", + "tnews", + "wsc", + "cmrc", + "chid", + "drcd", +] + +TASK2PATH = { + "afqmc": "https://storage.googleapis.com/cluebenchmark/tasks/afqmc_public.zip", + "cmnli": "https://storage.googleapis.com/cluebenchmark/tasks/cmnli_public.zip", + "copa": "https://storage.googleapis.com/cluebenchmark/tasks/copa_public.zip", + "csl": "https://storage.googleapis.com/cluebenchmark/tasks/csl_public.zip", + "iflytek": "https://storage.googleapis.com/cluebenchmark/tasks/iflytek_public.zip", + "tnews": "https://storage.googleapis.com/cluebenchmark/tasks/tnews_public.zip", + "wsc": "https://storage.googleapis.com/cluebenchmark/tasks/cluewsc2020_public.zip", + "cmrc": "https://storage.googleapis.com/cluebenchmark/tasks/cmrc2018_public.zip", + "chid": "https://storage.googleapis.com/cluebenchmark/tasks/chid_public.zip", + "drcd": "https://storage.googleapis.com/cluebenchmark/tasks/drcd_public.zip", +} + + +def download_and_extract(task, data_dir): + print("Downloading and extracting %s..." % task) + if not os.path.isdir(data_dir): + os.mkdir(data_dir) + data_file = os.path.join(data_dir, "%s_public.zip" % task) + save_dir = os.path.join(data_dir, task) + if not os.path.isdir(save_dir): + os.mkdir(save_dir) + urllib.request.urlretrieve(TASK2PATH[task], data_file) + with zipfile.ZipFile(data_file) as zip_ref: + zip_ref.extractall(save_dir) + os.remove(data_file) + print(f"\tCompleted! Downloaded {task} data to directory {save_dir}") + + +def get_tasks(task_names): + task_names = task_names.split(",") + if "all" in task_names: + tasks = TASKS + else: + tasks = [] + for task_name in task_names: + assert task_name in TASKS, "Task %s not found!" % task_name + tasks.append(task_name) + return tasks + + +def main(arguments): + parser = argparse.ArgumentParser() + parser.add_argument( + "-d", + "--data_dir", + help="directory to save data to", + type=str, + default="./clue_data", + ) + parser.add_argument( + "-t", + "--tasks", + help="tasks to download data for as a comma separated string", + type=str, + default="all", + ) + args = parser.parse_args(arguments) + + if not os.path.exists(args.data_dir): + os.mkdir(args.data_dir) + tasks = get_tasks(args.tasks) + + for task in tasks: + download_and_extract(task, args.data_dir) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/projects/text_classification/dataset/download_glue_data.py b/projects/text_classification/dataset/download_glue_data.py new file mode 100644 index 0000000000000000000000000000000000000000..6eacfab9cca52250950f87f234e7d2b5c6a41404 --- /dev/null +++ b/projects/text_classification/dataset/download_glue_data.py @@ -0,0 +1,178 @@ +# flake8: noqa +""" Script for downloading all GLUE data. +Modify from https://github.com/nyu-mll/GLUE-baselines/blob/master/download_glue_data.py + +Note: for legal reasons, we are unable to host MRPC. +You can either use the version hosted by the SentEval team, which is already tokenized, +or you can download the original data from (https://download.microsoft.com/download/D/4/6/D46FF87A-F6B9-4252-AA8B-3604ED519838/MSRParaphraseCorpus.msi) and extract the data from it manually. +For Windows users, you can run the .msi file. For Mac and Linux users, consider an external library such as 'cabextract' (see below for an example). +You should then rename and place specific files in a folder (see below for an example). +mkdir MRPC +cabextract MSRParaphraseCorpus.msi -d MRPC +cat MRPC/_2DEC3DBE877E4DB192D17C0256E90F1D | tr -d $'\r' > MRPC/msr_paraphrase_train.txt +cat MRPC/_D7B391F9EAFF4B1B8BCE8F21B20B1B61 | tr -d $'\r' > MRPC/msr_paraphrase_test.txt +rm MRPC/_* +rm MSRParaphraseCorpus.msi +""" + +import argparse +import io +import os +import sys +import urllib + +if sys.version_info >= (3, 0): + import urllib.request + +import zipfile + +URLLIB = urllib +if sys.version_info >= (3, 0): + URLLIB = urllib.request + +TASKS = ["CoLA", "SST", "MRPC", "QQP", "STS", "MNLI", "QNLI", "RTE", "WNLI", "diagnostic"] +TASK2PATH = { + "CoLA": "https://dl.fbaipublicfiles.com/glue/data/CoLA.zip", + "SST": "https://dl.fbaipublicfiles.com/glue/data/SST-2.zip", + "QQP": "https://dl.fbaipublicfiles.com/glue/data/STS-B.zip", + "STS": "https://dl.fbaipublicfiles.com/glue/data/QQP-clean.zip", + "MNLI": "https://dl.fbaipublicfiles.com/glue/data/MNLI.zip", + "QNLI": "https://dl.fbaipublicfiles.com/glue/data/QNLIv2.zip", + "RTE": "https://dl.fbaipublicfiles.com/glue/data/RTE.zip", + "WNLI": "https://dl.fbaipublicfiles.com/glue/data/WNLI.zip", + "diagnostic": "https://dl.fbaipublicfiles.com/glue/data/AX.tsv", +} + +MRPC_TRAIN = "https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_train.txt" +MRPC_TEST = "https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_test.txt" + + +def download_and_extract(task, data_dir): + print("Downloading and extracting %s..." % task) + if task == "MNLI": + print( + "\tNote (12/10/20): This script no longer downloads SNLI. You will need to manually download and format the data to use SNLI." + ) + data_file = "%s.zip" % task + URLLIB.urlretrieve(TASK2PATH[task], data_file) + with zipfile.ZipFile(data_file) as zip_ref: + zip_ref.extractall(data_dir) + os.remove(data_file) + print("\tCompleted!") + + +def format_mrpc(data_dir, path_to_data): + print("Processing MRPC...") + mrpc_dir = os.path.join(data_dir, "MRPC") + if not os.path.isdir(mrpc_dir): + os.mkdir(mrpc_dir) + if path_to_data: + mrpc_train_file = os.path.join(path_to_data, "msr_paraphrase_train.txt") + mrpc_test_file = os.path.join(path_to_data, "msr_paraphrase_test.txt") + else: + try: + mrpc_train_file = os.path.join(mrpc_dir, "msr_paraphrase_train.txt") + mrpc_test_file = os.path.join(mrpc_dir, "msr_paraphrase_test.txt") + URLLIB.urlretrieve(MRPC_TRAIN, mrpc_train_file) + URLLIB.urlretrieve(MRPC_TEST, mrpc_test_file) + except urllib.error.HTTPError: + print("Error downloading MRPC") + return + assert os.path.isfile(mrpc_train_file), "Train data not found at %s" % mrpc_train_file + assert os.path.isfile(mrpc_test_file), "Test data not found at %s" % mrpc_test_file + + with io.open(mrpc_test_file, encoding="utf-8") as data_fh, io.open( + os.path.join(mrpc_dir, "test.tsv"), "w", encoding="utf-8" + ) as test_fh: + header = data_fh.readline() + test_fh.write("index\t#1 ID\t#2 ID\t#1 String\t#2 String\n") + for idx, row in enumerate(data_fh): + label, id1, id2, s1, s2 = row.strip().split("\t") + test_fh.write("%d\t%s\t%s\t%s\t%s\n" % (idx, id1, id2, s1, s2)) + + try: + URLLIB.urlretrieve(TASK2PATH["MRPC"], os.path.join(mrpc_dir, "dev_ids.tsv")) + except KeyError or urllib.error.HTTPError: + print( + "\tError downloading standard development IDs for MRPC. You will need to manually split your data." + ) + return + + dev_ids = [] + with io.open(os.path.join(mrpc_dir, "dev_ids.tsv"), encoding="utf-8") as ids_fh: + for row in ids_fh: + dev_ids.append(row.strip().split("\t")) + + with io.open(mrpc_train_file, encoding="utf-8") as data_fh, io.open( + os.path.join(mrpc_dir, "train.tsv"), "w", encoding="utf-8" + ) as train_fh, io.open(os.path.join(mrpc_dir, "dev.tsv"), "w", encoding="utf-8") as dev_fh: + header = data_fh.readline() + train_fh.write(header) + dev_fh.write(header) + for row in data_fh: + label, id1, id2, s1, s2 = row.strip().split("\t") + if [id1, id2] in dev_ids: + dev_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2)) + else: + train_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2)) + + print("\tCompleted!") + + +def download_diagnostic(data_dir): + print("Downloading and extracting diagnostic...") + if not os.path.isdir(os.path.join(data_dir, "diagnostic")): + os.mkdir(os.path.join(data_dir, "diagnostic")) + data_file = os.path.join(data_dir, "diagnostic", "diagnostic.tsv") + URLLIB.urlretrieve(TASK2PATH["diagnostic"], data_file) + print("\tCompleted!") + return + + +def get_tasks(task_names): + task_names = task_names.split(",") + if "all" in task_names: + tasks = TASKS + else: + tasks = [] + for task_name in task_names: + assert task_name in TASKS, "Task %s not found!" % task_name + tasks.append(task_name) + return tasks + + +def main(arguments): + parser = argparse.ArgumentParser() + parser.add_argument( + "-d", "--data_dir", help="directory to save data to", type=str, default="glue_data" + ) + parser.add_argument( + "-t", + "--tasks", + help="tasks to download data for as a comma separated string", + type=str, + default="all", + ) + parser.add_argument( + "--path_to_mrpc", + help="path to directory containing extracted MRPC data, msr_paraphrase_train.txt and msr_paraphrase_text.txt", + type=str, + default="", + ) + args = parser.parse_args(arguments) + + if not os.path.isdir(args.data_dir): + os.mkdir(args.data_dir) + tasks = get_tasks(args.tasks) + + for task in tasks: + if task == "MRPC": + format_mrpc(args.data_dir, args.path_to_mrpc) + elif task == "diagnostic": + download_diagnostic(args.data_dir) + else: + download_and_extract(task, args.data_dir) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/projects/text_classification/dataset/glue_dataset.py b/projects/text_classification/dataset/glue_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..bd4aa4737321aa87592f6770d1d75c865f6e477e --- /dev/null +++ b/projects/text_classification/dataset/glue_dataset.py @@ -0,0 +1,128 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import time +from enum import Enum +from typing import Optional, Union + +import oneflow as flow +from filelock import FileLock +from oneflow.utils.data import Dataset + +from libai.data.structures import DistTensorData, Instance + +from .utils import EncodePattern +from .utils_glue import glue_convert_examples_to_features, glue_output_modes, glue_processors + +logger = logging.getLogger(__name__) + + +class Split(Enum): + train = "train" + dev = "dev" + test = "test" + + +class GlueDataset(Dataset): + def __init__( + self, + task_name, + data_dir, + tokenizer, + max_seq_length: int = 128, + mode: Union[str, Split] = Split.train, + pattern: Union[str, EncodePattern] = EncodePattern.bert_pattern, + cache_dir: Optional[str] = None, + overwrite_cache: bool = False, + ): + self.processor = glue_processors[task_name]() + self.output_mode = glue_output_modes[task_name] + if isinstance(mode, str): + try: + mode = Split[mode] + except KeyError: + raise KeyError("mode is not a valid split name") + if isinstance(pattern, str): + try: + pattern = EncodePattern[pattern] + except KeyError: + raise KeyError("pattern is not a valid pattern method") + # Load data features from cache or dataset file + cached_features_file = os.path.join( + cache_dir if cache_dir is not None else data_dir, + f"cached_{mode.value}_{tokenizer.__class__.__name__}_{max_seq_length}_{task_name}", + ) + label_list = self.processor.get_labels() + self.label_list = label_list + + # Make sure only the first process in distributed training processes the dataset, + # and the others will use the cache. + lock_path = cached_features_file + ".lock" + with FileLock(lock_path): + + if os.path.exists(cached_features_file) and not overwrite_cache: + start = time.time() + self.features = flow.load(cached_features_file) + logger.info( + f"Loading features from cached file {cached_features_file} [took %.3f s]", + time.time() - start, + ) + else: + logger.info(f"Creating features from dataset file at {data_dir}") + + if mode == Split.dev: + examples = self.processor.get_dev_examples(data_dir) + elif mode == Split.test: + examples = self.processor.get_test_examples(data_dir) + else: + examples = self.processor.get_train_examples(data_dir) + + self.features = glue_convert_examples_to_features( + examples, + tokenizer, + max_length=max_seq_length, + pattern=pattern, + label_list=label_list, + output_mode=self.output_mode, + ) + start = time.time() + flow.save(self.features, cached_features_file) + logger.info( + f"Saving features into cached file {cached_features_file} " + f"[took {time.time() - start:.3f} s]" + ) + + def __len__(self): + return len(self.features) + + def __getitem__(self, i): + feature = self.features[i] + tensors = {} + for k, v in feature.__dict__.items(): + if v is not None: + if k == "label": + dtype = flow.long if isinstance(v, int) else flow.float + t = flow.tensor(v, dtype=dtype) + tensors[k] = DistTensorData(t, placement_idx=-1) + else: + t = flow.tensor(v, dtype=flow.long) + tensors[k] = DistTensorData(t) + sample = Instance(**tensors) + return sample + + def get_labels(self): + return self.label_list diff --git a/projects/text_classification/dataset/utils.py b/projects/text_classification/dataset/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..24fa80fed793f4d341f6888b78dbe15a3fcaf556 --- /dev/null +++ b/projects/text_classification/dataset/utils.py @@ -0,0 +1,118 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import json +from dataclasses import dataclass +from enum import Enum +from typing import List, Optional, Union + + +class EncodePattern(Enum): + """encode pattern + bert pattern: + single sentence: [CLS] A [SEP] + pair of sentences: [CLS] A [SEP] B [SEP] + roberta/bart pattern: + single sentence: A + pair of sentences: A B + """ + + bert_pattern = "S*E*E" + roberta_pattern = "S*EE*E" + + +@dataclass +class InputExample: + """ + A single training/test example for simple sequence classification. + + Args: + guid: Unique id for the example. + text_a: string. The untokenized text of the first sequence. For single + sequence tasks, only this sequence must be specified. + text_b: (Optional) string. The untokenized text of the second sequence. + Only must be specified for sequence pair tasks. + label: (Optional) string. The label of the example. This should be + specified for train and dev examples, but not for test examples. + """ + + guid: str + text_a: str + text_b: Optional[str] = None + label: Optional[str] = None + + +@dataclass(frozen=True) +class InputFeatures: + """ + A single set of features of data. + Property names are the same names as the corresponding inputs to a model. + + Args: + input_ids: Indices of input sequence tokens in the vocabulary. + attention_mask: Mask to avoid performing attention on padding token indices. + Mask values selected in `[0, 1]`: Usually `1` for tokens that are NOT MASKED, + `0` for MASKED (padded) tokens. + token_type_ids: (Optional) Segment token indices to indicate first and second + portions of the inputs. Only some models use them. + label: (Optional) Label corresponding to the input. Int for classification problems, + float for regression problems. + """ + + input_ids: List[int] + attention_mask: Optional[List[int]] = None + token_type_ids: Optional[List[int]] = None + labels: Optional[Union[int, float]] = None + + +class DataProcessor: + """Base class for data converters for sequence classification data sets.""" + + def get_train_examples(self, data_dir): + """Gets a collection of [`InputExample`] for the train set.""" + raise NotImplementedError() + + def get_dev_examples(self, data_dir): + """Gets a collection of [`InputExample`] for the dev set.""" + raise NotImplementedError() + + def get_test_examples(self, data_dir): + """Gets a collection of [`InputExample`] for the test set.""" + raise NotImplementedError() + + def get_labels(self): + """Gets the list of labels for this data set.""" + raise NotImplementedError() + + @classmethod + def _read_tsv(cls, input_file, quotechar=None): + """Reads a tab separated value file.""" + with open(input_file, "r", encoding="utf-8-sig") as f: + reader = csv.reader(f, delimiter="\t", quotechar=quotechar) + lines = [] + for line in reader: + lines.append(line) + return lines + + @classmethod + def _read_json(cls, input_file): + """Reads a json list file.""" + with open(input_file, "r") as f: + reader = f.readlines() + lines = [] + for line in reader: + lines.append(json.loads(line.strip())) + return lines diff --git a/projects/text_classification/dataset/utils_clue.py b/projects/text_classification/dataset/utils_clue.py new file mode 100644 index 0000000000000000000000000000000000000000..ba7a0d87b60ec97b7b538efc83f14e664e930715 --- /dev/null +++ b/projects/text_classification/dataset/utils_clue.py @@ -0,0 +1,490 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +from .utils import DataProcessor, EncodePattern, InputExample, InputFeatures + +logger = logging.getLogger(__name__) + + +def clue_convert_examples_to_features( + examples, + tokenizer, + max_length, + task=None, + pattern=EncodePattern.bert_pattern, + label_list=None, + output_mode=None, +): + if task is not None: + processor = clue_processors[task]() + if label_list is None: + label_list = processor.get_labels() + logger.info(f"Using label list {label_list} for task {task}") + if output_mode is None: + output_mode = clue_output_modes[task] + logger.info(f"Using output mode {output_mode} for task {task}") + + label_map = {label: i for i, label in enumerate(label_list)} + + start_token = [] if tokenizer.start_token is None else [tokenizer.start_token] + end_token = [] if tokenizer.end_token is None else [tokenizer.end_token] + pad_id = tokenizer.pad_token_id + + if pattern == EncodePattern.bert_pattern: + added_special_tokens = [2, 3] + elif pattern == EncodePattern.roberta_pattern: + added_special_tokens = [2, 4] + else: + raise KeyError("pattern is not a valid EncodePattern") + + features = [] + for (ex_index, example) in enumerate(examples): + if ex_index % 10000 == 0: + logger.info("Writing example %d of %d" % (ex_index, len(examples))) + + tokens_a = tokenizer.tokenize(example.text_a) + + tokens_b = None + if example.text_b: + tokens_b = tokenizer.tokenize(example.text_b) + _truncate_seq_pair(tokens_a, tokens_b, max_length - added_special_tokens[1]) + else: + if len(tokens_a) > max_length - added_special_tokens[0]: + tokens_a = tokens_a[: (max_length - added_special_tokens[0])] + + if pattern is EncodePattern.bert_pattern: + tokens = start_token + tokens_a + end_token + token_type_ids = [0] * len(tokens) + if tokens_b: + tokens += tokens_b + end_token + token_type_ids += [1] * (len(tokens) - len(token_type_ids)) + elif pattern is EncodePattern.roberta_pattern: + tokens = start_token + tokens_a + end_token + token_type_ids = [0] * len(tokens) + if tokens_b: + tokens += end_token + tokens_b + end_token + token_type_ids += [1] * (len(tokens) - len(token_type_ids)) + else: + raise KeyError("pattern is not a valid EncodePattern") + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + attention_mask = [1] * len(input_ids) + + padding_length = max_length - len(input_ids) + input_ids = input_ids + ([pad_id] * padding_length) + attention_mask = attention_mask + ([0] * padding_length) + token_type_ids = token_type_ids + ([0] * padding_length) + + label = None + if example.label is not None: + if output_mode == "classification": + label = label_map[example.label] + elif output_mode == "regression": + label = float(example.label) + + if ex_index < 5: + logger.info("*** Example ***") + logger.info("guid: %s" % (example.guid)) + logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + logger.info("attention_mask: %s" % " ".join([str(x) for x in attention_mask])) + logger.info("token_type_ids: %s" % " ".join([str(x) for x in token_type_ids])) + logger.info("label: %s (id = %d)" % (example.label, label)) + + features.append( + InputFeatures( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + labels=label, + ) + ) + + return features + + +def _truncate_seq_pair(tokens_a, tokens_b, max_length): + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + tokens_b.pop() + + +class TnewsProcessor(DataProcessor): + """Processor for the TNEWS data set (CLUE version). + Single sentence classification task. + The task is to predict which category the title belongs to. + """ + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "train.json")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "dev.json")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "test.json")), "test") + + def get_labels(self): + """See base class.""" + labels = [] + for i in range(17): + if i == 5 or i == 11: + continue + labels.append(str(100 + i)) + return labels + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + guid = f"{set_type}-{i}" + text_a = line["sentence"] + label = None if set_type == "test" else str(line["label"]) + examples.append(InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + + +class IflytekProcessor(DataProcessor): + """Processor for the IFLYTEK data set (CLUE version). + Single sentence classification task. + The task is to predict the categories according to discription. + """ + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "train.json")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "dev.json")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "test.json")), "test") + + def get_labels(self): + """See base class.""" + labels = [] + for i in range(119): + labels.append(str(i)) + return labels + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + guid = f"{set_type}-{i}" + text_a = line["sentence"] + label = None if set_type == "test" else str(line["label"]) + examples.append(InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + + +class AfqmcProcessor(DataProcessor): + """Processor for the AFQMC data set (CLUE version). + Sentence pair classification task. + This task is to predict whether two sentences are semantically similar. + """ + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "train.json")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "dev.json")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "test.json")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + guid = f"{set_type}-{i}" + text_a = line["sentence1"] + text_b = line["sentence2"] + label = None if set_type == "test" else str(line["label"]) + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class OcnliProcessor(DataProcessor): + """Processor for the OCNLI data set (CLUE version). + Sentence pair classification task. + Given a premise sentence and a hypothesis sentence, + the task is to predict whether the premise entails the hypothesis (entailment), + contradicts the hypothesis (contradiction), or neither (neutral). + """ + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "train.json")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "dev.json")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "test.json")), "test") + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + guid = f"{set_type}-{i}" + text_a = line["sentence1"] + text_b = line["sentence2"] + label = None if set_type == "test" else str(line["label"]) + if label.strip() == "-": + continue + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class CmnliProcessor(DataProcessor): + """Processor for the CMNLI data set (CLUE version). + Sentence pair classification task. + Given a premise sentence and a hypothesis sentence, + the task is to predict whether the premise entails the hypothesis (entailment), + contradicts the hypothesis (contradiction), or neither (neutral). + """ + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "train.json")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "dev.json")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "test.json")), "test") + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + guid = f"{set_type}-{i}" + text_a = line["sentence1"] + text_b = line["sentence2"] + label = None if set_type == "test" else str(line["label"]) + if label.strip() == "-": + continue + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class CslProcessor(DataProcessor): + """Processor for the CSL data set (CLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "train.json")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "dev.json")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "test.json")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + guid = f"{set_type}-{i}" + text_a = " ".join(line["keyword"]) + text_b = line["abst"] + label = None if set_type == "test" else str(line["label"]) + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class WscProcessor(DataProcessor): + """Processor for the WSC data set (CLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "train.json")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "dev.json")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "test.json")), "test") + + def get_labels(self): + """See base class.""" + return ["true", "false"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + guid = f"{set_type}-{i}" + text_a = line["text"] + text_a_list = list(text_a) + target = line["target"] + query = target["span1_text"] + query_idx = target["span1_index"] + pronoun = target["span2_text"] + pronoun_idx = target["span2_index"] + assert ( + text_a[pronoun_idx : (pronoun_idx + len(pronoun))] == pronoun + ), "pronoun: {}".format(pronoun) + assert text_a[query_idx : (query_idx + len(query))] == query, "query: {}".format(query) + if pronoun_idx > query_idx: + text_a_list.insert(query_idx, "_") + text_a_list.insert(query_idx + len(query) + 1, "_") + text_a_list.insert(pronoun_idx + 2, "[") + text_a_list.insert(pronoun_idx + len(pronoun) + 2 + 1, "]") + else: + text_a_list.insert(pronoun_idx, "[") + text_a_list.insert(pronoun_idx + len(pronoun) + 1, "]") + text_a_list.insert(query_idx + 2, "_") + text_a_list.insert(query_idx + len(query) + 2 + 1, "_") + text_a = "".join(text_a_list) + label = None if set_type == "test" else str(line["label"]) + examples.append(InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + + +class CopaProcessor(DataProcessor): + """Processor for the COPA data set (CLUE version).""" + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "train.json")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "dev.json")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_json(os.path.join(data_dir, "test.json")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + examples = [] + for (i, line) in enumerate(lines): + i = 2 * i + guid1 = f"{set_type}-{i}" + guid2 = "%s-%s" % (set_type, i + 1) + premise = line["premise"] + choice0 = line["choice0"] + label = None if set_type == "test" else str(1 if line["label"] == 0 else 0) + choice1 = line["choice1"] + label2 = None if set_type == "test" else str(1 if line["label"] == 0 else 0) + if line["question"] == "effect": + text_a = premise + text_b = choice0 + text_a2 = premise + text_b2 = choice1 + elif line["question"] == "cause": + text_a = choice0 + text_b = premise + text_a2 = choice1 + text_b2 = premise + else: + raise ValueError(f'unknowed {line["question"]} type') + examples.append(InputExample(guid=guid1, text_a=text_a, text_b=text_b, label=label)) + examples.append(InputExample(guid=guid2, text_a=text_a2, text_b=text_b2, label=label2)) + return examples + + def _create_examples_version2(self, lines, set_type): + """Creates examples for the training and dev sets.""" + examples = [] + for (i, line) in enumerate(lines): + guid = f"{set_type}-{i}" + if line["question"] == "cause": + text_a = line["premise"] + "这是什么原因造成的？" + line["choice0"] + text_b = line["premise"] + "这是什么原因造成的？" + line["choice1"] + else: + text_a = line["premise"] + "这造成了什么影响？" + line["choice0"] + text_b = line["premise"] + "这造成了什么影响？" + line["choice1"] + label = None if set_type == "test" else str(1 if line["label"] == 0 else 0) + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +clue_tasks_num_labels = { + "iflytek": 119, + "cmnli": 3, + "ocnli": 3, + "afqmc": 2, + "csl": 2, + "wsc": 2, + "copa": 2, + "tnews": 15, +} + +clue_processors = { + "tnews": TnewsProcessor, + "iflytek": IflytekProcessor, + "cmnli": CmnliProcessor, + "ocnli": OcnliProcessor, + "afqmc": AfqmcProcessor, + "csl": CslProcessor, + "wsc": WscProcessor, + "copa": CopaProcessor, +} + +clue_output_modes = { + "tnews": "classification", + "iflytek": "classification", + "cmnli": "classification", + "ocnli": "classification", + "afqmc": "classification", + "csl": "classification", + "wsc": "classification", + "copa": "classification", +} diff --git a/projects/text_classification/dataset/utils_glue.py b/projects/text_classification/dataset/utils_glue.py new file mode 100644 index 0000000000000000000000000000000000000000..bb41f1bb72ae99e6bb69d3bb195a661a2dbb73cb --- /dev/null +++ b/projects/text_classification/dataset/utils_glue.py @@ -0,0 +1,525 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +from .utils import DataProcessor, EncodePattern, InputExample, InputFeatures + +logger = logging.getLogger(__name__) + + +def glue_convert_examples_to_features( + examples, + tokenizer, + max_length, + task=None, + pattern=EncodePattern.bert_pattern, + label_list=None, + output_mode=None, +): + if task is not None: + processor = glue_processors[task]() + if label_list is None: + label_list = processor.get_labels() + logger.info(f"Using label list {label_list} for task {task}") + if output_mode is None: + output_mode = glue_output_modes[task] + logger.info(f"Using output mode {output_mode} for task {task}") + + label_map = {label: i for i, label in enumerate(label_list)} + + start_token = [] if tokenizer.start_token is None else [tokenizer.start_token] + end_token = [] if tokenizer.end_token is None else [tokenizer.end_token] + pad_id = tokenizer.pad_token_id + + if pattern == EncodePattern.bert_pattern: + added_special_tokens = [2, 3] + elif pattern == EncodePattern.roberta_pattern: + added_special_tokens = [2, 4] + else: + raise KeyError("pattern is not a valid EncodePattern") + + features = [] + for (ex_index, example) in enumerate(examples): + if ex_index % 10000 == 0: + logger.info("Writing example %d of %d" % (ex_index, len(examples))) + + tokens_a = tokenizer.tokenize(example.text_a) + + tokens_b = None + if example.text_b: + tokens_b = tokenizer.tokenize(example.text_b) + _truncate_seq_pair(tokens_a, tokens_b, max_length - added_special_tokens[1]) + else: + if len(tokens_a) > max_length - added_special_tokens[0]: + tokens_a = tokens_a[: (max_length - added_special_tokens[0])] + + if pattern is EncodePattern.bert_pattern: + tokens = start_token + tokens_a + end_token + token_type_ids = [0] * len(tokens) + if tokens_b: + tokens += tokens_b + end_token + token_type_ids += [1] * (len(tokens) - len(token_type_ids)) + elif pattern is EncodePattern.roberta_pattern: + tokens = start_token + tokens_a + end_token + token_type_ids = [0] * len(tokens) + if tokens_b: + tokens += end_token + tokens_b + end_token + token_type_ids += [1] * (len(tokens) - len(token_type_ids)) + else: + raise KeyError("pattern is not a valid EncodePattern") + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + attention_mask = [1] * len(input_ids) + + padding_length = max_length - len(input_ids) + input_ids = input_ids + ([pad_id] * padding_length) + attention_mask = attention_mask + ([0] * padding_length) + token_type_ids = token_type_ids + ([0] * padding_length) + + label = None + if example.label is not None: + if output_mode == "classification": + label = label_map[example.label] + elif output_mode == "regression": + label = float(example.label) + + if ex_index < 5: + logger.info("*** Example ***") + logger.info("guid: %s" % (example.guid)) + logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + logger.info("attention_mask: %s" % " ".join([str(x) for x in attention_mask])) + logger.info("token_type_ids: %s" % " ".join([str(x) for x in token_type_ids])) + logger.info("label: %s (id = %d)" % (example.label, label)) + + features.append( + InputFeatures( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + labels=label, + ) + ) + + return features + + +def _truncate_seq_pair(tokens_a, tokens_b, max_length): + while True: + total_length = len(tokens_a) + len(tokens_b) + if total_length <= max_length: + break + if len(tokens_a) > len(tokens_b): + tokens_a.pop() + else: + tokens_b.pop() + + +class MrpcProcessor(DataProcessor): + """Processor for the MRPC data set (GLUE version). + Sentence pair classification task. + Determine whether the two sentences have the same meaning. + """ + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training, dev and test sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = f"{set_type}-{i}" + text_a = line[3] + text_b = line[4] + label = None if set_type == "test" else line[0] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class MnliProcessor(DataProcessor): + """Processor for the MultiNLI data set (GLUE version). + Sentence pair classification task. + Given a premise sentence and a hypothesis sentence, + the task is to predict whether the premise entails the hypothesis (entailment), + contradicts the hypothesis (contradiction), or neither (neutral). + """ + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev_matched.tsv")), "dev_matched" + ) + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "test_matched.tsv")), "test_matched" + ) + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training, dev and test sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = f"{set_type}-{line[0]}" + text_a = line[8] + text_b = line[9] + label = None if set_type.startswith("test") else line[-1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class MnliMismatchedProcessor(MnliProcessor): + """Processor for the MultiNLI Mismatched data set (GLUE version).""" + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "dev_mismatched.tsv")), + "dev_mismatched", + ) + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples( + self._read_tsv(os.path.join(data_dir, "test_mismatched.tsv")), + "test_mismatched", + ) + + +class ColaProcessor(DataProcessor): + """Processor for the CoLA data set (GLUE version). + Single sentence classification task. + Each example is a sequence of words annotated with whether it is a grammatical English sentence. + """ + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training, dev and test sets.""" + test_mode = set_type == "test" + if test_mode: + lines = lines[1:] + text_index = 1 if test_mode else 3 + examples = [] + for (i, line) in enumerate(lines): + guid = f"{set_type}-{i}" + text_a = line[text_index] + label = None if test_mode else line[1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + + +class Sst2Processor(DataProcessor): + """Processor for the SST-2 data set (GLUE version). + Single sentence classification task. + The task is to predict the sentiment of a given sentence. + We use the two-way (positive/negative) class split, and use only sentence-level labels. + """ + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training, dev and test sets.""" + examples = [] + text_index = 1 if set_type == "test" else 0 + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = f"{set_type}-{i}" + text_a = line[text_index] + label = None if set_type == "test" else line[1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + return examples + + +class StsbProcessor(DataProcessor): + """Processor for the STS-B data set (GLUE version). + Sentence pair task but it is a regression task. + This task is to predict the similarity score of two sentences. + """ + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_labels(self): + """See base class.""" + return [None] + + def _create_examples(self, lines, set_type): + """Creates examples for the training, dev and test sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = f"{set_type}-{line[0]}" + text_a = line[7] + text_b = line[8] + label = None if set_type == "test" else line[-1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class QqpProcessor(DataProcessor): + """Processor for the QQP data set (GLUE version). + Sentence pair classification task. + The task is to determine whether a pair of questions are semantically equivalent. + """ + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training, dev and test sets.""" + test_mode = set_type == "test" + q1_index = 1 if test_mode else 3 + q2_index = 2 if test_mode else 4 + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = f"{set_type}-{line[0]}" + try: + text_a = line[q1_index] + text_b = line[q2_index] + label = None if test_mode else line[5] + except IndexError: + continue + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class QnliProcessor(DataProcessor): + """Processor for the QNLI data set (GLUE version). + Sentence pair classification task. + The task is to determine whether the context sentence contains the answer to the question. + """ + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["entailment", "not_entailment"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training, dev and test sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = f"{set_type}-{line[0]}" + text_a = line[1] + text_b = line[2] + label = None if set_type == "test" else line[-1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class RteProcessor(DataProcessor): + """Processor for the RTE data set (GLUE version). + Sentence pair classification task. + Recognizing Textual Entailment. + Predict whether the two sentences is entailment or not entailment (neutral and contradiction). + """ + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["entailment", "not_entailment"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training, dev and test sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = f"{set_type}-{line[0]}" + text_a = line[1] + text_b = line[2] + label = None if set_type == "test" else line[-1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +class WnliProcessor(DataProcessor): + """Processor for the WNLI data set (GLUE version). + Sentence pair classification task. + The task is to predict if the sentence with the pronoun substituted is entailed + by the original sentence. + """ + + def get_train_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "train.tsv")), "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev") + + def get_test_examples(self, data_dir): + """See base class.""" + return self._create_examples(self._read_tsv(os.path.join(data_dir, "test.tsv")), "test") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, lines, set_type): + """Creates examples for the training, dev and test sets.""" + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = f"{set_type}-{line[0]}" + text_a = line[1] + text_b = line[2] + label = None if set_type == "test" else line[-1] + examples.append(InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + +glue_tasks_num_labels = { + "cola": 2, + "mnli": 3, + "mrpc": 2, + "sst-2": 2, + "sts-b": 1, + "qqp": 2, + "qnli": 2, + "rte": 2, + "wnli": 2, +} + +glue_processors = { + "cola": ColaProcessor, + "mnli": MnliProcessor, + "mnli-mm": MnliMismatchedProcessor, + "mrpc": MrpcProcessor, + "sst-2": Sst2Processor, + "sts-b": StsbProcessor, + "qqp": QqpProcessor, + "qnli": QnliProcessor, + "rte": RteProcessor, + "wnli": WnliProcessor, +} + +glue_output_modes = { + "cola": "classification", + "mnli": "classification", + "mnli-mm": "classification", + "mrpc": "classification", + "sst-2": "classification", + "sts-b": "regression", + "qqp": "classification", + "qnli": "classification", + "rte": "classification", + "wnli": "classification", +} diff --git a/projects/text_classification/modeling/load_megatron_weight.py b/projects/text_classification/modeling/load_megatron_weight.py new file mode 100644 index 0000000000000000000000000000000000000000..2303ebf59b069208ef3380354054f32c809193ae --- /dev/null +++ b/projects/text_classification/modeling/load_megatron_weight.py @@ -0,0 +1,143 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import oneflow as flow +import torch + +from libai.utils.checkpoint import get_missing_parameters_message, get_unexpected_parameters_message + +logger = logging.getLogger("libai." + __name__) + + +def convert_tensor(tensor: torch.Tensor): + tensor = tensor.float() + return flow.Tensor(tensor.cpu().numpy()) + + +def change_megatron_key(state_dict): + of_state_dict = {} + + # Language model. + language_model = state_dict["language_model"] + + # Embedding. + embedding = language_model["embedding"] + of_state_dict["embeddings.vocab_embeddings.weight"] = convert_tensor( + embedding["word_embeddings"]["weight"] + ) + of_state_dict["embeddings.position_embeddings.weight"] = convert_tensor( + embedding["position_embeddings"]["weight"] + ) + of_state_dict["embeddings.tokentype_embeddings.weight"] = convert_tensor( + embedding["tokentype_embeddings"]["weight"] + ) + + # Encoder. + encoder = language_model["encoder"] + for key, value in encoder.items(): + # Change layers.0.input_layernorm.weight -> encoder.layers_0.input_layernorm.weight + key = "encoders." + key.replace("layers.", "") + if key.startswith("encoders.final_layernorm"): + key = key.replace("encoders.", "") + of_state_dict[key] = convert_tensor(value) + + # Pooler. + pooler = language_model["pooler"] + of_state_dict["pooler.dense.weight"] = convert_tensor(pooler["dense.weight"]) + of_state_dict["pooler.dense.bias"] = convert_tensor(pooler["dense.bias"]) + + # LM head. + lm_head = state_dict["lm_head"] + of_state_dict["cls.predictions.dense.weight"] = convert_tensor(lm_head["dense.weight"]) + of_state_dict["cls.predictions.dense.bias"] = convert_tensor(lm_head["dense.bias"]) + + of_state_dict["cls.predictions.layernorm.weight"] = convert_tensor(lm_head["layernorm.weight"]) + of_state_dict["cls.predictions.layernorm.bias"] = convert_tensor(lm_head["layernorm.bias"]) + + of_state_dict["lm_logits.bias"] = convert_tensor(lm_head["bias"]) + + # Binary head. + binary_head = state_dict["binary_head"] + of_state_dict["cls.seq_relationship.weight"] = convert_tensor(binary_head["weight"]) + of_state_dict["cls.seq_relationship.bias"] = convert_tensor((binary_head["bias"])) + + return of_state_dict + + +def load_tensor(tensor_lhs, tensor_rhs): + tensor_rhs = flow.to_global(tensor_rhs, placement=tensor_lhs.placement, sbp=tensor_lhs.sbp) + tensor_lhs.copy_(tensor_rhs) + + +def load_model(model: flow.nn.Module, state_dict): + model_state_dict = model.state_dict() + + # Decide shape + incorrect_shapes = [] + for k in list(state_dict.keys()): + if k in model_state_dict: + if ( + (k.find("weight") != -1) + and (k.find("embeddings") == -1) + and (k.find("layernorm") == -1) + ): + # Transpose from (M, N) -> (N, M), because the weight + # shape in megatron and oneflow missing one transpose. + shape_model = tuple(model_state_dict[k].shape[::-1]) + else: + shape_model = tuple(model_state_dict[k].shape) + shape_ckpt = tuple(state_dict[k].shape) + if shape_model != shape_ckpt: + incorrect_shapes.append((k, shape_ckpt, shape_model)) + state_dict.pop(k) + + unexpected_keys = [] + for key, value in state_dict.items(): + if key not in model_state_dict: + unexpected_keys.append(key) + continue + model_state_dict.pop(key) + if ( + (key.find("weight") != -1) + and (key.find("embeddings") == -1) + and (key.find("layernorm") == -1) + ): + value = flow.transpose(value, 0, 1) + load_tensor(model.state_dict()[key], value) + + missing_keys = list(model_state_dict.keys()) + + for k, shape_checkpoint, shape_model in incorrect_shapes: + logger.warning( + "Skip loading parameter '{}' to the model due to incompatible " + "shapes: {} in the checkpoint but {} in the " + "model! You might want to double check if this is expected.".format( + k, shape_checkpoint, shape_model + ) + ) + if missing_keys: + logger.info(get_missing_parameters_message(missing_keys)) + if unexpected_keys: + logger.info(get_unexpected_parameters_message(unexpected_keys)) + + +def load_megatron_bert(model: flow.nn.Module, model_weight_path: str): + import torch + + megatron_state_dict = torch.load(model_weight_path, map_location="cpu")["model"] + of_state_dict = change_megatron_key(megatron_state_dict) + load_model(model, of_state_dict) diff --git a/projects/text_classification/modeling/model.py b/projects/text_classification/modeling/model.py new file mode 100644 index 0000000000000000000000000000000000000000..1ce4f7d27bfbc15460022fc734700f06bb40a5f0 --- /dev/null +++ b/projects/text_classification/modeling/model.py @@ -0,0 +1,77 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +import oneflow as flow +from oneflow import nn + +from libai.layers import Linear +from libai.models.bert_model import BertModel +from libai.models.utils import init_method_normal +from libai.utils import distributed as dist + +logger = logging.getLogger("libai." + __name__) + + +class ClassificationLoss(nn.Module): + def __init__(self): + super().__init__() + + def forward(self, classification_logits, label): + loss = nn.CrossEntropyLoss()(classification_logits, label) + # NOTE: Change loss sbp sign [P, P] -> [P, B] to add with sop loss + # whose sbp sign: [P, B] + loss = loss.to_global(sbp=dist.get_nd_sbp([flow.sbp.partial_sum, flow.sbp.broadcast])) + return loss + + +class ModelForSequenceClassification(nn.Module): + def __init__(self, cfg): + super().__init__() + self.num_classes = cfg.num_classes + self.model = BertModel(cfg) + if cfg.pretrain_megatron_weight is not None: + from .load_megatron_weight import load_megatron_bert + + logger.info(f"loading pretraining: {cfg.pretrain_megatron_weight}") + load_megatron_bert(self.model, cfg.pretrain_megatron_weight) + logger.info("load succeed") + + init_method = init_method_normal(cfg.initializer_range) + self.dropout = nn.Dropout(cfg.hidden_dropout_prob) + + self.classifier = Linear( + cfg.hidden_size, + self.num_classes, + bias=True, + parallel="row", + init_method=init_method, + layer_idx=-1, + ) + self.loss_fct = ClassificationLoss() + + def forward(self, input_ids, attention_mask, token_type_ids=None, labels=None): + + encoder_output, pooled_output = self.model(input_ids, attention_mask, token_type_ids) + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + + if self.training and labels is not None: + loss = self.loss_fct(logits.view(-1, self.num_classes), labels.view(-1)) + loss_dict = {"loss": loss} + return loss_dict + + return {"prediction_scores": logits} diff --git a/projects/text_classification/requirements.txt b/projects/text_classification/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..715e23a79d00e785ff4ee89dd3f3b99539b603c5 --- /dev/null +++ b/projects/text_classification/requirements.txt @@ -0,0 +1,2 @@ +filelock +scikit-learn \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b9e8ff3c86d0ffe0b09a8ff55e45802711658a3a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,28 @@ +boto3 +botocore +cloudpickle +flowvision==0.1.0 +wget +hydra-core +nltk +numpy==1.23.4 +omegaconf==2.1.0 +Pygments +PyYAML +jieba +regex +requests +scipy +sentencepiece>=0.1 +tabulate +termcolor +tqdm +pybind11 +portalocker +dill +flake8==3.8.1 +isort==5.10.1 +black==21.4b2 +autoflake +tensorboardX +pytest diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000000000000000000000000000000000..6ea27db05c4a331e232a877860f818c6fc95e900 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,20 @@ +[isort] +line_length=100 +multi_line_output=3 +include_trailing_comma=True +skip=./datasets,docs +skip_glob=*/__init__.py,**/configs/**,tests/config/** + +[mypy] +python_version=3.6 +ignore_missing_imports = True +warn_unused_configs = True +disallow_untyped_defs = True +check_untyped_defs = True +warn_unused_ignores = True +warn_redundant_casts = True +show_column_numbers = True +follow_imports = silent +allow_redefinition = True +; Require all functions to be annotated +disallow_incomplete_defs = True \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..a4234b08fca0b0b8792d4dc28e6b58b946603321 --- /dev/null +++ b/setup.py @@ -0,0 +1,143 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import os +import shutil +import subprocess +import sys +from os import path +from typing import List + +from setuptools import Extension, find_packages, setup + +version = "0.2.0" +package_name = "LiBai" +cwd = os.path.dirname(os.path.abspath(__file__)) + +sha = "Unknown" +try: + sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=cwd).decode("ascii").strip() +except Exception: + pass + + +def write_version_file(): + version_path = os.path.join(cwd, "libai", "version.py") + with open(version_path, "w") as f: + f.write(f"__version__ = '{version}'\n") + f.write(f"git_version = {repr(sha)}\n") + + +if sys.version_info < (3,): + sys.exit("Sorry, Python3 is required for LiBai.") + + +def get_pybind11(): + import pybind11 as pb + + return pb + + +extensions = [ + Extension( + "libai.data.data_utils.helpers", + sources=["libai/data/data_utils/helpers.cpp"], + extra_compile_args=[ + "-O3", + "-Wall", + "-shared", + "-std=c++11", + "-fPIC", + "-fdiagnostics-color", + ], + include_dirs=[get_pybind11().get_include()], + ), +] + + +def get_libai_configs() -> List[str]: + """ + Return a list of configs to include in package for model zoo. + """ + source_configs_dir = path.join(path.dirname(path.realpath(__file__)), "configs") + destination = path.join(path.dirname(path.realpath(__file__)), "libai", "config", "configs") + # Symlink the config directory inside package to have a cleaner pip install. + + # Remove stale symlink/directory from a previous build. + if path.exists(source_configs_dir): + if path.islink(destination): + os.unlink(destination) + elif path.isdir(destination): + shutil.rmtree(destination) + + if not path.exists(destination): + try: + os.symlink(source_configs_dir, destination) + except OSError: + # Fall back to copying if symlink fails: ex. on Windows. + shutil.copytree(source_configs_dir, destination) + config_paths = glob.glob("configs/**/*.py", recursive=True) + return config_paths + + +if __name__ == "__main__": + print(f"Building wheel {package_name}-{version}") + + with open("LICENSE", "r", encoding="utf-8") as f: + license = f.read() + + write_version_file() + + setup( + name=package_name, + version=version, + description="Toolkit for Pretraining Models with OneFlow", + license=license, + install_requires=[ + "boto3", + "botocore", + "cloudpickle", + "flowvision==0.1.0", + "wget", + "hydra-core", + "nltk", + "numpy==1.23.4", + "omegaconf==2.1.0", + "Pygments", + "PyYAML", + "jieba", + "regex", + "requests", + "scipy", + "sentencepiece>=0.1", + "tabulate", + "termcolor", + "tqdm", + "pybind11", + "portalocker", + "dill", + "flake8==3.8.1 ", + "isort==5.10.1", + "black==21.4b ", + "autoflake", + "tensorboardX", + "pytest", + ], + packages=find_packages(), + package_data={"libai.config": get_libai_configs()}, + ext_modules=extensions, + test_suite="tests", + ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/config/dir1/dir1_a.py b/tests/config/dir1/dir1_a.py new file mode 100644 index 0000000000000000000000000000000000000000..a939955124556355524f48c0f0c16abb07cfc4c4 --- /dev/null +++ b/tests/config/dir1/dir1_a.py @@ -0,0 +1,3 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +dir1a_str = "base_a_1" +dir1a_dict = {"a": 1, "b": 2} diff --git a/tests/config/dir1/dir1_b.py b/tests/config/dir1/dir1_b.py new file mode 100644 index 0000000000000000000000000000000000000000..a5e7d009d7a5cb3922726f31ac368b164527b2a6 --- /dev/null +++ b/tests/config/dir1/dir1_b.py @@ -0,0 +1,11 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from libai.config import LazyConfig + +# equivalent to relative import +dir1a_str, dir1a_dict = LazyConfig.load_rel("dir1_a.py", ("dir1a_str", "dir1a_dict")) + +dir1b_str = dir1a_str + "_from_b" +dir1b_dict = dir1a_dict + +# Every import is a reload: not modified by other config files +assert dir1a_dict.a == 1 diff --git a/tests/config/root_cfg.py b/tests/config/root_cfg.py new file mode 100644 index 0000000000000000000000000000000000000000..99d1307d046e579239e95c15bdf165f50240ad34 --- /dev/null +++ b/tests/config/root_cfg.py @@ -0,0 +1,14 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from itertools import count + +from libai.config import LazyCall + +from .dir1.dir1_a import dir1a_dict, dir1a_str + +dir1a_dict.a = "modified" + +# modification above won't affect future imports +from .dir1.dir1_b import dir1b_dict, dir1b_str + + +lazyobj = LazyCall(count)(x=dir1a_str, y=dir1b_str) diff --git a/tests/config/test_instantiate_config.py b/tests/config/test_instantiate_config.py new file mode 100644 index 0000000000000000000000000000000000000000..9a44a9e8d9e7481a9260687f339222aba72ce5c3 --- /dev/null +++ b/tests/config/test_instantiate_config.py @@ -0,0 +1,148 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unittests followed +https://github.com/facebookresearch/detectron2/blob/main/tests/config/test_instantiate_config.py +""" + +from collections import namedtuple +import os +import unittest +import yaml +import tempfile +from libai.config import instantiate, LazyCall +from omegaconf import OmegaConf +from dataclasses import dataclass +from omegaconf import __version__ as oc_version + +OC_VERSION = tuple(int(x) for x in oc_version.split(".")[:2]) + + +class ShapeSpec(namedtuple("_ShapeSpec", ["channels", "width"])): + """ + A simple structure that contains basic shape specification about a tensor. + It is often used as the auxiliary inputs/outputs of models, + to complement the lack of shape inference ability among pytorch modules. + Attributes: + channels: + width: + """ + + def __new__(cls, channels=None, width=None): + return super().__new__(cls, channels, width) + + +class TestClass: + def __init__(self, int_arg, list_arg=None, dict_arg=None, extra_arg=None) -> None: + self.int_arg = int_arg + self.list_arg = list_arg + self.dict_arg = dict_arg + self.extra_arg = extra_arg + + def __call__(self, call_arg): + return call_arg + self.int_arg + + +@dataclass +class TestDataClass: + x: int + y: str + + +@unittest.skipIf(OC_VERSION < (2, 1), "omegaconf version too old") +class TestConstruction(unittest.TestCase): + def test_basic_construct(self): + objconf = LazyCall(TestClass)( + int_arg=3, + list_arg=[10], + dict_arg={}, + extra_arg=LazyCall(TestClass)(int_arg=4, list_arg="${..list_arg}"), + ) + + obj = instantiate(objconf) + self.assertIsInstance(obj, TestClass) + self.assertEqual(obj.int_arg, 3) + self.assertEqual(obj.extra_arg.int_arg, 4) + self.assertEqual(obj.extra_arg.list_arg, obj.list_arg) + + objconf.extra_arg.list_arg = [5] + obj = instantiate(objconf) + self.assertIsInstance(obj, TestClass) + self.assertEqual(obj.extra_arg.list_arg, [5]) + + def test_instantiate_other_obj(self): + # do nothing for other obj + self.assertEqual(instantiate(5), 5) + x = [3, 4, 5] + self.assertEqual(instantiate(x), x) + x = TestClass(1) + self.assertIs(instantiate(x), x) + x = {"xx": "yy"} + self.assertEqual(instantiate(x), x) + + def test_instantiate_lazy_target(self): + # _target_ is result of instantiate + objconf = LazyCall(LazyCall(len)(int_arg=3))(call_arg=4) + objconf._target_._target_ = TestClass + self.assertEqual(instantiate(objconf), 7) + + def test_instantiate_lst(self): + lst = [1, 2, LazyCall(TestClass)(int_arg=1)] + x = LazyCall(TestClass)( + int_arg=lst + ) # list as an argument should be recursively instantiated + x = instantiate(x).int_arg + self.assertEqual(x[:2], [1, 2]) + self.assertIsInstance(x[2], TestClass) + self.assertEqual(x[2].int_arg, 1) + + def test_instantiate_namedtuple(self): + x = LazyCall(TestClass)(int_arg=ShapeSpec(channels=1, width=3)) + # test serialization + with tempfile.TemporaryDirectory() as d: + fname = os.path.join(d, "lb_test.yaml") + OmegaConf.save(x, fname) + with open(fname) as f: + x = yaml.unsafe_load(f) + + x = instantiate(x) + self.assertIsInstance(x.int_arg, ShapeSpec) + self.assertEqual(x.int_arg.channels, 1) + + def test_bad_lazycall(self): + with self.assertRaises(Exception): + LazyCall(3) + + def test_instantiate_dataclass(self): + a = LazyCall(TestDataClass)(x=1, y="s") + a = instantiate(a) + self.assertEqual(a.x, 1) + self.assertEqual(a.y, "s") + + def test_instantiate_no_recursive(self): + def helper_func(obj): + self.assertNotIsInstance(obj, TestClass) + obj = instantiate(obj) + self.assertIsInstance(obj, TestClass) + return obj.int_arg + + objconf = LazyCall(helper_func)(obj=LazyCall(TestClass)(int_arg=4)) + self.assertEqual(instantiate(objconf, _recursive_=False), 4) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/config/test_lazy_config.py b/tests/config/test_lazy_config.py new file mode 100644 index 0000000000000000000000000000000000000000..55e149d3b957d0947f8f10ac3737fd84c15c4016 --- /dev/null +++ b/tests/config/test_lazy_config.py @@ -0,0 +1,102 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unittests followed +https://github.com/facebookresearch/detectron2/blob/main/tests/config/test_lazy_config.py +""" +import os +import unittest +import tempfile +from itertools import count + +from libai.config import LazyConfig, LazyCall +from omegaconf import DictConfig + + +class TestLazyPythonConfig(unittest.TestCase): + def setUp(self): + self.root_filename = os.path.join(os.path.dirname(__file__), "root_cfg.py") + + def test_load(self): + cfg = LazyConfig.load(self.root_filename) + + self.assertEqual(cfg.dir1a_dict.a, "modified") + self.assertEqual(cfg.dir1b_dict.a, 1) + self.assertEqual(cfg.lazyobj.x, "base_a_1") + + cfg.lazyobj.x = "new_x" + # reload + cfg = LazyConfig.load(self.root_filename) + self.assertEqual(cfg.lazyobj.x, "base_a_1") + + def test_save_load(self): + cfg = LazyConfig.load(self.root_filename) + with tempfile.TemporaryDirectory(prefix="detectron2") as d: + fname = os.path.join(d, "test_config.yaml") + LazyConfig.save(cfg, fname) + cfg2 = LazyConfig.load(fname) + + self.assertEqual(cfg2.lazyobj._target_, "itertools.count") + self.assertEqual(cfg.lazyobj._target_, count) + cfg2.lazyobj.pop("_target_") + cfg.lazyobj.pop("_target_") + # the rest are equal + self.assertEqual(cfg, cfg2) + + def test_failed_save(self): + cfg = DictConfig({"x": lambda: 3}, flags={"allow_objects": True}) + with tempfile.TemporaryDirectory(prefix="detectron2") as d: + fname = os.path.join(d, "test_config.yaml") + LazyConfig.save(cfg, fname) + self.assertTrue(os.path.exists(fname)) + self.assertTrue(os.path.exists(fname + ".pkl")) + + def test_overrides(self): + cfg = LazyConfig.load(self.root_filename) + LazyConfig.apply_overrides(cfg, ["lazyobj.x=123", 'dir1b_dict.a="123"']) + self.assertEqual(cfg.dir1b_dict.a, "123") + self.assertEqual(cfg.lazyobj.x, 123) + + def test_invalid_overrides(self): + cfg = LazyConfig.load(self.root_filename) + with self.assertRaises(KeyError): + LazyConfig.apply_overrides(cfg, ["lazyobj.x.xxx=123"]) + + def test_to_py(self): + cfg = LazyConfig.load(self.root_filename) + cfg.lazyobj.x = { + "a": 1, + "b": 2, + "c": LazyCall(count)(x={"r": "a", "s": 2.4, "t": [1, 2, 3, "z"]}), + } + cfg.list = ["a", 1, "b", 3.2] + py_str = LazyConfig.to_py(cfg) + expected = """cfg.dir1a_dict.a = "modified" +cfg.dir1a_dict.b = 2 +cfg.dir1b_dict.a = 1 +cfg.dir1b_dict.b = 2 +cfg.lazyobj = itertools.count( + x={ + "a": 1, + "b": 2, + "c": itertools.count(x={"r": "a", "s": 2.4, "t": [1, 2, 3, "z"]}), + }, + y="base_a_1_from_b", +) +cfg.list = ["a", 1, "b", 3.2] +""" + self.assertEqual(py_str, expected) diff --git a/tests/data/__init__.py b/tests/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/data/test_sampler.py b/tests/data/test_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..70b794ebb7ff1bc8287a4bb55584c159eea1e080 --- /dev/null +++ b/tests/data/test_sampler.py @@ -0,0 +1,193 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import itertools +import unittest + +import oneflow.utils.data as flowdata + +from libai.data.samplers import CyclicSampler, SingleRoundSampler + + +class TestCyclicSampler(unittest.TestCase): + def test_cyclic_sampler_iterable(self): + sampler = CyclicSampler( + list(range(100)), + micro_batch_size=4, + shuffle=True, + consumed_samples=0, + seed=123, + ) + output_iter = itertools.islice(sampler, 25) # iteration=100/4=25 + sample_output = list() + for batch in output_iter: + sample_output.extend(batch) + self.assertEqual(set(sample_output), set(range(100))) + + data_sampler = CyclicSampler( + list(range(100)), + micro_batch_size=4, + shuffle=True, + consumed_samples=0, + seed=123, + ) + + data_loader = flowdata.DataLoader( + list(range(100)), batch_sampler=data_sampler, num_workers=0, collate_fn=lambda x: x + ) + + data_loader_iter = itertools.islice(data_loader, 25) + output = list() + for data in data_loader_iter: + output.extend(data) + self.assertEqual(output, sample_output) + + def test_cyclic_sampler_seed(self): + sampler = CyclicSampler( + list(range(100)), + micro_batch_size=4, + shuffle=True, + seed=123, + ) + + data = list(itertools.islice(sampler, 65)) + + sampler = CyclicSampler( + list(range(100)), + micro_batch_size=4, + shuffle=True, + seed=123, + ) + + data2 = list(itertools.islice(sampler, 65)) + self.assertEqual(data, data2) + + def test_cyclic_sampler_resume(self): + # Single rank + sampler = CyclicSampler( + list(range(10)), + micro_batch_size=4, + shuffle=True, + seed=123, + ) + + all_output = list(itertools.islice(sampler, 50)) # iteration 50 times + + sampler = CyclicSampler( + list(range(10)), + micro_batch_size=4, + shuffle=True, + seed=123, + consumed_samples=4 * 11, # consumed 11 iters + ) + + resume_output = list(itertools.islice(sampler, 39)) + self.assertEqual(all_output[11:], resume_output) + + def test_cyclic_sampler_resume_multi_rank(self): + # Multiple ranks + sampler_rank0 = CyclicSampler( + list(range(10)), + micro_batch_size=4, + shuffle=True, + seed=123, + data_parallel_rank=0, + data_parallel_size=2, + ) + sampler_rank1 = CyclicSampler( + list(range(10)), + micro_batch_size=4, + shuffle=True, + seed=123, + data_parallel_rank=1, + data_parallel_size=2, + ) + + all_output_rank0 = list(itertools.islice(sampler_rank0, 50)) # iteration 50 times + all_output_rank1 = list(itertools.islice(sampler_rank1, 50)) # iteration 50 times + + sampler_rank0 = CyclicSampler( + list(range(10)), + micro_batch_size=4, + shuffle=True, + seed=123, + data_parallel_rank=0, + data_parallel_size=2, + consumed_samples=4 * 11, # consumed 11 iters + ) + sampler_rank1 = CyclicSampler( + list(range(10)), + micro_batch_size=4, + shuffle=True, + seed=123, + data_parallel_rank=1, + data_parallel_size=2, + consumed_samples=4 * 11, # consumed 11 iters + ) + + resume_output_rank0 = list(itertools.islice(sampler_rank0, 39)) + resume_output_rank1 = list(itertools.islice(sampler_rank1, 39)) + + self.assertEqual(all_output_rank0[11:], resume_output_rank0) + self.assertEqual(all_output_rank1[11:], resume_output_rank1) + + +class TestSingleRoundSampler(unittest.TestCase): + def test_single_sampler_iterable(self): + sampler = SingleRoundSampler( + list(range(100)), + micro_batch_size=4, + shuffle=False, + ) + output_iter = itertools.islice(sampler, 30) # exceed iteration number + sample_output = list() + for batch in output_iter: + sample_output.extend(batch) + self.assertEqual(sample_output, list(range(100))) + + def test_single_sampler_multi_rank(self): + sampler_rank0 = SingleRoundSampler( + list(range(101)), + micro_batch_size=4, + shuffle=False, + data_parallel_rank=0, + data_parallel_size=2, + ) + sampler_rank1 = SingleRoundSampler( + list(range(101)), + micro_batch_size=4, + shuffle=False, + data_parallel_rank=1, + data_parallel_size=2, + ) + + output_iter_rank0 = itertools.islice(sampler_rank0, 30) + sample_output_rank0 = list() + for batch in output_iter_rank0: + sample_output_rank0.extend(batch) + + output_iter_rank1 = itertools.islice(sampler_rank1, 30) + sample_output_rank1 = list() + for batch in output_iter_rank1: + sample_output_rank1.extend(batch) + + # Padding 0 if it's not enough for a batch, otherwise `to_global` + # will raise errors for imbalanced data shape in different ranks + self.assertEqual(sample_output_rank0, list(range(51))) + self.assertEqual(sample_output_rank1, list(range(51, 101)) + [0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/fixtures/sample_text.txt b/tests/fixtures/sample_text.txt new file mode 100644 index 0000000000000000000000000000000000000000..a42812060c576bae870eb29b1ac083fda0d239d3 --- /dev/null +++ b/tests/fixtures/sample_text.txt @@ -0,0 +1,33 @@ +This text is included to make sure Unicode is handled properly: 力加勝北区ᴵᴺᵀᵃছজটডণত +Text should be one-sentence-per-line, with empty lines between documents. +This sample text is public domain and was randomly selected from Project Guttenberg. + +The rain had only ceased with the gray streaks of morning at Blazing Star, and the settlement awoke to a moral sense of cleanliness, and the finding of forgotten knives, tin cups, and smaller camp utensils, where the heavy showers had washed away the debris and dust heaps before the cabin doors. +Indeed, it was recorded in Blazing Star that a fortunate early riser had once picked up on the highway a solid chunk of gold quartz which the rain had freed from its incumbering soil, and washed into immediate and glittering popularity. +Possibly this may have been the reason why early risers in that locality, during the rainy season, adopted a thoughtful habit of body, and seldom lifted their eyes to the rifted or india-ink washed skies above them. +"Cass" Beard had risen early that morning, but not with a view to discovery. +A leak in his cabin roof,--quite consistent with his careless, improvident habits,--had roused him at 4 A. M., with a flooded "bunk" and wet blankets. +The chips from his wood pile refused to kindle a fire to dry his bed-clothes, and he had recourse to a more provident neighbor's to supply the deficiency. +This was nearly opposite. +Mr. Cassius crossed the highway, and stopped suddenly. +Something glittered in the nearest red pool before him. +Gold, surely! +But, wonderful to relate, not an irregular, shapeless fragment of crude ore, fresh from Nature's crucible, but a bit of jeweler's handicraft in the form of a plain gold ring. +Looking at it more attentively, he saw that it bore the inscription, "May to Cass." +Like most of his fellow gold-seekers, Cass was superstitious. + +The fountain of classic wisdom, Hypatia herself. +As the ancient sage--the name is unimportant to a monk--pumped water nightly that he might study by day, so I, the guardian of cloaks and parasols, at the sacred doors of her lecture-room, imbibe celestial knowledge. +From my youth I felt in me a soul above the matter-entangled herd. +She revealed to me the glorious fact, that I am a spark of Divinity itself. +A fallen star, I am, sir!' continued he, pensively, stroking his lean stomach--'a fallen star!--fallen, if the dignity of philosophy will allow of the simile, among the hogs of the lower world--indeed, even into the hog-bucket itself. Well, after all, I will show you the way to the Archbishop's. +There is a philosophic pleasure in opening one's treasures to the modest young. +Perhaps you will assist me by carrying this basket of fruit?' And the little man jumped up, put his basket on Philammon's head, and trotted off up a neighbouring street. +Philammon followed, half contemptuous, half wondering at what this philosophy might be, which could feed the self-conceit of anything so abject as his ragged little apish guide; +but the novel roar and whirl of the street, the perpetual stream of busy faces, the line of curricles, palanquins, laden asses, camels, elephants, which met and passed him, and squeezed him up steps and into doorways, as they threaded their way through the great Moon-gate into the ample street beyond, drove everything from his mind but wondering curiosity, and a vague, helpless dread of that great living wilderness, more terrible than any dead wilderness of sand which he had left behind. +Already he longed for the repose, the silence of the Laura--for faces which knew him and smiled upon him; but it was too late to turn back now. +His guide held on for more than a mile up the great main street, crossed in the centre of the city, at right angles, by one equally magnificent, at each end of which, miles away, appeared, dim and distant over the heads of the living stream of passengers, the yellow sand-hills of the desert; +while at the end of the vista in front of them gleamed the blue harbour, through a network of countless masts. +At last they reached the quay at the opposite end of the street; +and there burst on Philammon's astonished eyes a vast semicircle of blue sea, ringed with palaces and towers. +He stopped involuntarily; and his little guide stopped also, and looked askance at the young monk, to watch the effect which that grand panorama should produce on him. diff --git a/tests/fixtures/test_sentencepiece.model b/tests/fixtures/test_sentencepiece.model new file mode 100644 index 0000000000000000000000000000000000000000..376dda73010c6f93acfa3b974bea81a9ac9e1740 Binary files /dev/null and b/tests/fixtures/test_sentencepiece.model differ diff --git a/tests/fixtures/utils.py b/tests/fixtures/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6dac216c20bd98f8e1bda366824b2be471912822 --- /dev/null +++ b/tests/fixtures/utils.py @@ -0,0 +1,16 @@ +from libai.utils.download import download + +fixtrue_urls = { + "sample_text.txt": "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/fixtures/sample_text.txt", # noqa + "spiece.model": "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/fixtures/spiece.model", # noqa + "test_sentencepiece.model": "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/fixtures/test_sentencepiece.model", # noqa +} + +BASE_DIR = "tests/fixtures" + + +def get_fixtures(fixture_dir): + fixture_name = fixture_dir.split("/")[-1] + if fixture_name not in fixtrue_urls: + raise RuntimeError("{} not available in LiBai tests fixtrues!".format(fixture_name)) + return download(fixtrue_urls[fixture_name], BASE_DIR) diff --git a/tests/inference/__init__.py b/tests/inference/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/inference/test_image_classification.py b/tests/inference/test_image_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..aaeeea9260bac1dec7ed7f20c135aacb9c7b3e49 --- /dev/null +++ b/tests/inference/test_image_classification.py @@ -0,0 +1,94 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import unittest +import zipfile + +import numpy as np +import oneflow as flow +import oneflow.unittest + +from libai.inference.image_classification import ImageClassificationPipeline +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache + +IMAGE_URL = "https://oneflow-public.oss-cn-beijing.aliyuncs.com/model_zoo/LiBai/Inference/ILSVRC2012_val_00000293.JPEG" # noqa +CONFIG_URL = "https://oneflow-public.oss-cn-beijing.aliyuncs.com/model_zoo/LiBai/ImageNet/vit_small_patch16_224/config.yaml" # noqa +MODEL_URL = "https://oneflow-public.oss-cn-beijing.aliyuncs.com/model_zoo/LiBai/ImageNet/vit_small_patch16_224/model_best.zip" # noqa + +IMAGE_MD5 = "65ac8a72466e859cd3c6b279ed8e532a" +CONFIG_MD5 = "4cf8e662d76f855f4d99ce7129050e79" +MODEL_MD5 = "2bfc9cb7df5739d1a1d11db97f54d93f" + + +def _legacy_zip_load(filename, model_dir): + # Note: extractall() defaults to overwrite file if exists. No need to clean up beforehand. + # We deliberately don't handle tarfile here since our legacy serialization format was in tar. + with zipfile.ZipFile(filename) as f: + members = f.infolist() + extracted_name = members[0].filename + extracted_file = os.path.join(model_dir, extracted_name) + if not os.path.exists(extracted_file): + os.mkdir(extracted_file) + f.extractall(model_dir) + + +class TestImageClassificationPipeline(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join( + os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "inference_test_data" + ) + model_path = os.path.join(cache_dir, MODEL_URL.split("/")[-1]) + if dist.get_local_rank() == 0: + # download dataset on main process of each node + get_data_from_cache(IMAGE_URL, cache_dir, md5=IMAGE_MD5) + get_data_from_cache(CONFIG_URL, cache_dir, md5=CONFIG_MD5) + get_data_from_cache(MODEL_URL, cache_dir, md5=MODEL_MD5) + _legacy_zip_load(model_path, os.path.dirname(model_path)) + self.image_path = os.path.join(cache_dir, IMAGE_URL.split("/")[-1]) + self.config_path = os.path.join(cache_dir, CONFIG_URL.split("/")[-1]) + self.model_path = model_path.replace(".zip", "") + assert os.path.isdir(self.model_path) + + @unittest.skipIf(not flow.cuda.is_available(), "only test gpu cases") + @flow.unittest.skip_unless_1n4d() + def test_smallvitpipeline_with_pipeline_parallel(self): + self.pipeline = ImageClassificationPipeline(self.config_path, 1, 1, 4, self.model_path) + rst = self.pipeline(self.image_path) + if flow.env.get_rank() == 0: + self.assertTrue(rst["label"] == "tench, Tinca tinca") + self.assertTrue( + np.allclose(np.array(0.7100194096565247), np.array(rst["score"]), 1e-4, 1e-4) + ) + + @unittest.skipIf(not flow.cuda.is_available(), "only test gpu cases") + @flow.unittest.skip_unless_1n4d() + def test_pipeline_with_pipeline_parallel(self): + self.pipeline = ImageClassificationPipeline("configs/vit_imagenet.py", 1, 1, 4) + self.pipeline(self.image_path) + + @unittest.skipIf(not flow.cuda.is_available(), "only test gpu cases") + @flow.unittest.skip_unless_1n4d() + def test_pipeline_with_tensor_parallel(self): + pass + # TODO: bug occurs when tensor parallel + # self.pipeline = ImageClassificationPipeline("configs/vit_imagenet.py", 1, 4, 1) + # self.pipeline(self.image_path) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/inference/test_text_classification.py b/tests/inference/test_text_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..21ff6588543ab9cfd856e9625312ffae8a2676fc --- /dev/null +++ b/tests/inference/test_text_classification.py @@ -0,0 +1,80 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import unittest + +import numpy as np +import oneflow as flow +import oneflow.unittest + +from libai.inference.text_classification import TextClassificationPipeline +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache + +VOCAB_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/bert-base-chinese-vocab.txt" # noqa + +VOCAB_MD5 = "65ac8a72466e859cd3c6b279ed8e532a" + + +class TestTextClassificationPipeline(flow.unittest.TestCase): + def setUp(self) -> None: + self.texts = ["cat ", "you ", "dog ", "dragon ", "牛 ", "羊 "] + cache_dir = os.path.join(os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "bert_data") + # prepare tokenizer + if dist.get_local_rank() == 0: + # download tokenzier vocab on main process of each node + get_data_from_cache(VOCAB_URL, cache_dir, md5=VOCAB_MD5) + + @unittest.skipIf(not flow.cuda.is_available(), "only test gpu cases") + @flow.unittest.skip_unless_1n4d() + def test_pipeline_with_tensor_parallel(self): + self.pipeline = TextClassificationPipeline("configs/bert_classification.py", 1, 4, 1) + + text = list(np.random.randint(0, 6, 10)) + text = "".join([self.texts[i] for i in text]) + dict1 = self.pipeline(text) + dict2 = self.pipeline(text) + if dist.is_main_process(): + assert dict1["score"] == dict2["score"] + + @unittest.skipIf(not flow.cuda.is_available(), "only test gpu cases") + @flow.unittest.skip_unless_1n4d() + def test_pipeline_with_pipeline_parallel(self): + self.pipeline = TextClassificationPipeline("configs/bert_classification.py", 1, 1, 4) + + text = list(np.random.randint(0, 6, 10)) + text = "".join([self.texts[i] for i in text]) + dict1 = self.pipeline(text) + dict2 = self.pipeline(text) + if dist.is_main_process(): + assert dict1["score"] == dict2["score"] + + @unittest.skipIf(not flow.cuda.is_available(), "only test gpu cases") + @flow.unittest.skip_unless_1n4d() + def test_pipeline_with_tensor_pipeline_parallel(self): + self.pipeline = TextClassificationPipeline("configs/bert_classification.py", 1, 2, 2) + + text = list(np.random.randint(0, 6, 10)) + text = "".join([self.texts[i] for i in text]) + dict1 = self.pipeline(text) + dict2 = self.pipeline(text) + if dist.is_main_process(): + assert dict1["score"] == dict2["score"] + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/inference/test_text_generation.py b/tests/inference/test_text_generation.py new file mode 100644 index 0000000000000000000000000000000000000000..70158c36c0493c42648d1528c170194ccc948f29 --- /dev/null +++ b/tests/inference/test_text_generation.py @@ -0,0 +1,94 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest + +import numpy as np +import oneflow as flow +import oneflow.unittest + +from libai.inference.text_generation import TextGenerationPipeline +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache + +VOCAB_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/bert-base-chinese-vocab.txt" # noqa + +VOCAB_MD5 = "65ac8a72466e859cd3c6b279ed8e532a" + + +class TestTextGenerationPipeline(flow.unittest.TestCase): + def setUp(self) -> None: + self.texts = ["cat ", "you ", "dog ", "dragon ", "牛 ", "羊 "] + cache_dir = os.path.join(os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "bert_data") + # prepare tokenizer + if dist.get_local_rank() == 0: + # download tokenzier vocab on main process of each node + get_data_from_cache(VOCAB_URL, cache_dir, md5=VOCAB_MD5) + + @unittest.skipIf(not flow.cuda.is_available(), "only test gpu cases") + @flow.unittest.skip_unless_1n4d() + def test_pipeline_with_tensor_parallel(self): + self.pipeline = TextGenerationPipeline("configs/t5_large_pretrain.py", 1, 4, 1) + + for _ in range(5): + text = list(np.random.randint(0, 5, 10)) + text = "".join([self.texts[i] for i in text]) + dict1 = self.pipeline( + text, use_cache=False, max_generate_length=15, return_type="new_text" + ) + dict2 = self.pipeline( + text, use_cache=True, max_generate_length=15, return_type="new_text" + ) + if dist.is_main_process(): + assert dict1["generated_text"] == dict2["generated_text"] + + @unittest.skipIf(not flow.cuda.is_available(), "only test gpu cases") + @flow.unittest.skip_unless_1n4d() + def test_pipeline_with_pipeline_parallel(self): + self.pipeline = TextGenerationPipeline("configs/t5_large_pretrain.py", 1, 1, 4) + + for _ in range(5): + text = list(np.random.randint(0, 5, 10)) + text = "".join([self.texts[i] for i in text]) + dict1 = self.pipeline( + text, use_cache=False, max_generate_length=15, return_type="new_text" + ) + dict2 = self.pipeline( + text, use_cache=True, max_generate_length=15, return_type="new_text" + ) + if dist.is_main_process(): + assert dict1["generated_text"] == dict2["generated_text"] + + @unittest.skipIf(not flow.cuda.is_available(), "only test gpu cases") + @flow.unittest.skip_unless_1n4d() + def test_pipeline_with_tensor_pipeline_parallel(self): + self.pipeline = TextGenerationPipeline("configs/t5_large_pretrain.py", 1, 2, 2) + + for _ in range(5): + text = list(np.random.randint(0, 5, 10)) + text = "".join([self.texts[i] for i in text]) + dict1 = self.pipeline( + text, use_cache=False, max_generate_length=15, return_type="new_text" + ) + dict2 = self.pipeline( + text, use_cache=True, max_generate_length=15, return_type="new_text" + ) + if dist.is_main_process(): + assert dict1["generated_text"] == dict2["generated_text"] + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/layers/__init__.py b/tests/layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/layers/test_evaluator_model.py b/tests/layers/test_evaluator_model.py new file mode 100644 index 0000000000000000000000000000000000000000..722d20c5c6ce4a9156cbcff6db5ea5bdad1ada4c --- /dev/null +++ b/tests/layers/test_evaluator_model.py @@ -0,0 +1,55 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +from oneflow import nn + +from libai.models.utils.graph_base import GraphBase + + +class demo_model(nn.Module): + def __init__(self, input_dim=512, out_dim=3): + super().__init__() + self.linear1 = nn.Linear(input_dim, input_dim // 2) + self.linear2 = nn.Linear(input_dim // 2, out_dim) + + def forward(self, x, label=None): + x = x.to(dtype=flow.float32) + x = self.linear1(x) + x = self.linear2(x) + if label is None: + return x + loss = self.get_loss(x) + return loss + + def get_loss(self, x): + return x.sum() + + +def build_model(cfg): + model = demo_model() + placement = flow.env.all_device_placement("cuda") + model = model.to_global(placement=placement, sbp=flow.sbp.broadcast) + return model + + +class GraphModel(GraphBase): + def build(self, x, label=None): + if self.is_train: + loss = self.model(x, label) + loss.backward() + return loss + else: + return self.model(x) diff --git a/tests/layers/test_linear.py b/tests/layers/test_linear.py new file mode 100644 index 0000000000000000000000000000000000000000..804d6c6a5c1c2c70e4a2fa686a3e7b678611b49e --- /dev/null +++ b/tests/layers/test_linear.py @@ -0,0 +1,114 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest + +import numpy as np +import oneflow as flow +import oneflow.unittest +from omegaconf import DictConfig +from oneflow import nn + +from libai.layers import Linear +from libai.utils import distributed as dist + + +class TestLinear(flow.unittest.TestCase): + @unittest.skipIf(not flow.cuda.is_available(), "only test gpu cases") + @flow.unittest.skip_unless_1n1d() + def test_nn_linear(self): + dist.setup_dist_util( + DictConfig( + dict( + data_parallel_size=1, + tensor_parallel_size=1, + pipeline_parallel_size=1, + ) + ) + ) + + inputs = flow.rand(8, 8, sbp=flow.sbp.broadcast, placement=dist.get_layer_placement(0)) + weight = flow.rand(4, 8, sbp=flow.sbp.broadcast, placement=dist.get_layer_placement(0)) + bias = flow.rand(4, sbp=flow.sbp.broadcast, placement=dist.get_layer_placement(0)) + + nn_linear = nn.Linear(8, 4).to("cuda") + nn_linear.weight.data.copy_(dist.ttol(weight).to("cuda")) + nn_linear.bias.data.copy_(dist.ttol(bias).to("cuda")) + nn_output = nn_linear(dist.ttol(inputs).to("cuda")) + + libai_linear = Linear(8, 4) + libai_linear.weight.data.copy_(weight) + libai_linear.bias.data.copy_(bias) + libai_output = libai_linear(inputs) + + self.assertTrue(np.allclose(nn_output.cpu().numpy(), dist.tton(libai_output), 1e-7, 1e-7)) + + @flow.unittest.skip_unless_1n2d() + def test_col_parallel_linear(self): + dist.setup_dist_util( + DictConfig( + dict( + data_parallel_size=1, + tensor_parallel_size=2, + pipeline_parallel_size=1, + ) + ) + ) + + inputs = flow.rand(8, 8, sbp=flow.sbp.broadcast, placement=dist.get_layer_placement(0)) + weight = flow.rand(4, 8, sbp=flow.sbp.split(0), placement=dist.get_layer_placement(0)) + bias = flow.rand(4, sbp=flow.sbp.split(0), placement=dist.get_layer_placement(0)) + + nn_linear = nn.Linear(8, 4).to("cuda") + nn_linear.weight.data.copy_(dist.ttol(weight).to("cuda")) + nn_linear.bias.data.copy_(dist.ttol(bias).to("cuda")) + nn_output = nn_linear(dist.ttol(inputs).to("cuda")) + + libai_linear = Linear(8, 4, parallel="col") + libai_linear.weight.data.copy_(weight) + libai_linear.bias.data.copy_(bias) + libai_output = libai_linear(inputs) + + self.assertTrue(np.allclose(nn_output.cpu().numpy(), dist.tton(libai_output), 1e-7, 1e-7)) + + @flow.unittest.skip_unless_1n2d() + def test_row_parallel_linear(self): + dist.setup_dist_util( + DictConfig( + dict( + data_parallel_size=1, + tensor_parallel_size=2, + pipeline_parallel_size=1, + ) + ) + ) + + inputs = flow.rand(8, 8, sbp=flow.sbp.broadcast, placement=dist.get_layer_placement(0)) + weight = flow.rand(4, 8, sbp=flow.sbp.split(1), placement=dist.get_layer_placement(0)) + bias = flow.rand(4, sbp=flow.sbp.broadcast, placement=dist.get_layer_placement(0)) + + # move local tensor to cuda + nn_linear = nn.Linear(8, 4).to("cuda") + nn_linear.weight.data.copy_(dist.ttol(weight).to("cuda")) + nn_linear.bias.data.copy_(dist.ttol(bias).to("cuda")) + nn_output = nn_linear(dist.ttol(inputs).to("cuda")) + + libai_linear = Linear(8, 4, parallel="row") + libai_linear.weight.data.copy_(weight) + libai_linear.bias.data.copy_(bias) + libai_output = libai_linear(inputs) + + self.assertTrue(np.allclose(nn_output.cpu().numpy(), dist.tton(libai_output), 1e-7, 1e-7)) diff --git a/tests/layers/test_trainer_model.py b/tests/layers/test_trainer_model.py new file mode 100644 index 0000000000000000000000000000000000000000..2caf35a94dfe605beda49f8669b2cb350dc73929 --- /dev/null +++ b/tests/layers/test_trainer_model.py @@ -0,0 +1,69 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import oneflow as flow +from oneflow import nn + + +class demo_model(nn.Module): + def __init__(self, input_dim=512, out_dim=64): + super().__init__() + self.linear1 = nn.Linear(input_dim, out_dim) + self.linear2 = nn.Linear(out_dim, out_dim) + + def forward(self, x): + x = self.linear1(x) + x = self.linear2(x) + loss = self.get_loss(x) + return loss + + def get_loss(self, x): + return x.sum() + + +def build_model(cfg): + model = demo_model() + placement = flow.env.all_device_placement("cuda") + model = model.to_global(placement=placement, sbp=flow.sbp.broadcast) + return model + + +def build_graph(cfg, model, optimizer, lr_scheduler, fp16=False): + class GraphModel(nn.Graph): + def __init__(self, model, optimizer, lr_scheduler, fp16=False): + super().__init__() + self.model = model + self.add_optimizer(optimizer, lr_sch=lr_scheduler) + self.config.allow_fuse_add_to_output(True) + self.config.allow_fuse_model_update_ops(True) + if fp16: + self.config.enable_amp(True) + grad_scaler = flow.amp.GradScaler( + init_scale=2 ** 30, + growth_factor=2.0, + backoff_factor=0.5, + growth_interval=2000, + ) + self.set_grad_scaler(grad_scaler) + + def build(self, x): + loss = self.model(x) + loss.backward() + return loss + + if optimizer: + return GraphModel(model, optimizer, lr_scheduler, fp16=fp16) + else: + return None diff --git a/tests/model_utils/__init__.py b/tests/model_utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/model_utils/test_bert_loader.py b/tests/model_utils/test_bert_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..75a6d75c42eb0a49e565b29f3cbd8bbcc3f1f6c6 --- /dev/null +++ b/tests/model_utils/test_bert_loader.py @@ -0,0 +1,168 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import shutil +import unittest + +import numpy as np +import oneflow as flow +import oneflow.unittest +from omegaconf import DictConfig + +import libai +from configs.common.models.bert import cfg as libai_cfg +from libai.models.utils import BertLoaderHuggerFace +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger + +PRETRAINED_MODEL_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/bert_utils/pytorch_model.bin" # noqa +PRETRAINED_MODEL_CONFIG_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/bert_utils/config.json" # noqa + +PRETRAINED_MODEL_MD5 = "ea97b42698d3b5f6d8e8011eba3d1611" +PRETRAINED_MODEL_CONFIG_MD5 = "0939b914fc32135f6c12d8ef281dbd7a" + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_bert_utils") + + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestBertLoder(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join( + os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "bert_utils_data" + ) + self.pretrained_model_path = cache_dir + + # prepare dataset + if dist.get_local_rank() == 0: + # download dataset on main process of each node + get_data_from_cache(PRETRAINED_MODEL_URL, cache_dir, md5=PRETRAINED_MODEL_MD5) + get_data_from_cache( + PRETRAINED_MODEL_CONFIG_URL, cache_dir, md5=PRETRAINED_MODEL_CONFIG_MD5 + ) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + # prepare input data + self.input_ids = [ + [101, 2009, 1005, 1055, 2986, 2651, 1012, 102], + [101, 2028, 12314, 3377, 102, 0, 0, 0], + [101, 2064, 2017, 3305, 2009, 102, 0, 0], + ] + self.mask = [[1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1]] + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_bert_loader_with_data_tensor_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = BertLoaderHuggerFace( + model=libai.models.BertModel, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + apply_residual_post_layernorm=True, + amp_enabled=False, + ) + model = load_func.load() + model.eval() + + input_ids = flow.tensor( + self.input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.embeddings.vocab_embeddings.weight.placement, + ) + mask = flow.tensor( + self.mask, + dtype=flow.bool, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.embeddings.vocab_embeddings.weight.placement, + ) + + last_hidden_state, _ = model(input_ids, mask) + self.assertTrue( + np.allclose(np.array(-214.9335), last_hidden_state.sum().data.numpy(), 1e-4, 1e-4) + ) + + @flow.unittest.skip_unless_1n4d() + def test_bert_loader_with_data_tensor_pipeline_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=1, + pipeline_parallel_size=2, + pipeline_num_layers=12, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = BertLoaderHuggerFace( + model=libai.models.BertModel, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + apply_residual_post_layernorm=True, + amp_enabled=False, + ) + model = load_func.load() + model.eval() + + input_ids = flow.tensor( + self.input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.embeddings.vocab_embeddings.weight.placement, + ) + mask = flow.tensor( + self.mask, + dtype=flow.bool, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.embeddings.vocab_embeddings.weight.placement, + ) + + last_hidden_state, _ = model(input_ids, mask) + self.assertTrue( + np.allclose(np.array(-214.9335), last_hidden_state.sum().data.numpy(), 1e-4, 1e-4) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/model_utils/test_gpt_loader.py b/tests/model_utils/test_gpt_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..8df206a0166e030de7de8242cd353d5fa21a9a3e --- /dev/null +++ b/tests/model_utils/test_gpt_loader.py @@ -0,0 +1,260 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import shutil +import unittest + +import numpy as np +import oneflow as flow +import oneflow.unittest +from omegaconf import DictConfig + +import libai +from configs.common.models.gpt import cfg as libai_cfg +from libai.models.utils import GPT2LoaderHuggerFace +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger + +PRETRAINED_MODEL_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/gpt_utils/pytorch_model.bin" # noqa +PRETRAINED_MODEL_CONFIG_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/gpt_utils/config.json" # noqa + +PRETRAINED_MODEL_MD5 = "c086214036308afc71896da17ca0442a" +PRETRAINED_MODEL_CONFIG_MD5 = "6e1dba197b511b8759d6ad4551095a29" + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_gpt_utils") + + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestGPT2Loader(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join( + os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "gpt_utils_data" + ) + self.pretrained_model_path = cache_dir + + # prepare dataset + if dist.get_local_rank() == 0: + # download dataset on main process of each node + get_data_from_cache(PRETRAINED_MODEL_URL, cache_dir, md5=PRETRAINED_MODEL_MD5) + get_data_from_cache( + PRETRAINED_MODEL_CONFIG_URL, cache_dir, md5=PRETRAINED_MODEL_CONFIG_MD5 + ) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + # prepare input data + self.input_ids = [ + [101, 2009, 1005, 1055, 2986, 2651, 1012, 102], + [101, 2028, 12314, 3377, 102, 0, 0, 0], + [101, 2064, 2017, 3305, 2009, 102, 0, 0], + ] + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_gpt_loader_with_data_tensor_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = GPT2LoaderHuggerFace( + model=libai.models.GPTModel, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=True, + apply_query_key_layer_scaling=True, + apply_residual_post_layernorm=False, + amp_enabled=False, + attention_dropout_prob=0, + output_dropout_prob=0, + ) + model = load_func.load() + model.eval() + + input_ids = flow.tensor( + self.input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.embeddings.token_embeddings.weight.placement, + ) + + logits = model(input_ids) + self.assertTrue( + np.allclose( + np.array(-93505072.0), + logits.sum().data.numpy(), + ) + ) + + @flow.unittest.skip_unless_1n4d() + def test_gpt_loader_with_data_tensor_pipeline_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=1, + pipeline_parallel_size=2, + pipeline_num_layers=12, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = GPT2LoaderHuggerFace( + model=libai.models.GPTModel, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=True, + apply_query_key_layer_scaling=True, + apply_residual_post_layernorm=False, + amp_enabled=False, + attention_dropout_prob=0, + output_dropout_prob=0, + ) + model = load_func.load() + model.eval() + + input_ids = flow.tensor( + self.input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.embeddings.token_embeddings.weight.placement, + ) + + logits = model(input_ids) + self.assertTrue( + np.allclose( + np.array(-93505072.0), + logits.sum().data.numpy(), + ) + ) + + @flow.unittest.skip_unless_1n4d() + def test_gpt_loader_with_data_tensor_parallel_backward(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = GPT2LoaderHuggerFace( + model=libai.models.GPTModel, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=True, + apply_query_key_layer_scaling=True, + apply_residual_post_layernorm=False, + amp_enabled=False, + attention_dropout_prob=0, + output_dropout_prob=0, + embedding_dropout_prob=0, + ) + model = load_func.load() + + input_ids = flow.tensor( + self.input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.embeddings.token_embeddings.weight.placement, + ) + logits = model(input_ids) + loss = logits.sum() + loss.backward() + + self.assertTrue( + np.allclose(-24882176.0, model.transformer.layernorm_f.weight.grad.sum().numpy()) + ) + self.assertTrue( + np.allclose( + 3.1779e08, model.embeddings.token_embeddings.weight.grad.sum().numpy(), 1e-3 + ) + ) + + @flow.unittest.skip_unless_1n4d() + def test_gpt_loader_with_data_tensor_pipeline_parallel_backward(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=1, + pipeline_parallel_size=2, + pipeline_num_layers=12, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = GPT2LoaderHuggerFace( + model=libai.models.GPTModel, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=True, + apply_query_key_layer_scaling=True, + apply_residual_post_layernorm=False, + amp_enabled=False, + attention_dropout_prob=0, + output_dropout_prob=0, + embedding_dropout_prob=0, + ) + model = load_func.load() + + input_ids = flow.tensor( + self.input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.embeddings.token_embeddings.weight.placement, + ) + logits = model(input_ids) + loss = logits.sum() + loss.backward() + + self.assertTrue( + np.allclose(-24882176.0, model.transformer.layernorm_f.weight.grad.sum().numpy()) + ) + self.assertTrue( + np.allclose(317785760.0, model.embeddings.token_embeddings.weight.grad.sum().numpy()) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/model_utils/test_mt5_loader.py b/tests/model_utils/test_mt5_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..b7da98b7472b4501dfa4b10be35c5d1be12b187c --- /dev/null +++ b/tests/model_utils/test_mt5_loader.py @@ -0,0 +1,208 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import shutil +import unittest + +import numpy as np +import oneflow as flow +import oneflow.unittest +from omegaconf import DictConfig + +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger +from projects.MT5.configs.mt5_base import cfg as libai_cfg +from projects.MT5.mt5_model import MT5Model +from projects.MT5.utils.mt5_loader import T5LoaderHuggerFace + +PRETRAINED_MODEL_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/mt5_utils/pytorch_model.bin" # noqa +PRETRAINED_MODEL_CONFIG_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/mt5_utils/config.json" # noqa + +PRETRAINED_MODEL_MD5 = "4c9c0be541b89de9b01c597ec4cc371a" +PRETRAINED_MODEL_CONFIG_MD5 = "b159e41603b7eeaf9a9c489165bbcaca" + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_mt5_utils") + + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestMT5Loader(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join( + os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "mt5_utils_data" + ) + self.pretrained_model_path = cache_dir + + # prepare dataset + if dist.get_local_rank() == 0: + # download dataset on main process of each node + get_data_from_cache(PRETRAINED_MODEL_URL, cache_dir, md5=PRETRAINED_MODEL_MD5) + get_data_from_cache( + PRETRAINED_MODEL_CONFIG_URL, cache_dir, md5=PRETRAINED_MODEL_CONFIG_MD5 + ) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + # prepare input data + self.encoder_input_ids = [ + [101, 2009, 1005, 1055, 2986, 2651, 1012, 102], + [101, 2028, 12314, 3377, 102, 0, 0, 0], + [101, 2064, 2017, 3305, 2009, 102, 0, 0], + ] + self.encoder_att_mask = [ + [1, 1, 1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1, 1], + ] + self.decoder_input_ids = [ + [101, 2009, 1005, 1055, 2986], + [101, 2028, 12314, 3377, 102], + [101, 2064, 2017, 3305, 2009], + ] + self.decoder_att_mask = [[1, 1, 1, 1, 1], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1]] + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_mt5_loader_with_data_tensor_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = T5LoaderHuggerFace( + model=MT5Model, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + embedding_dropout_prob=0.0, + model_type="mt5", + ) + model = load_func.load() + model.eval() + + encoder_input_ids = flow.tensor( + self.encoder_input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + decoder_input_ids = flow.tensor( + self.decoder_input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + encode_att_mask = flow.tensor( + self.encoder_att_mask, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + decoder_att_mask = flow.tensor( + self.decoder_att_mask, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + + logits = model( + encoder_input_ids, decoder_input_ids, encode_att_mask, decoder_att_mask, encode_att_mask + )["logits"] + self.assertTrue( + np.allclose( + np.array(-83584720.0), + logits.sum().data.numpy(), + ) + ) + + @flow.unittest.skip_unless_1n4d() + def test_mt5_loader_with_data_tensor_pipeline_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=1, + pipeline_parallel_size=2, + pipeline_num_layers=16, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = T5LoaderHuggerFace( + model=MT5Model, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + embedding_dropout_prob=0.0, + model_type="mt5", + ) + model = load_func.load() + model.eval() + + encoder_input_ids = flow.tensor( + self.encoder_input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + decoder_input_ids = flow.tensor( + self.decoder_input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + encode_att_mask = flow.tensor( + self.encoder_att_mask, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + decoder_att_mask = flow.tensor( + self.decoder_att_mask, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + + logits = model( + encoder_input_ids, decoder_input_ids, encode_att_mask, decoder_att_mask, encode_att_mask + )["logits"] + self.assertTrue( + np.allclose( + np.array(-83584720.0), + logits.sum().data.numpy(), + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/model_utils/test_roberta_loader.py b/tests/model_utils/test_roberta_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..d83eb1f502a365f9cc3b3ab780566d0302f885ae --- /dev/null +++ b/tests/model_utils/test_roberta_loader.py @@ -0,0 +1,168 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import shutil +import unittest + +import numpy as np +import oneflow as flow +import oneflow.unittest +from omegaconf import DictConfig + +import libai +from configs.common.models.roberta import cfg as libai_cfg +from libai.models.utils import RobertaLoaderHuggerFace +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger + +PRETRAINED_MODEL_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/roberta_utils/pytorch_model.bin" # noqa +PRETRAINED_MODEL_CONFIG_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/roberta_utils/config.json" # noqa + +PRETRAINED_MODEL_MD5 = "73db58b6c51b028e0ee031f12261b51d" # noqa +PRETRAINED_MODEL_CONFIG_MD5 = "a53c22291c7f25d5077260ad5ca4d5fa" # noqa + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_roberta_utils") + + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestRobertaLoader(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join( + os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "roberta_utils_data" + ) + self.pretrained_model_path = cache_dir + + # prepare dataset + if dist.get_local_rank() == 0: + # download dataset on main process of each node + get_data_from_cache(PRETRAINED_MODEL_URL, cache_dir, md5=PRETRAINED_MODEL_MD5) + get_data_from_cache( + PRETRAINED_MODEL_CONFIG_URL, cache_dir, md5=PRETRAINED_MODEL_CONFIG_MD5 + ) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + # prepare input data + self.input_ids = [ + [101, 2009, 1005, 1055, 2986, 2651, 1012, 102], + [101, 2028, 12314, 3377, 102, 0, 0, 0], + [101, 2064, 2017, 3305, 2009, 102, 0, 0], + ] + self.mask = [[1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1]] + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_roberta_loader_with_data_tensor_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = RobertaLoaderHuggerFace( + model=libai.models.RobertaModel, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + apply_residual_post_layernorm=True, + amp_enabled=False, + ) + model = load_func.load() + model.eval() + + input_ids = flow.tensor( + self.input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.embeddings.vocab_embeddings.weight.placement, + ) + mask = flow.tensor( + self.mask, + dtype=flow.bool, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.embeddings.vocab_embeddings.weight.placement, + ) + + last_hidden_state, _ = model(input_ids, mask) + self.assertTrue( + np.allclose(np.array(341.5831), last_hidden_state.sum().data.numpy(), 1e-4, 1e-4) + ) + + @flow.unittest.skip_unless_1n4d() + def test_roberta_loader_with_data_tensor_pipeline_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=1, + pipeline_parallel_size=2, + pipeline_num_layers=12, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = RobertaLoaderHuggerFace( + model=libai.models.RobertaModel, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + bias_gelu_fusion=False, + bias_dropout_fusion=False, + scale_mask_softmax_fusion=False, + apply_query_key_layer_scaling=False, + apply_residual_post_layernorm=True, + amp_enabled=False, + ) + model = load_func.load() + model.eval() + + input_ids = flow.tensor( + self.input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.embeddings.vocab_embeddings.weight.placement, + ) + mask = flow.tensor( + self.mask, + dtype=flow.bool, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.embeddings.vocab_embeddings.weight.placement, + ) + + last_hidden_state, _ = model(input_ids, mask) + self.assertTrue( + np.allclose(np.array(341.5831), last_hidden_state.sum().data.numpy(), 1e-4, 1e-4) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/model_utils/test_swin_loader.py b/tests/model_utils/test_swin_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..40f1b999ff92a92afab3255726bc629669314108 --- /dev/null +++ b/tests/model_utils/test_swin_loader.py @@ -0,0 +1,218 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import shutil +import unittest + +import numpy as np +import oneflow as flow +import oneflow.unittest +from omegaconf import DictConfig + +import libai +from configs.common.models.swin.swin_tiny_patch4_window7_224 import cfg as libai_cfg +from libai.models.utils import SwinLoaderHuggerFace +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger + +PRETRAINED_MODEL_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/swin_utils/pytorch_model.bin" # noqa +PRETRAINED_MODEL_CONFIG_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/swin_utils/config.json" # noqa +INIT_DATA = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/swin_utils/init_data.npz" # noqa + +PRETRAINED_MODEL_MD5 = "cd8c03d9cd4a9c536a5a245f663035b6" +PRETRAINED_MODEL_CONFIG_MD5 = "a8a71ed22b99323edd6a1457bede5819" +INIT_DATA_MD5 = "5fecdcd8d46bfefa310d19e084bd4815" + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_swin_utils") + + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestSwinLoder(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join( + os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "swin_utils_data" + ) + self.pretrained_model_path = cache_dir + self.init_data_path = os.path.join(cache_dir, "init_data.npz") + + # download model and data + if dist.get_local_rank() == 0: + # download dataset on main process of each node + get_data_from_cache(PRETRAINED_MODEL_URL, cache_dir, md5=PRETRAINED_MODEL_MD5) + get_data_from_cache( + PRETRAINED_MODEL_CONFIG_URL, cache_dir, md5=PRETRAINED_MODEL_CONFIG_MD5 + ) + get_data_from_cache(INIT_DATA, cache_dir, md5=INIT_DATA_MD5) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + # prepare input data + self.input_image = np.load(self.init_data_path)["arr_0"] + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_swin_loader_with_data_tensor_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = SwinLoaderHuggerFace( + model=libai.models.SwinTransformer, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + ) + model = load_func.load() + model.eval() + + input_image = flow.tensor( + self.input_image.tolist(), + dtype=flow.float32, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.patch_embed.proj.weight.placement, + ) + + prediction_scores = model(input_image)["prediction_scores"] + + self.assertTrue(np.allclose(np.array(80.9373), prediction_scores.sum().data.numpy(), 1e-3)) + + @flow.unittest.skip_unless_1n4d() + def test_swin_loader_with_data_tensor_pipeline_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=1, + pipeline_parallel_size=2, + pipeline_num_layers=12, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = SwinLoaderHuggerFace( + model=libai.models.SwinTransformer, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + ) + model = load_func.load() + model.eval() + + input_image = flow.tensor( + self.input_image, + dtype=flow.float32, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.patch_embed.proj.weight.placement, + ) + + prediction_scores = model(input_image)["prediction_scores"] + + self.assertTrue(np.allclose(np.array(80.9373), prediction_scores.sum().data.numpy(), 1e-3)) + + @flow.unittest.skip_unless_1n4d() + def test_swin_loader_with_data_tensor_parallel_backward(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = SwinLoaderHuggerFace( + model=libai.models.SwinTransformer, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + drop_rate=0.0, + drop_path_rate=0.0, + ) + model = load_func.load() + + input_image = flow.tensor( + self.input_image, + dtype=flow.float32, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.patch_embed.proj.weight.placement, + ) + + prediction_scores = model(input_image)["prediction_scores"] + loss = prediction_scores.sum() + loss.backward() + + self.assertTrue(np.allclose(108775.88, model.head.weight.grad.sum().numpy(), 1e-3)) + self.assertTrue( + np.allclose(24.320518, model.patch_embed.norm.weight.grad.sum().numpy(), 1e-2) + ) + + @flow.unittest.skip_unless_1n4d() + def test_swin_loader_with_data_tensor_pipeline_parallel_backward(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=1, + pipeline_parallel_size=2, + pipeline_num_layers=12, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = SwinLoaderHuggerFace( + model=libai.models.SwinTransformer, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + drop_rate=0.0, + drop_path_rate=0.0, + ) + model = load_func.load() + + input_image = flow.tensor( + self.input_image, + dtype=flow.float32, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.patch_embed.proj.weight.placement, + ) + + prediction_scores = model(input_image)["prediction_scores"] + loss = prediction_scores.sum() + loss.backward() + + self.assertTrue(np.allclose(108775.88, model.head.weight.grad.sum().numpy(), 1e-3)) + self.assertTrue( + np.allclose(24.320518, model.patch_embed.norm.weight.grad.sum().numpy(), 1e-2) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/model_utils/test_swinv2_loader.py b/tests/model_utils/test_swinv2_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..f058a178f081b64df33b7bef251a0741a2df6ea4 --- /dev/null +++ b/tests/model_utils/test_swinv2_loader.py @@ -0,0 +1,222 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import shutil +import unittest + +import numpy as np +import oneflow as flow +import oneflow.unittest +from omegaconf import DictConfig + +import libai +from configs.common.models.swinv2.swinv2_tiny_patch4_window8_256 import cfg as libai_cfg +from libai.models.utils import SwinV2LoaderHuggerFace +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger + +PRETRAINED_MODEL_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/swinv2_utils/pytorch_model.bin" # noqa +PRETRAINED_MODEL_CONFIG_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/swinv2_utils/config.json" # noqa +INIT_DATA = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/swinv2_utils/init_data.npz" # noqa + +PRETRAINED_MODEL_MD5 = "40f085f8916974dcb5d86fc6e03aa0df" +PRETRAINED_MODEL_CONFIG_MD5 = "2d3874d58f3d5684f51f70ca29a7de9f" +INIT_DATA_MD5 = "c19b2ad8afe9a708aac9d2a0ff15f7bd" + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_swinv2_utils") + + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestSwinV2Loder(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join( + os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "swinv2_utils_data" + ) + self.pretrained_model_path = cache_dir + self.init_data_path = os.path.join(cache_dir, "init_data.npz") + + # download model and data + if dist.get_local_rank() == 0: + # download dataset on main process of each node + get_data_from_cache(PRETRAINED_MODEL_URL, cache_dir, md5=PRETRAINED_MODEL_MD5) + get_data_from_cache( + PRETRAINED_MODEL_CONFIG_URL, cache_dir, md5=PRETRAINED_MODEL_CONFIG_MD5 + ) + get_data_from_cache(INIT_DATA, cache_dir, md5=INIT_DATA_MD5) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + # prepare input data + self.input_image = np.load(self.init_data_path)["arr_0"] + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_swinv2_loader_with_data_tensor_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = SwinV2LoaderHuggerFace( + model=libai.models.SwinTransformerV2, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + ) + model = load_func.load() + model.eval() + + input_image = flow.tensor( + self.input_image.tolist(), + dtype=flow.float32, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.patch_embed.proj.weight.placement, + ) + + prediction_scores = model(input_image)["prediction_scores"] + + self.assertTrue( + np.allclose(np.array(221.7827), prediction_scores.sum().data.numpy(), 1e-4, 1e-4) + ) + + @flow.unittest.skip_unless_1n4d() + def test_swinv2_loader_with_data_tensor_pipeline_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=1, + pipeline_parallel_size=2, + pipeline_num_layers=12, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = SwinV2LoaderHuggerFace( + model=libai.models.SwinTransformerV2, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + ) + model = load_func.load() + model.eval() + + input_image = flow.tensor( + self.input_image, + dtype=flow.float32, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.patch_embed.proj.weight.placement, + ) + + prediction_scores = model(input_image)["prediction_scores"] + + self.assertTrue( + np.allclose(np.array(221.7827), prediction_scores.sum().data.numpy(), 1e-4, 1e-4) + ) + + @flow.unittest.skip_unless_1n4d() + def test_swinv2_loader_with_data_tensor_parallel_backward(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = SwinV2LoaderHuggerFace( + model=libai.models.SwinTransformerV2, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + drop_rate=0, + drop_path_rate=0, + ) + model = load_func.load() + + input_image = flow.tensor( + self.input_image.tolist(), + dtype=flow.float32, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.patch_embed.proj.weight.placement, + ) + + prediction_scores = model(input_image)["prediction_scores"] + loss = prediction_scores.sum() + loss.backward() + + self.assertTrue(np.allclose(373520.47, model.head.weight.grad.sum().numpy(), 1e-3)) + self.assertTrue( + np.allclose(259.379, model.patch_embed.norm.weight.grad.sum().numpy(), 1e-3) + ) + + @flow.unittest.skip_unless_1n4d() + def test_swinv2_loader_with_data_tensor_pipeline_parallel_backward(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=1, + pipeline_parallel_size=2, + pipeline_num_layers=12, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = SwinV2LoaderHuggerFace( + model=libai.models.SwinTransformerV2, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + drop_rate=0, + drop_path_rate=0, + ) + model = load_func.load() + + input_image = flow.tensor( + self.input_image.tolist(), + dtype=flow.float32, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.patch_embed.proj.weight.placement, + ) + + prediction_scores = model(input_image)["prediction_scores"] + loss = prediction_scores.sum() + loss.backward() + + self.assertTrue(np.allclose(373520.47, model.head.weight.grad.sum().numpy(), 1e-3)) + self.assertTrue( + np.allclose(259.379, model.patch_embed.norm.weight.grad.sum().numpy(), 1e-3) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/model_utils/test_t5_loader.py b/tests/model_utils/test_t5_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..0a7c94360139cdf71462f7243914a91c2749501a --- /dev/null +++ b/tests/model_utils/test_t5_loader.py @@ -0,0 +1,208 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import shutil +import unittest + +import numpy as np +import oneflow as flow +import oneflow.unittest +from omegaconf import DictConfig + +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger +from projects.MT5.configs.mt5_base import cfg as libai_cfg +from projects.MT5.mt5_model import MT5Model +from projects.MT5.utils.mt5_loader import T5LoaderHuggerFace + +PRETRAINED_MODEL_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/t5_utils/pytorch_model.bin" # noqa +PRETRAINED_MODEL_CONFIG_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/t5_utils/config.json" # noqa + +PRETRAINED_MODEL_MD5 = "952862a8ba425a25739a69e5f33b0df8" +PRETRAINED_MODEL_CONFIG_MD5 = "7ebc91dc4377c01190f4116c3c1ac6cd" + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_t5_utils") + + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestT5Loader(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join( + os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "t5_utils_data" + ) + self.pretrained_model_path = cache_dir + + # prepare dataset + if dist.get_local_rank() == 0: + # download dataset on main process of each node + get_data_from_cache(PRETRAINED_MODEL_URL, cache_dir, md5=PRETRAINED_MODEL_MD5) + get_data_from_cache( + PRETRAINED_MODEL_CONFIG_URL, cache_dir, md5=PRETRAINED_MODEL_CONFIG_MD5 + ) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + # prepare input data + self.encoder_input_ids = [ + [101, 2009, 1005, 1055, 2986, 2651, 1012, 102], + [101, 2028, 12314, 3377, 102, 0, 0, 0], + [101, 2064, 2017, 3305, 2009, 102, 0, 0], + ] + self.encoder_att_mask = [ + [1, 1, 1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1, 1], + ] + self.decoder_input_ids = [ + [101, 2009, 1005, 1055, 2986], + [101, 2028, 12314, 3377, 102], + [101, 2064, 2017, 3305, 2009], + ] + self.decoder_att_mask = [[1, 1, 1, 1, 1], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1]] + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_t5_loader_with_data_tensor_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = T5LoaderHuggerFace( + model=MT5Model, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + embedding_dropout_prob=0.0, + model_type="t5", + ) + model = load_func.load() + model.eval() + + encoder_input_ids = flow.tensor( + self.encoder_input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + decoder_input_ids = flow.tensor( + self.decoder_input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + encode_att_mask = flow.tensor( + self.encoder_att_mask, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + decoder_att_mask = flow.tensor( + self.decoder_att_mask, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + + logits = model( + encoder_input_ids, decoder_input_ids, encode_att_mask, decoder_att_mask, encode_att_mask + )["logits"] + self.assertTrue( + np.allclose( + np.array(-9836561.0), + logits.sum().data.numpy(), + ) + ) + + @flow.unittest.skip_unless_1n4d() + def test_t5_loader_with_data_tensor_pipeline_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=1, + pipeline_parallel_size=2, + pipeline_num_layers=24, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = T5LoaderHuggerFace( + model=MT5Model, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, + embedding_dropout_prob=0.0, + model_type="t5", + ) + model = load_func.load() + model.eval() + + encoder_input_ids = flow.tensor( + self.encoder_input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + decoder_input_ids = flow.tensor( + self.decoder_input_ids, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + encode_att_mask = flow.tensor( + self.encoder_att_mask, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + decoder_att_mask = flow.tensor( + self.decoder_att_mask, + dtype=flow.long, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=dist.get_layer_placement(0), + ) + + logits = model( + encoder_input_ids, decoder_input_ids, encode_att_mask, decoder_att_mask, encode_att_mask + )["logits"] + self.assertTrue( + np.allclose( + np.array(-9836561.0), + logits.sum().data.numpy(), + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/model_utils/test_vit_loader.py b/tests/model_utils/test_vit_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..69dfa85091569753b2483500c5242cd7229d08b3 --- /dev/null +++ b/tests/model_utils/test_vit_loader.py @@ -0,0 +1,218 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import shutil +import unittest + +import numpy as np +import oneflow as flow +import oneflow.unittest +from omegaconf import DictConfig + +import libai +from configs.common.models.vit.vit_tiny_patch16_224 import cfg as libai_cfg +from libai.models.utils import ViTLoaderHuggerFace +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger + +PRETRAINED_MODEL_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/vit_utils/pytorch_model.bin" # noqa +PRETRAINED_MODEL_CONFIG_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/vit_utils/config.json" # noqa +INIT_DATA = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/model_utils_test/vit_utils/init_data.npz" # noqa + +PRETRAINED_MODEL_MD5 = "c587693e5e312064c56f27aa2d4f1e81" +PRETRAINED_MODEL_CONFIG_MD5 = "9ea94d9e5bc3543b1de7d12956321c50" +INIT_DATA_MD5 = "5fecdcd8d46bfefa310d19e084bd4815" + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_vit_utils") + + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestViTLoder(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join( + os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "vit_utils_data" + ) + self.pretrained_model_path = cache_dir + self.init_data_path = os.path.join(cache_dir, "init_data.npz") + + # download model and data + if dist.get_local_rank() == 0: + # download dataset on main process of each node + get_data_from_cache(PRETRAINED_MODEL_URL, cache_dir, md5=PRETRAINED_MODEL_MD5) + get_data_from_cache( + PRETRAINED_MODEL_CONFIG_URL, cache_dir, md5=PRETRAINED_MODEL_CONFIG_MD5 + ) + get_data_from_cache(INIT_DATA, cache_dir, md5=INIT_DATA_MD5) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + # prepare input data + self.input_image = np.load(self.init_data_path)["arr_0"] + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_vit_loader_with_data_tensor_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = ViTLoaderHuggerFace( + model=libai.models.VisionTransformer, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + ) + model = load_func.load() + model.eval() + + input_image = flow.tensor( + self.input_image.tolist(), + dtype=flow.float32, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.patch_embed.proj.weight.placement, + ) + + prediction_scores = model(input_image)["prediction_scores"] + + self.assertTrue( + np.allclose(np.array(3.1374), prediction_scores.sum().data.numpy(), 1e-4, 1e-4) + ) + + @flow.unittest.skip_unless_1n4d() + def test_vit_loader_with_data_tensor_pipeline_parallel(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=1, + pipeline_parallel_size=2, + pipeline_num_layers=12, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = ViTLoaderHuggerFace( + model=libai.models.VisionTransformer, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + ) + model = load_func.load() + model.eval() + + input_image = flow.tensor( + self.input_image, + dtype=flow.float32, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.patch_embed.proj.weight.placement, + ) + + prediction_scores = model(input_image)["prediction_scores"] + + self.assertTrue( + np.allclose(np.array(3.1374), prediction_scores.sum().data.numpy(), 1e-4, 1e-4) + ) + + @flow.unittest.skip_unless_1n4d() + def test_vit_loader_with_data_tensor_parallel_backward(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=2, + pipeline_parallel_size=1, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = ViTLoaderHuggerFace( + model=libai.models.VisionTransformer, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + drop_rate=0, + attn_drop_rate=0, + drop_path_rate=0, + ) + model = load_func.load() + + input_image = flow.tensor( + self.input_image, + dtype=flow.float32, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.patch_embed.proj.weight.placement, + ) + + prediction_scores = model(input_image)["prediction_scores"] + loss = prediction_scores.sum() + loss.backward() + + self.assertTrue(np.allclose(-173459.77, model.head.weight.grad.sum().numpy(), 1e-3)) + + @flow.unittest.skip_unless_1n4d() + def test_vit_loader_with_data_tensor_pipeline_parallel_backward(self): + # set distributed config + dist_cfg = DictConfig( + dict( + data_parallel_size=2, + tensor_parallel_size=1, + pipeline_parallel_size=2, + pipeline_num_layers=12, + ) + ) + dist.setup_dist_util(dist_cfg) + + # load model + load_func = ViTLoaderHuggerFace( + model=libai.models.VisionTransformer, + libai_cfg=libai_cfg, + pretrained_model_path=self.pretrained_model_path, + drop_rate=0, + attn_drop_rate=0, + drop_path_rate=0, + ) + model = load_func.load() + + input_image = flow.tensor( + self.input_image, + dtype=flow.float32, + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.broadcast]), + placement=model.patch_embed.proj.weight.placement, + ) + + prediction_scores = model(input_image)["prediction_scores"] + loss = prediction_scores.sum() + loss.backward() + + self.assertTrue(np.allclose(-173459.77, model.head.weight.grad.sum().numpy(), 1e-3)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/models/__init__.py b/tests/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/models/test_bert.py b/tests/models/test_bert.py new file mode 100644 index 0000000000000000000000000000000000000000..71d04d79ae53f8350e2f9751bc0d670fa9e6be26 --- /dev/null +++ b/tests/models/test_bert.py @@ -0,0 +1,180 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import shutil +import unittest + +import oneflow as flow +import oneflow.unittest + +from libai.config import LazyConfig +from libai.engine import DefaultTrainer +from libai.engine.default import _check_batch_size +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger + +VOCAB_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/bert-base-chinese-vocab.txt" # noqa +BIN_DATA_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/loss_compara_content_sentence.bin" # noqa +IDX_DATA_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/loss_compara_content_sentence.idx" # noqa + +VOCAB_MD5 = "3b5b76c4aef48ecf8cb3abaafe960f09" +BIN_DATA_MD5 = "b842467bd5ea7e52f7a612ea6b4faecc" +IDX_DATA_MD5 = "cf5963b8543f0a7a867361eb980f0372" + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_bert") + + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestBertModel(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join(os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "bert_data") + + cfg = LazyConfig.load("configs/bert_large_pretrain.py") + + # prepare dataset + if dist.get_local_rank() == 0: + # download dataset on main process of each node + get_data_from_cache(VOCAB_URL, cache_dir, md5=VOCAB_MD5) + get_data_from_cache(BIN_DATA_URL, cache_dir, md5=BIN_DATA_MD5) + get_data_from_cache(IDX_DATA_URL, cache_dir, md5=IDX_DATA_MD5) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + vocab_path = get_data_from_cache(VOCAB_URL, cache_dir, md5=VOCAB_MD5) + data_prefix_path = get_data_from_cache(BIN_DATA_URL, cache_dir, md5=BIN_DATA_MD5) + data_prefix = data_prefix_path[:-4] + + # set tokenizer and data config + cfg.tokenization.tokenizer.vocab_file = vocab_path + cfg.dataloader.train.dataset[0].data_prefix = data_prefix + cfg.dataloader.train.dataset[0].indexed_dataset.data_prefix = data_prefix + # FIXME(RenTianhe): fix dataloader worker bug + cfg.dataloader.train.num_workers = 0 + + # set training config + cfg.train.train_epoch = 0 + cfg.train.train_iter = 10 + cfg.train.evaluation.eval_period = 10 + cfg.train.evaluation.eval_iter = 10 + cfg.train.log_period = 1 + cfg.train.train_micro_batch_size = 8 + cfg.train.test_micro_batch_size = 4 + cfg.train.num_accumulation_steps = 1 + cfg.train.resume = False + cfg.train.output_dir = TEST_OUTPUT + + # set model + cfg.model.cfg.num_attention_heads = 8 + cfg.model.cfg.hidden_size = 384 + cfg.model.cfg.hidden_layers = 4 + cfg.train.activation_checkpoint.enabled = True + cfg.train.amp.enabled = True + + cfg.train.rdma_enabled = False + + self.cfg = cfg + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_bert_eager_with_data_tensor_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + self.cfg.train.dist.tensor_parallel_size = 2 + # pipeline parallelism not supported in eager global now! + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_bert_eager_with_pipeline_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 1 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 4 + self.cfg.train.dist.pipeline_num_layers = self.cfg.model.cfg.hidden_layers + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_bert_graph_with_data_tensor_parallel(self): + self.cfg.train.num_accumulation_steps = 1 + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + self.cfg.train.dist.tensor_parallel_size = 2 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_bert_graph_with_data_tensor_pipeline_parallel(self): + self.cfg.train.num_accumulation_steps = 4 + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + # change to 2 when 2d sbp bugfix + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 2 + self.cfg.train.dist.pipeline_num_layers = self.cfg.model.cfg.hidden_layers + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + @unittest.skip("There are still bugs in ZeRO") + def test_bert_with_zero(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 4 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + self.cfg.train.zero_optimization.enabled = True + self.cfg.train.zero_optimization.stage = 3 + trainer = DefaultTrainer(self.cfg) + trainer.train() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/models/test_gpt.py b/tests/models/test_gpt.py new file mode 100644 index 0000000000000000000000000000000000000000..a7918a178c837f8cc8061084ca15ddb8d1c721cb --- /dev/null +++ b/tests/models/test_gpt.py @@ -0,0 +1,204 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import shutil +import unittest + +import oneflow as flow +import oneflow.unittest + +from libai.config import LazyConfig +from libai.engine import DefaultTrainer, hooks +from libai.engine.default import _check_batch_size +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger + +VOCAB_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/gpt_dataset/gpt2-vocab.json" # noqa +MERGE_FILE_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/gpt_dataset/gpt2-merges.txt" # noqa +BIN_DATA_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/loss_compara_content_sentence.bin" # noqa +IDX_DATA_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/loss_compara_content_sentence.idx" # noqa + +VOCAB_MD5 = "dffec25a898b1f5e569bec4dffd7e5c0" +MERGE_FILE_MD5 = "75a37753dd7a28a2c5df80c28bf06e4e" +BIN_DATA_MD5 = "b842467bd5ea7e52f7a612ea6b4faecc" +IDX_DATA_MD5 = "cf5963b8543f0a7a867361eb980f0372" + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_gpt") + + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestGPTModel(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join(os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "gpt_data") + + cfg = LazyConfig.load("configs/gpt2_pretrain.py") + + # prepare dataset + if dist.get_local_rank() == 0: + # download dataset on main process of each node + get_data_from_cache(VOCAB_URL, cache_dir, md5=VOCAB_MD5) + get_data_from_cache(MERGE_FILE_URL, cache_dir, md5=MERGE_FILE_MD5) + get_data_from_cache(BIN_DATA_URL, cache_dir, md5=BIN_DATA_MD5) + get_data_from_cache(IDX_DATA_URL, cache_dir, md5=IDX_DATA_MD5) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + vocab_path = get_data_from_cache(VOCAB_URL, cache_dir, md5=VOCAB_MD5) + merges_file = get_data_from_cache(MERGE_FILE_URL, cache_dir, md5=MERGE_FILE_MD5) + data_prefix_path = get_data_from_cache(BIN_DATA_URL, cache_dir, md5=BIN_DATA_MD5) + data_prefix = data_prefix_path[:-4] + + # set tokenizer and data config + cfg.tokenization.tokenizer.vocab_file = vocab_path + cfg.tokenization.tokenizer.merges_file = merges_file + cfg.dataloader.train.dataset[0].data_prefix = data_prefix + cfg.dataloader.train.dataset[0].indexed_dataset.data_prefix = data_prefix + + cfg.dataloader.train.num_workers = 0 + + # set training config + cfg.train.train_epoch = 0 + cfg.train.train_iter = 10 + cfg.train.evaluation.enabled = True + cfg.train.evaluation.eval_period = 10 + cfg.train.evaluation.eval_iter = 10 + cfg.train.log_period = 1 + cfg.train.train_micro_batch_size = 4 + cfg.train.num_accumulation_steps = 1 + cfg.train.resume = False + cfg.train.output_dir = TEST_OUTPUT + + # set model + cfg.model.cfg.max_seq_length = 256 + cfg.model.cfg.num_attention_heads = 8 + cfg.model.cfg.hidden_size = 384 + cfg.model.cfg.hidden_layers = 4 + cfg.model.cfg.hidden_layers = 4 + cfg.train.activation_checkpoint.enabled = True + cfg.train.amp.enabled = True + + cfg.train.rdma_enabled = False + + for ds in cfg.dataloader.train.dataset: + ds.max_seq_length = cfg.model.cfg.max_seq_length + + self.cfg = cfg + + def build_hooks(self): + ret = [ + hooks.IterationTimer(), + hooks.LRScheduler(), + ] + + if dist.is_main_process(): + # run writers in the end, so that evaluation metrics are written + ret.append(hooks.PeriodicWriter(self.build_writers(), self.cfg.train.log_period)) + return ret + + DefaultTrainer.build_hooks = build_hooks + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_gpt_eager_with_data_tensor_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + self.cfg.train.dist.tensor_parallel_size = 2 + # pipeline parallelism not supported in eager global now! + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_gpt_eager_with_pipeline_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 1 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 4 + self.cfg.train.dist.pipeline_num_layers = self.cfg.model.cfg.hidden_layers + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_gpt_graph_with_data_tensor_parallel(self): + self.cfg.train.num_accumulation_steps = 1 + + # set distributed config + self.cfg.train.dist.data_parallel_size = 4 + self.cfg.train.dist.tensor_parallel_size = 2 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_gpt_graph_with_data_tensor_pipeline_parallel(self): + self.cfg.train.num_accumulation_steps = 4 + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + # change to 2 when 2d sbp bugfix + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 2 + self.cfg.train.dist.pipeline_num_layers = self.cfg.model.cfg.hidden_layers + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + @unittest.skip("There are still bugs in ZeRO") + def test_gpt_with_zero(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 4 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + self.cfg.train.zero_optimization.enabled = True + self.cfg.train.zero_optimization.stage = 3 + trainer = DefaultTrainer(self.cfg) + trainer.train() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/models/test_mt5.py b/tests/models/test_mt5.py new file mode 100644 index 0000000000000000000000000000000000000000..64f664b16a0d60a4a6960a66132396d2921b67de --- /dev/null +++ b/tests/models/test_mt5.py @@ -0,0 +1,193 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import shutil +import unittest + +import oneflow as flow +import oneflow.unittest + +from libai.config import LazyConfig +from libai.engine import DefaultTrainer, hooks +from libai.engine.default import _check_batch_size +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger + +VOCAB_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/bert-base-chinese-vocab.txt" # noqa +BIN_DATA_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/loss_compara_content_sentence.bin" # noqa +IDX_DATA_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/loss_compara_content_sentence.idx" # noqa + +VOCAB_MD5 = "3b5b76c4aef48ecf8cb3abaafe960f09" +BIN_DATA_MD5 = "b842467bd5ea7e52f7a612ea6b4faecc" +IDX_DATA_MD5 = "cf5963b8543f0a7a867361eb980f0372" + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_mt5") + + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestMT5Model(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join(os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "bert_data") + + cfg = LazyConfig.load("projects/MT5/configs/mt5_pretrain.py") + + # prepare dataset + if dist.get_local_rank() == 0: + # download dataset on main process of each node + get_data_from_cache(VOCAB_URL, cache_dir, md5=VOCAB_MD5) + get_data_from_cache(BIN_DATA_URL, cache_dir, md5=BIN_DATA_MD5) + get_data_from_cache(IDX_DATA_URL, cache_dir, md5=IDX_DATA_MD5) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + vocab_path = get_data_from_cache(VOCAB_URL, cache_dir, md5=VOCAB_MD5) + data_prefix_path = get_data_from_cache(BIN_DATA_URL, cache_dir, md5=BIN_DATA_MD5) + data_prefix = data_prefix_path[:-4] + + # set tokenizer and data config + cfg.tokenization.tokenizer.vocab_file = vocab_path + cfg.dataloader.train.dataset[0].data_prefix = data_prefix + cfg.dataloader.train.dataset[0].indexed_dataset.data_prefix = data_prefix + # FIXME(RenTianhe): fix dataloader worker bug + cfg.dataloader.train.num_workers = 0 + + # set training config + cfg.train.train_epoch = 0 + cfg.train.train_iter = 10 + cfg.train.evaluation.enabled = True + cfg.train.evaluation.eval_period = 10 + cfg.train.evaluation.eval_iter = 10 + cfg.train.log_period = 1 + cfg.train.train_micro_batch_size = 8 + cfg.train.num_accumulation_steps = 1 + cfg.train.resume = False + cfg.train.output_dir = TEST_OUTPUT + + # set model + cfg.model.cfg.num_attention_heads = 8 + cfg.model.cfg.hidden_size = 384 + cfg.model.cfg.hidden_layers = 3 + cfg.train.activation_checkpoint.enabled = True + cfg.train.amp.enabled = True + + cfg.train.rdma_enabled = False + + self.cfg = cfg + + def build_hooks(self): + ret = [ + hooks.IterationTimer(), + hooks.LRScheduler(), + ] + + if dist.is_main_process(): + # run writers in the end, so that evaluation metrics are written + ret.append(hooks.PeriodicWriter(self.build_writers(), self.cfg.train.log_period)) + return ret + + DefaultTrainer.build_hooks = build_hooks + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_mt5_eager_with_data_tensor_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + self.cfg.train.dist.tensor_parallel_size = 2 + # pipeline parallelism not supported in eager global now! + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_mt5_eager_with_pipeline_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 1 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 4 + self.cfg.train.dist.pipeline_num_layers = 2 * self.cfg.model.cfg.hidden_layers + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_mt5_graph_with_data_tensor_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + self.cfg.train.dist.tensor_parallel_size = 2 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_mt5_graph_with_data_tensor_pipeline_parallel(self): + self.cfg.train.num_accumulation_steps = 4 + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + # change to 2 when 2d sbp bugfix + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 2 + # encoder_layers + decoder_layers + self.cfg.train.dist.pipeline_num_layers = 2 * self.cfg.model.cfg.hidden_layers + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + @unittest.skip("There are still bugs in ZeRO") + def test_mt5_with_zero(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 4 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + self.cfg.train.zero_optimization.enabled = True + self.cfg.train.zero_optimization.stage = 3 + trainer = DefaultTrainer(self.cfg) + trainer.train() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/models/test_roberta.py b/tests/models/test_roberta.py new file mode 100644 index 0000000000000000000000000000000000000000..d607b3fdddce11bd215961629bd829cb1167a026 --- /dev/null +++ b/tests/models/test_roberta.py @@ -0,0 +1,185 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import shutil +import unittest + +import oneflow as flow +import oneflow.unittest + +from libai.config import LazyConfig +from libai.engine import DefaultTrainer +from libai.engine.default import _check_batch_size +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger + +VOCAB_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/roberta_dataset/roberta-vocab.json" # noqa +MERGE_FILE_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/roberta_dataset/roberta-merges.txt" # noqa +BIN_DATA_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/roberta_dataset/loss_compara_content_sentence.bin" # noqa +IDX_DATA_URL = "http://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/roberta_dataset/loss_compara_content_sentence.idx" # noqa + +VOCAB_MD5 = "be4d3c6f3f5495426b2c03b334334354" +MERGES_FILE_MD5 = "75a37753dd7a28a2c5df80c28bf06e4e" +BIN_DATA_MD5 = "b842467bd5ea7e52f7a612ea6b4faecc" +IDX_DATA_MD5 = "cf5963b8543f0a7a867361eb980f0372" + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_roberta") + + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestRoBERTaModel(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join(os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "roberta_data") + + cfg = LazyConfig.load("configs/roberta_pretrain.py") + + # prepare dataset + if dist.get_local_rank() == 0: + # download dataset on main process of each node + get_data_from_cache(VOCAB_URL, cache_dir, md5=VOCAB_MD5) + get_data_from_cache(MERGE_FILE_URL, cache_dir, md5=MERGES_FILE_MD5) + get_data_from_cache(BIN_DATA_URL, cache_dir, md5=BIN_DATA_MD5) + get_data_from_cache(IDX_DATA_URL, cache_dir, md5=IDX_DATA_MD5) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + vocab_path = get_data_from_cache(VOCAB_URL, cache_dir, md5=VOCAB_MD5) + merges_file_path = get_data_from_cache(MERGE_FILE_URL, cache_dir, md5=MERGES_FILE_MD5) + data_prefix_path = get_data_from_cache(BIN_DATA_URL, cache_dir, md5=BIN_DATA_MD5) + data_prefix = data_prefix_path[:-4] + + # set tokenizer and data config + cfg.tokenization.tokenizer.vocab_file = vocab_path + cfg.tokenization.tokenizer.merges_file = merges_file_path + cfg.dataloader.train.dataset[0].data_prefix = data_prefix + cfg.dataloader.train.dataset[0].indexed_dataset.data_prefix = data_prefix + + cfg.dataloader.train.num_workers = 0 + + # set training config + cfg.train.train_epoch = 0 + cfg.train.train_iter = 10 + cfg.train.evaluation.eval_period = 10 + cfg.train.evaluation.eval_iter = 10 + cfg.train.log_period = 1 + cfg.train.train_micro_batch_size = 8 + cfg.train.test_micro_batch_size = 4 + cfg.train.num_accumulation_steps = 1 + cfg.train.resume = False + cfg.train.output_dir = TEST_OUTPUT + + # set model + cfg.model.cfg.num_attention_heads = 8 + cfg.model.cfg.hidden_size = 384 + cfg.model.cfg.hidden_layers = 4 + cfg.train.activation_checkpoint.enabled = True + cfg.train.amp.enabled = True + + cfg.train.rdma_enabled = False + + self.cfg = cfg + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_roberta_eager_with_data_tensor_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + self.cfg.train.dist.tensor_parallel_size = 2 + # pipeline parallelism not supported in eager global now! + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_roberta_eager_with_pipeline_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 1 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 4 + self.cfg.train.dist.pipeline_num_layers = self.cfg.model.cfg.hidden_layers + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_roberta_graph_with_data_tensor_parallel(self): + self.cfg.train.num_accumulation_steps = 1 + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + self.cfg.train.dist.tensor_parallel_size = 2 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_roberta_graph_with_data_tensor_pipeline_parallel(self): + self.cfg.train.num_accumulation_steps = 4 + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + # change to 2 when 2d sbp bugfix + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 2 + self.cfg.train.dist.pipeline_num_layers = self.cfg.model.cfg.hidden_layers + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + @unittest.skip("There are still bugs in ZeRO") + def test_roberta_with_zero(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 4 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + self.cfg.train.zero_optimization.enabled = True + self.cfg.train.zero_optimization.stage = 3 + trainer = DefaultTrainer(self.cfg) + trainer.train() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/models/test_swin.py b/tests/models/test_swin.py new file mode 100644 index 0000000000000000000000000000000000000000..a9b88e5ecf14c2ea04dccdb367a65da3afc5eced --- /dev/null +++ b/tests/models/test_swin.py @@ -0,0 +1,178 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import unittest + +import oneflow as flow +import oneflow.unittest +from flowvision.loss.cross_entropy import SoftTargetCrossEntropy + +import libai.utils.distributed as dist +from configs.common.models.swin.swin_tiny_patch4_window7_224 import model +from libai.config import LazyConfig +from libai.data.datasets import CIFAR10Dataset +from libai.engine import DefaultTrainer +from libai.engine.default import _check_batch_size +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger + +DATA_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/cifar10/cifar-10-python.tar.gz" # noqa + +DATA_MD5 = "c58f30108f718f92721af3b95e74349a" + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_swin") + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestSwinModel(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join(os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "swin_data") + + cfg = LazyConfig.load("configs/swin_cifar100.py") + + # set model + cfg.model = model + cfg.model.cfg.num_classes = 10 + cfg.model.cfg.loss_func = SoftTargetCrossEntropy() + + # prepare data path + if dist.get_local_rank() == 0: + get_data_from_cache(DATA_URL, cache_dir, md5=DATA_MD5) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + data_path = get_data_from_cache(DATA_URL, cache_dir, md5=DATA_MD5) + + cfg.dataloader.train.dataset[0]._target_ = CIFAR10Dataset + cfg.dataloader.train.dataset[0].root = "/".join(data_path.split("/")[:-1]) + cfg.dataloader.train.dataset[0].download = True + cfg.dataloader.train.num_workers = 0 + + cfg.dataloader.test[0].dataset._target_ = CIFAR10Dataset + cfg.dataloader.test[0].dataset.train = False + cfg.dataloader.test[0].dataset.root = "/".join(data_path.split("/")[:-1]) + cfg.dataloader.test[0].dataset.download = True + cfg.dataloader.test[0].num_workers = 0 + + # refine mixup cfg + cfg.dataloader.train.mixup_func.num_classes = 10 + + # set training config + cfg.train.train_epoch = 0 + cfg.train.train_iter = 10 + cfg.train.evaluation.eval_period = 10 + cfg.train.evaluation.eval_iter = 10 + cfg.train.log_period = 1 + cfg.train.train_micro_batch_size = 8 + cfg.train.num_accumulation_steps = 1 + cfg.train.resume = False + cfg.train.output_dir = TEST_OUTPUT + cfg.train.activation_checkpoint.enabled = True + cfg.train.amp.enabled = True + cfg.train.rdma_enabled = False + + self.cfg = cfg + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_swin_eager_with_data_tensor_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + self.cfg.train.dist.tensor_parallel_size = 2 + # pipeline parallelism not supported in eager global now! + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_swin_eager_with_pipeline_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 1 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 4 + self.cfg.train.dist.pipeline_num_layers = sum(self.cfg.model.cfg.depths) + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_swin_graph_with_data_tensor_parallel(self): + self.cfg.train.num_accumulation_steps = 1 + + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + self.cfg.train.dist.tensor_parallel_size = 2 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_swin_graph_with_data_tensor_pipeline_parallel(self): + self.cfg.train.num_accumulation_steps = 4 + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + # change to 2 when 2d sbp bugfix + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 2 + self.cfg.train.dist.pipeline_num_layers = sum(self.cfg.model.cfg.depths) + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + @unittest.skip("There are still bugs in ZeRO") + def test_swin_with_zero(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 4 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + self.cfg.train.zero_optimization.enabled = True + self.cfg.train.zero_optimization.stage = 3 + trainer = DefaultTrainer(self.cfg) + trainer.train() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/models/test_swinv2.py b/tests/models/test_swinv2.py new file mode 100644 index 0000000000000000000000000000000000000000..af0da47aab6af93c30927da7cf64a5bf473454ac --- /dev/null +++ b/tests/models/test_swinv2.py @@ -0,0 +1,178 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import unittest + +import oneflow as flow +import oneflow.unittest +from flowvision.loss.cross_entropy import SoftTargetCrossEntropy + +import libai.utils.distributed as dist +from configs.common.models.swinv2.swinv2_tiny_patch4_window8_256 import model +from libai.config import LazyConfig +from libai.data.datasets import CIFAR10Dataset +from libai.engine import DefaultTrainer +from libai.engine.default import _check_batch_size +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger + +DATA_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/cifar10/cifar-10-python.tar.gz" # noqa + +DATA_MD5 = "c58f30108f718f92721af3b95e74349a" + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_swinv2") + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestSwinV2Model(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join(os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "swinv2_data") + + cfg = LazyConfig.load("configs/swinv2_cifar100.py") + + # set model + cfg.model = model + cfg.model.cfg.num_classes = 10 + cfg.model.cfg.loss_func = SoftTargetCrossEntropy() + + # prepare data path + if dist.get_local_rank() == 0: + get_data_from_cache(DATA_URL, cache_dir, md5=DATA_MD5) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + data_path = get_data_from_cache(DATA_URL, cache_dir, md5=DATA_MD5) + + cfg.dataloader.train.dataset[0]._target_ = CIFAR10Dataset + cfg.dataloader.train.dataset[0].root = "/".join(data_path.split("/")[:-1]) + cfg.dataloader.train.dataset[0].download = True + cfg.dataloader.train.num_workers = 0 + + cfg.dataloader.test[0].dataset._target_ = CIFAR10Dataset + cfg.dataloader.test[0].dataset.train = False + cfg.dataloader.test[0].dataset.root = "/".join(data_path.split("/")[:-1]) + cfg.dataloader.test[0].dataset.download = True + cfg.dataloader.test[0].num_workers = 0 + + # refine mixup cfg + cfg.dataloader.train.mixup_func.num_classes = 10 + + # set training config + cfg.train.train_epoch = 0 + cfg.train.train_iter = 10 + cfg.train.evaluation.eval_period = 10 + cfg.train.evaluation.eval_iter = 10 + cfg.train.log_period = 1 + cfg.train.train_micro_batch_size = 8 + cfg.train.num_accumulation_steps = 1 + cfg.train.resume = False + cfg.train.output_dir = TEST_OUTPUT + cfg.train.activation_checkpoint.enabled = True + cfg.train.amp.enabled = True + cfg.train.rdma_enabled = False + + self.cfg = cfg + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_swinv2_eager_with_data_tensor_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + self.cfg.train.dist.tensor_parallel_size = 2 + # pipeline parallelism not supported in eager global now! + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_swinv2_eager_with_pipeline_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 1 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 4 + self.cfg.train.dist.pipeline_num_layers = sum(self.cfg.model.cfg.depths) + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_swinv2_graph_with_data_tensor_parallel(self): + self.cfg.train.num_accumulation_steps = 1 + + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + self.cfg.train.dist.tensor_parallel_size = 2 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_swinv2_graph_with_data_tensor_pipeline_parallel(self): + self.cfg.train.num_accumulation_steps = 4 + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + # change to 2 when 2d sbp bugfix + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 2 + self.cfg.train.dist.pipeline_num_layers = sum(self.cfg.model.cfg.depths) + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + @unittest.skip("There are still bugs in ZeRO") + def test_swinv2_with_zero(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 4 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + self.cfg.train.zero_optimization.enabled = True + self.cfg.train.zero_optimization.stage = 3 + trainer = DefaultTrainer(self.cfg) + trainer.train() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/models/test_t5.py b/tests/models/test_t5.py new file mode 100644 index 0000000000000000000000000000000000000000..140f302091ed0e4f31908f94930c37a1d76b5297 --- /dev/null +++ b/tests/models/test_t5.py @@ -0,0 +1,193 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import shutil +import unittest + +import oneflow as flow +import oneflow.unittest + +from libai.config import LazyConfig +from libai.engine import DefaultTrainer, hooks +from libai.engine.default import _check_batch_size +from libai.utils import distributed as dist +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger + +VOCAB_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/bert-base-chinese-vocab.txt" # noqa +BIN_DATA_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/loss_compara_content_sentence.bin" # noqa +IDX_DATA_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/bert_dataset/loss_compara_content_sentence.idx" # noqa + +VOCAB_MD5 = "3b5b76c4aef48ecf8cb3abaafe960f09" +BIN_DATA_MD5 = "b842467bd5ea7e52f7a612ea6b4faecc" +IDX_DATA_MD5 = "cf5963b8543f0a7a867361eb980f0372" + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_t5") + + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestT5Model(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join(os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "bert_data") + + cfg = LazyConfig.load("configs/t5_large_pretrain.py") + + # prepare dataset + if dist.get_local_rank() == 0: + # download dataset on main process of each node + get_data_from_cache(VOCAB_URL, cache_dir, md5=VOCAB_MD5) + get_data_from_cache(BIN_DATA_URL, cache_dir, md5=BIN_DATA_MD5) + get_data_from_cache(IDX_DATA_URL, cache_dir, md5=IDX_DATA_MD5) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + vocab_path = get_data_from_cache(VOCAB_URL, cache_dir, md5=VOCAB_MD5) + data_prefix_path = get_data_from_cache(BIN_DATA_URL, cache_dir, md5=BIN_DATA_MD5) + data_prefix = data_prefix_path[:-4] + + # set tokenizer and data config + cfg.tokenization.tokenizer.vocab_file = vocab_path + cfg.dataloader.train.dataset[0].data_prefix = data_prefix + cfg.dataloader.train.dataset[0].indexed_dataset.data_prefix = data_prefix + # FIXME(RenTianhe): fix dataloader worker bug + cfg.dataloader.train.num_workers = 0 + + # set training config + cfg.train.train_epoch = 0 + cfg.train.train_iter = 10 + cfg.train.evaluation.enabled = True + cfg.train.evaluation.eval_period = 10 + cfg.train.evaluation.eval_iter = 10 + cfg.train.log_period = 1 + cfg.train.train_micro_batch_size = 8 + cfg.train.num_accumulation_steps = 1 + cfg.train.resume = False + cfg.train.output_dir = TEST_OUTPUT + + # set model + cfg.model.cfg.num_attention_heads = 8 + cfg.model.cfg.hidden_size = 384 + cfg.model.cfg.hidden_layers = 3 + cfg.train.activation_checkpoint.enabled = True + cfg.train.amp.enabled = True + + cfg.train.rdma_enabled = False + + self.cfg = cfg + + def build_hooks(self): + ret = [ + hooks.IterationTimer(), + hooks.LRScheduler(), + ] + + if dist.is_main_process(): + # run writers in the end, so that evaluation metrics are written + ret.append(hooks.PeriodicWriter(self.build_writers(), self.cfg.train.log_period)) + return ret + + DefaultTrainer.build_hooks = build_hooks + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_t5_eager_with_data_tensor_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + self.cfg.train.dist.tensor_parallel_size = 2 + # pipeline parallelism not supported in eager global now! + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_t5_eager_with_pipeline_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 1 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 4 + self.cfg.train.dist.pipeline_num_layers = 2 * self.cfg.model.cfg.hidden_layers + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_t5_graph_with_data_tensor_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + self.cfg.train.dist.tensor_parallel_size = 2 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_t5_graph_with_data_tensor_pipeline_parallel(self): + self.cfg.train.num_accumulation_steps = 4 + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + # change to 2 when 2d sbp bugfix + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 2 + # encoder_layers + decoder_layers + self.cfg.train.dist.pipeline_num_layers = 2 * self.cfg.model.cfg.hidden_layers + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + @unittest.skip("There are still bugs in ZeRO") + def test_t5_with_zero(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 4 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + self.cfg.train.zero_optimization.enabled = True + self.cfg.train.zero_optimization.stage = 3 + trainer = DefaultTrainer(self.cfg) + trainer.train() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/models/test_vit.py b/tests/models/test_vit.py new file mode 100644 index 0000000000000000000000000000000000000000..211ace6992da74935ea62da8525c709bcd083c1e --- /dev/null +++ b/tests/models/test_vit.py @@ -0,0 +1,179 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import unittest + +import oneflow as flow +import oneflow.unittest +from flowvision.loss.cross_entropy import SoftTargetCrossEntropy + +import libai.utils.distributed as dist +from configs.common.models.vit.vit_small_patch16_224 import model +from libai.config import LazyCall, LazyConfig +from libai.data.datasets import CIFAR10Dataset +from libai.engine import DefaultTrainer +from libai.engine.default import _check_batch_size +from libai.utils.file_utils import get_data_from_cache +from libai.utils.logger import setup_logger + +DATA_URL = "https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/cifar10/cifar-10-python.tar.gz" # noqa + +DATA_MD5 = "c58f30108f718f92721af3b95e74349a" + +TEST_OUTPUT = os.path.join(os.getenv("TEST_OUTPUT", "output_unittest"), "test_vit") + +setup_logger(distributed_rank=dist.get_rank()) + + +class TestViTModel(flow.unittest.TestCase): + def setUp(self) -> None: + cache_dir = os.path.join(os.getenv("ONEFLOW_TEST_CACHE_DIR", "./data_test"), "vit_data") + + cfg = LazyConfig.load("configs/vit_imagenet.py") + + # set model + cfg.model = model + cfg.model.num_classes = 10 + cfg.model.depth = 6 + cfg.model.loss_func = LazyCall(SoftTargetCrossEntropy)() + + # prepare data path + if dist.get_local_rank() == 0: + get_data_from_cache(DATA_URL, cache_dir, md5=DATA_MD5) + os.makedirs(TEST_OUTPUT, exist_ok=True) + dist.synchronize() + + data_path = get_data_from_cache(DATA_URL, cache_dir, md5=DATA_MD5) + + cfg.dataloader.train.dataset[0]._target_ = CIFAR10Dataset + cfg.dataloader.train.dataset[0].root = "/".join(data_path.split("/")[:-1]) + cfg.dataloader.train.dataset[0].download = True + cfg.dataloader.train.num_workers = 0 + + cfg.dataloader.test[0].dataset._target_ = CIFAR10Dataset + cfg.dataloader.test[0].dataset.train = False + cfg.dataloader.test[0].dataset.root = "/".join(data_path.split("/")[:-1]) + cfg.dataloader.test[0].dataset.download = True + cfg.dataloader.test[0].num_workers = 0 + + # refine mixup cfg + cfg.dataloader.train.mixup_func.num_classes = 10 + + # set training config + cfg.train.train_epoch = 0 + cfg.train.train_iter = 10 + cfg.train.evaluation.eval_period = 10 + cfg.train.evaluation.eval_iter = 10 + cfg.train.log_period = 1 + cfg.train.train_micro_batch_size = 8 + cfg.train.num_accumulation_steps = 1 + cfg.train.resume = False + cfg.train.output_dir = TEST_OUTPUT + cfg.train.activation_checkpoint.enabled = True + cfg.train.amp.enabled = True + cfg.train.rdma_enabled = False + + self.cfg = cfg + + @classmethod + def tearDownClass(cls) -> None: + if os.path.isdir(TEST_OUTPUT) and dist.get_local_rank() == 0: + shutil.rmtree(TEST_OUTPUT) + + @flow.unittest.skip_unless_1n4d() + def test_vit_eager_with_data_tensor_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + self.cfg.train.dist.tensor_parallel_size = 2 + # pipeline parallelism not supported in eager global now! + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_vit_eager_with_pipeline_parallel(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 1 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 4 + self.cfg.train.dist.pipeline_num_layers = self.cfg.model.depth + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = False + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_vit_graph_with_data_tensor_parallel(self): + self.cfg.train.num_accumulation_steps = 1 + + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + self.cfg.train.dist.tensor_parallel_size = 2 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + def test_vit_graph_with_data_tensor_pipeline_parallel(self): + self.cfg.train.num_accumulation_steps = 4 + # set distributed config + self.cfg.train.dist.data_parallel_size = 2 + # change to 2 when 2d sbp bugfix + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 2 + self.cfg.train.dist.pipeline_num_layers = self.cfg.model.depth + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + trainer = DefaultTrainer(self.cfg) + trainer.train() + + @flow.unittest.skip_unless_1n4d() + @unittest.skip("There are still bugs in ZeRO") + def test_vit_with_zero(self): + # set distributed config + self.cfg.train.dist.data_parallel_size = 4 + self.cfg.train.dist.tensor_parallel_size = 1 + self.cfg.train.dist.pipeline_parallel_size = 1 + + dist.setup_dist_util(self.cfg.train.dist) + _check_batch_size(self.cfg) + + self.cfg.graph.enabled = True + self.cfg.train.zero_optimization.enabled = True + self.cfg.train.zero_optimization.stage = 3 + trainer = DefaultTrainer(self.cfg) + trainer.train() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/structures/__init__.py b/tests/structures/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/structures/test_instance.py b/tests/structures/test_instance.py new file mode 100644 index 0000000000000000000000000000000000000000..1e55450a43f2dffb07df30f215c9ff5c476ea311 --- /dev/null +++ b/tests/structures/test_instance.py @@ -0,0 +1,66 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import oneflow as flow + +from libai.data import DistTensorData, Instance + + +class TestInstance(unittest.TestCase): + def test_init_args(self): + inst = Instance(images=flow.rand(4, 5)) + inst.tokens = flow.rand(5, 10) + + self.assertTrue(inst.has("images")) + self.assertTrue(inst.has("tokens")) + + inst.remove("images") + self.assertFalse(inst.has("images")) + + inst.meta_tensor = DistTensorData(flow.rand(5, 6)) + self.assertTrue(inst.has("meta_tensor")) + self.assertTrue(isinstance(inst.get("meta_tensor"), DistTensorData)) + + def test_order_args(self): + inst = Instance(a=1, b=2, c=3) + inst.d = 4 + inst.e = 5 + + inst_key = [] + for key in inst.get_fields(): + inst_key.append(key) + + self.assertEqual(inst_key, ["a", "b", "c", "d", "e"]) + + def test_stack(self): + inst_list = [ + Instance(images=flow.rand(3, 4), masks=flow.rand(4, 5), bbox=[3, 4, 5, 6]) + for _ in range(10) + ] + + inst = Instance.stack(inst_list) + + self.assertTrue(inst.has("images")) + self.assertTrue(inst.has("masks")) + self.assertFalse(inst.has("tokens")) + self.assertEqual(inst.get("images").shape, (10, 3, 4)) + self.assertEqual(inst.get("masks").shape, (10, 4, 5)) + self.assertEqual(len(inst.get("bbox")), 10) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/structures/test_metadata.py b/tests/structures/test_metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..e5771ba721f107362b8d467f79b2af8a73956825 --- /dev/null +++ b/tests/structures/test_metadata.py @@ -0,0 +1,76 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import oneflow as flow + +from libai.data import DistTensorData +from libai.utils import distributed as dist + + +class TestMetadata(unittest.TestCase): + @unittest.skipIf(not flow.cuda.is_available(), "only test gpu cases") + def test_to_global(self): + x = flow.rand(10, 10) + x_meta = DistTensorData(x) + x_meta.to_global() + x_consistent = x.to_global( + sbp=flow.sbp.broadcast, + placement=flow.placement("cuda", [0]), + ) + + self.assertEqual(x_meta.tensor.sbp, x_consistent.sbp) + self.assertEqual(x_meta.tensor.placement, x_consistent.placement) + self.assertTrue((flow.equal(x_meta.tensor, x_consistent)).sum().item() == 100) + + x_meta.to_global( + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.split(0)]), + placement=dist.get_layer_placement(5), + ) + x_consistent = x.to_global( + sbp=dist.get_nd_sbp([flow.sbp.broadcast, flow.sbp.split(0)]), + placement=dist.get_layer_placement(5), + ) + + self.assertEqual(x_meta.tensor.sbp, x_consistent.sbp) + self.assertEqual(x_meta.tensor.placement, x_consistent.placement) + self.assertTrue((flow.equal(x_meta.tensor, x_consistent)).sum().item() == 100) + + @unittest.skipIf(not flow.cuda.is_available(), "only test gpu cases") + def test_stack(self): + x_list = [DistTensorData(flow.rand(10, 8)) for _ in range(5)] + + x_list.append(DistTensorData(flow.rand(10, 9))) # shape mismatch + with self.assertRaises(Exception): + DistTensorData.stack(x_list) + x_list.pop(-1) + + x_list.append(DistTensorData(flow.rand(10, 8), sbp_list=["broadcast"])) # sbp mismatch + with self.assertRaises(Exception): + DistTensorData.stack(x_list) + x_list.pop(-1) + + x_list.append(DistTensorData(flow.rand(10, 8), placement_idx=2)) # placement mismatch + with self.assertRaises(Exception): + DistTensorData.stack(x_list) + x_list.pop(-1) + + x_stack = DistTensorData.stack(x_list) + self.assertTrue(x_stack.tensor.shape == (5, 10, 8)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_file_io.py b/tests/test_file_io.py new file mode 100644 index 0000000000000000000000000000000000000000..f26f693c75da2d3eee3679babc69a3a080aa5e29 --- /dev/null +++ b/tests/test_file_io.py @@ -0,0 +1,314 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unittests followed https://github.com/facebookresearch/iopath/blob/v0.1.8/tests/test_file_io.py +""" + +import os +import shutil +import tempfile +import unittest +import uuid +from typing import Optional +from unittest.mock import MagicMock + +from libai.utils.file_io import LazyPath, PathManagerBase, PathManagerFactory, g_pathmgr + + +class TestNativeIO(unittest.TestCase): + _tmpdir: Optional[str] = None + _filename: Optional[str] = None + _tmpfile: Optional[str] = None + _tmpfile_contents = "Hello, World" + _pathmgr = PathManagerBase() + + @classmethod + def setUpClass(cls) -> None: + cls._tmpdir = tempfile.mkdtemp() + cls._filename = "test.txt" + # pyre-ignore + with open(os.path.join(cls._tmpdir, cls._filename), "w") as f: + cls._tmpfile = f.name + f.write(cls._tmpfile_contents) + f.flush() + + @classmethod + def tearDownClass(cls) -> None: + # Cleanup temp working dir. + if cls._tmpdir is not None: + shutil.rmtree(cls._tmpdir) # type: ignore + + def setUp(self) -> None: + # Reset class variables set by methods before each test. + self._pathmgr.set_cwd(None) + self._pathmgr._native_path_handler._non_blocking_io_manager = None + self._pathmgr._native_path_handler._non_blocking_io_executor = None + self._pathmgr._async_handlers.clear() + + def test_open(self) -> None: + # pyre-ignore + with self._pathmgr.open(self._tmpfile, "r") as f: + self.assertEqual(f.read(), self._tmpfile_contents) + + def test_factory_open(self) -> None: + with g_pathmgr.open(self._tmpfile, "r") as f: + self.assertEqual(f.read(), self._tmpfile_contents) + + _pathmgr = PathManagerFactory.get("test_pm") + with _pathmgr.open(self._tmpfile, "r") as f: + self.assertEqual(f.read(), self._tmpfile_contents) + + PathManagerFactory.remove("test_pm") + + def test_open_args(self) -> None: + self._pathmgr.set_strict_kwargs_checking(True) + f = self._pathmgr.open( + self._tmpfile, # type: ignore + mode="r", + buffering=1, + encoding="UTF-8", + errors="ignore", + newline=None, + closefd=True, + opener=None, + ) + f.close() + + def test_get_local_path(self) -> None: + self.assertEqual( + # pyre-ignore + self._pathmgr.get_local_path(self._tmpfile), + self._tmpfile, + ) + + def test_get_local_path_forced(self) -> None: + self.assertEqual( + # pyre-ignore + self._pathmgr.get_local_path(self._tmpfile, force=True), + self._tmpfile, + ) + + def test_exists(self) -> None: + # pyre-ignore + self.assertTrue(self._pathmgr.exists(self._tmpfile)) + # pyre-ignore + fake_path = os.path.join(self._tmpdir, uuid.uuid4().hex) + self.assertFalse(self._pathmgr.exists(fake_path)) + + def test_isfile(self) -> None: + self.assertTrue(self._pathmgr.isfile(self._tmpfile)) # pyre-ignore + # This is a directory, not a file, so it should fail + self.assertFalse(self._pathmgr.isfile(self._tmpdir)) # pyre-ignore + # This is a non-existing path, so it should fail + fake_path = os.path.join(self._tmpdir, uuid.uuid4().hex) # pyre-ignore + self.assertFalse(self._pathmgr.isfile(fake_path)) + + def test_isdir(self) -> None: + # pyre-ignore + self.assertTrue(self._pathmgr.isdir(self._tmpdir)) + # This is a file, not a directory, so it should fail + # pyre-ignore + self.assertFalse(self._pathmgr.isdir(self._tmpfile)) + # This is a non-existing path, so it should fail + # pyre-ignore + fake_path = os.path.join(self._tmpdir, uuid.uuid4().hex) + self.assertFalse(self._pathmgr.isdir(fake_path)) + + def test_ls(self) -> None: + # Create some files in the tempdir to ls out. + root_dir = os.path.join(self._tmpdir, "ls") # pyre-ignore + os.makedirs(root_dir, exist_ok=True) + files = sorted(["foo.txt", "bar.txt", "baz.txt"]) + for f in files: + open(os.path.join(root_dir, f), "a").close() + + children = sorted(self._pathmgr.ls(root_dir)) + self.assertListEqual(children, files) + + # Cleanup the tempdir + shutil.rmtree(root_dir) + + def test_mkdirs(self) -> None: + # pyre-ignore + new_dir_path = os.path.join(self._tmpdir, "new", "tmp", "dir") + self.assertFalse(self._pathmgr.exists(new_dir_path)) + self._pathmgr.mkdirs(new_dir_path) + self.assertTrue(self._pathmgr.exists(new_dir_path)) + + def test_copy(self) -> None: + _tmpfile_2 = self._tmpfile + "2" # pyre-ignore + _tmpfile_2_contents = "something else" + with open(_tmpfile_2, "w") as f: + f.write(_tmpfile_2_contents) + f.flush() + self.assertTrue(self._pathmgr.copy(self._tmpfile, _tmpfile_2, overwrite=True)) + with self._pathmgr.open(_tmpfile_2, "r") as f: + self.assertEqual(f.read(), self._tmpfile_contents) + + def test_move(self) -> None: + _tmpfile_2 = self._tmpfile + "2" + uuid.uuid4().hex # pyre-ignore + _tmpfile_3 = self._tmpfile + "3_" + uuid.uuid4().hex # pyre-ignore + _tmpfile_2_contents = "Hello Move" + with open(_tmpfile_2, "w") as f: + f.write(_tmpfile_2_contents) + f.flush() + # pyre-ignore + self.assertTrue(self._pathmgr.mv(_tmpfile_2, _tmpfile_3)) + with self._pathmgr.open(_tmpfile_3, "r") as f: + self.assertEqual(f.read(), _tmpfile_2_contents) + self.assertFalse(self._pathmgr.exists(_tmpfile_2)) + self._pathmgr.rm(_tmpfile_3) + + def test_symlink(self) -> None: + _symlink = self._tmpfile + "_symlink" # pyre-ignore + self.assertTrue(self._pathmgr.symlink(self._tmpfile, _symlink)) # pyre-ignore + with self._pathmgr.open(_symlink) as f: + self.assertEqual(f.read(), self._tmpfile_contents) + self.assertEqual(os.readlink(_symlink), self._tmpfile) + os.remove(_symlink) + + def test_rm(self) -> None: + # pyre-ignore + with open(os.path.join(self._tmpdir, "test_rm.txt"), "w") as f: + rm_file = f.name + f.write(self._tmpfile_contents) + f.flush() + self.assertTrue(self._pathmgr.exists(rm_file)) + self.assertTrue(self._pathmgr.isfile(rm_file)) + self._pathmgr.rm(rm_file) + self.assertFalse(self._pathmgr.exists(rm_file)) + self.assertFalse(self._pathmgr.isfile(rm_file)) + + def test_set_cwd(self) -> None: + # File not found since cwd not set yet. + self.assertFalse(self._pathmgr.isfile(self._filename)) + self.assertTrue(self._pathmgr.isfile(self._tmpfile)) + # Once cwd is set, relative file path works. + self._pathmgr.set_cwd(self._tmpdir) + self.assertTrue(self._pathmgr.isfile(self._filename)) + + # Set cwd to None + self._pathmgr.set_cwd(None) + self.assertFalse(self._pathmgr.isfile(self._filename)) + self.assertTrue(self._pathmgr.isfile(self._tmpfile)) + + # Set cwd to invalid path + with self.assertRaises(ValueError): + self._pathmgr.set_cwd("/nonexistent/path") + + def test_get_path_with_cwd(self) -> None: + self._pathmgr.set_cwd(self._tmpdir) + # Make sure _get_path_with_cwd() returns correctly. + self.assertEqual( + self._pathmgr._native_path_handler._get_path_with_cwd(self._filename), + self._tmpfile, + ) + self.assertEqual( + self._pathmgr._native_path_handler._get_path_with_cwd("/abs.txt"), + "/abs.txt", + ) + + def test_bad_args(self) -> None: + # TODO (T58240718): Replace with dynamic checks + with self.assertRaises(ValueError): + self._pathmgr.copy(self._tmpfile, self._tmpfile, foo="foo") # type: ignore + with self.assertRaises(ValueError): + self._pathmgr.exists(self._tmpfile, foo="foo") # type: ignore + with self.assertRaises(ValueError): + self._pathmgr.get_local_path(self._tmpfile, foo="foo") # type: ignore + with self.assertRaises(ValueError): + self._pathmgr.isdir(self._tmpfile, foo="foo") # type: ignore + with self.assertRaises(ValueError): + self._pathmgr.isfile(self._tmpfile, foo="foo") # type: ignore + with self.assertRaises(ValueError): + self._pathmgr.ls(self._tmpfile, foo="foo") # type: ignore + with self.assertRaises(ValueError): + self._pathmgr.mkdirs(self._tmpfile, foo="foo") # type: ignore + with self.assertRaises(ValueError): + self._pathmgr.open(self._tmpfile, foo="foo") # type: ignore + with self.assertRaises(ValueError): + self._pathmgr.opena(self._tmpfile, foo="foo") # type: ignore + with self.assertRaises(ValueError): + self._pathmgr.rm(self._tmpfile, foo="foo") # type: ignore + with self.assertRaises(ValueError): + self._pathmgr.set_cwd(self._tmpdir, foo="foo") # type: ignore + + self._pathmgr.set_strict_kwargs_checking(False) + + self._pathmgr.copy(self._tmpfile, self._tmpfile + "2", foo="foo") # type: ignore + self._pathmgr.exists(self._tmpfile, foo="foo") # type: ignore + self._pathmgr.get_local_path(self._tmpfile, foo="foo") # type: ignore + self._pathmgr.isdir(self._tmpfile, foo="foo") # type: ignore + self._pathmgr.isfile(self._tmpfile, foo="foo") # type: ignore + self._pathmgr.ls(self._tmpdir, foo="foo") # type: ignore + self._pathmgr.mkdirs(self._tmpdir, foo="foo") # type: ignore + f = self._pathmgr.open(self._tmpfile, foo="foo") # type: ignore + f.close() + # pyre-ignore + with open(os.path.join(self._tmpdir, "test_rm.txt"), "w") as f: + rm_file = f.name + f.write(self._tmpfile_contents) + f.flush() + self._pathmgr.rm(rm_file, foo="foo") # type: ignore + + +class TestLazyPath(unittest.TestCase): + _pathmgr = PathManagerBase() + + def test_materialize(self) -> None: + f = MagicMock(return_value="test") + x = LazyPath(f) + f.assert_not_called() + + p = os.fspath(x) + f.assert_called() + self.assertEqual(p, "test") + + p = os.fspath(x) + # should only be called once + f.assert_called_once() + self.assertEqual(p, "test") + + def test_join(self) -> None: + f = MagicMock(return_value="test") + x = LazyPath(f) + p = os.path.join(x, "a.txt") + f.assert_called_once() + self.assertEqual(p, "test/a.txt") + + def test_getattr(self) -> None: + x = LazyPath(lambda: "abc") + with self.assertRaises(AttributeError): + x.startswith("ab") + _ = os.fspath(x) + self.assertTrue(x.startswith("ab")) + + def test_PathManager(self) -> None: + x = LazyPath(lambda: "./") + output = self._pathmgr.ls(x) # pyre-ignore + output_gt = self._pathmgr.ls("./") + self.assertEqual(sorted(output), sorted(output_gt)) + + def test_getitem(self) -> None: + x = LazyPath(lambda: "abc") + with self.assertRaises(TypeError): + x[0] + _ = os.fspath(x) + self.assertEqual(x[0], "a") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_optim.py b/tests/test_optim.py new file mode 100644 index 0000000000000000000000000000000000000000..c269978097c951e06d99722fe79edf5865989814 --- /dev/null +++ b/tests/test_optim.py @@ -0,0 +1,62 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Unittests followed https://github.com/facebookresearch/detectron2/blob/main/tests/test_solver.py +""" + +import unittest + +from libai.optim.build import _expand_param_groups, reduce_param_groups + + +class TestOptimizer(unittest.TestCase): + def testExpandParamGroups(self): + params = [ + {"params": ["p1", "p2", "p3", "p4"], "lr": 1.0, "weight_decay": 3.0}, + {"params": ["p2", "p3", "p5"], "lr": 2.0, "momentum": 2.0}, + {"params": ["p1"], "weight_decay": 4.0}, + ] + out = _expand_param_groups(params) + gt = [ + dict(params=["p1"], lr=1.0, weight_decay=4.0), # noqa + dict(params=["p2"], lr=2.0, weight_decay=3.0, momentum=2.0), # noqa + dict(params=["p3"], lr=2.0, weight_decay=3.0, momentum=2.0), # noqa + dict(params=["p4"], lr=1.0, weight_decay=3.0), # noqa + dict(params=["p5"], lr=2.0, momentum=2.0), # noqa + ] + self.assertEqual(out, gt) + + def testReduceParamGroups(self): + params = [ + dict(params=["p1"], lr=1.0, weight_decay=4.0), # noqa + dict(params=["p2", "p6"], lr=2.0, weight_decay=3.0, momentum=2.0), # noqa + dict(params=["p3"], lr=2.0, weight_decay=3.0, momentum=2.0), # noqa + dict(params=["p4"], lr=1.0, weight_decay=3.0), # noqa + dict(params=["p5"], lr=2.0, momentum=2.0), # noqa + ] + gt_groups = [ + {"lr": 1.0, "weight_decay": 4.0, "params": ["p1"]}, + {"lr": 2.0, "weight_decay": 3.0, "momentum": 2.0, "params": ["p2", "p6", "p3"]}, + {"lr": 1.0, "weight_decay": 3.0, "params": ["p4"]}, + {"lr": 2.0, "momentum": 2.0, "params": ["p5"]}, + ] + out = reduce_param_groups(params) + self.assertTrue(out, gt_groups) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..dceae74599002d159e9f30849c329477f6f0dd85 --- /dev/null +++ b/tests/test_scheduler.py @@ -0,0 +1,199 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import unittest +from unittest import TestCase + +import numpy as np +import oneflow as flow +import oneflow.nn as nn + +from libai.scheduler import ( + WarmupCosineLR, + WarmupExponentialLR, + WarmupMultiStepLR, + WarmupPolynomialLR, + WarmupStepLR, +) + + +# @unittest.skip("Bugs in warmup scheduler") +class TestScheduler(TestCase): + def test_warmup_multistep(self): + p = nn.Parameter(flow.zeros(0)) + opt = flow.optim.SGD([p], lr=5.0) + + sched = WarmupMultiStepLR( + optimizer=opt, + max_iter=10, + milestones=[10, 15, 20], + gamma=0.1, + warmup_factor=0.001, + warmup_iter=5, + warmup_method="linear", + ) + + p.sum().backward() + opt.step() + + lrs = [0.005] + for _ in range(30): + sched.step() + lrs.append(opt.param_groups[0]["lr"]) + self.assertTrue(np.allclose(lrs[:5], [0.005, 1.004, 2.003, 3.002, 4.001])) + self.assertTrue(np.allclose(lrs[5:10], 5.0)) + self.assertTrue(np.allclose(lrs[10:15], 0.5)) + self.assertTrue(np.allclose(lrs[15:20], 0.05)) + self.assertTrue(np.allclose(lrs[20:], 0.005)) + + def test_warmup_step(self): + p = nn.Parameter(flow.zeros(0)) + opt = flow.optim.SGD([p], lr=5.0) + + sched = WarmupStepLR( + optimizer=opt, + max_iter=10, + step_size=10, + gamma=0.1, + warmup_factor=0.001, + warmup_iter=5, + warmup_method="linear", + ) + + p.sum().backward() + opt.step() + + lrs = [0.005] + for _ in range(30): + sched.step() + lrs.append(opt.param_groups[0]["lr"]) + self.assertTrue(np.allclose(lrs[:5], [0.005, 1.004, 2.003, 3.002, 4.001])) + self.assertTrue(np.allclose(lrs[5:10], 5.0)) + self.assertTrue(np.allclose(lrs[10:20], 0.5)) + self.assertTrue(np.allclose(lrs[20:30], 0.05)) + self.assertTrue(np.allclose(lrs[30:], 0.005)) + + def test_warmup_cosine(self): + p = nn.Parameter(flow.zeros(0)) + opt = flow.optim.SGD([p], lr=5.0) + + sched = WarmupCosineLR( + optimizer=opt, + max_iter=30, + warmup_factor=0.001, + warmup_iter=5, + warmup_method="linear", + ) + + p.sum().backward() + opt.step() + self.assertEqual(opt.param_groups[0]["lr"], 0.005) + lrs = [0.005] + + for _ in range(30): + sched.step() + lrs.append(opt.param_groups[0]["lr"]) + for idx, lr in enumerate(lrs): + expected_cosine = 2.5 * (1.0 + math.cos(math.pi * idx / 30)) + if idx >= 5: + self.assertAlmostEqual(lr, expected_cosine) + else: + self.assertNotAlmostEqual(lr, expected_cosine) + + def test_warmup_exponential(self): + p = nn.Parameter(flow.zeros(0)) + opt = flow.optim.SGD([p], lr=5.0) + + sched = WarmupExponentialLR( + optimizer=opt, + max_iter=10, + gamma=0.1, + warmup_factor=0.001, + warmup_iter=5, + warmup_method="linear", + ) + + p.sum().backward() + opt.step() + self.assertEqual(opt.param_groups[0]["lr"], 0.005) + lrs = [0.005] + + def _get_exponential_lr(base_lr, gamma, max_iters, warmup_iters): + valid_values = [] + for idx in range(warmup_iters, max_iters + 1): + valid_values.append(base_lr * (gamma ** idx)) + return valid_values + + for _ in range(30): + sched.step() + lrs.append(opt.param_groups[0]["lr"]) + self.assertTrue( + np.allclose( + lrs[:5], [0.005, 0.00401, 0.0030199999999999997, 0.00203, 0.0010399999999999997] + ) + ) + valid_intermediate_values = _get_exponential_lr( + base_lr=5.0, gamma=0.1, max_iters=30, warmup_iters=5 + ) + self.assertEqual(lrs[5:], valid_intermediate_values) + + def test_warmup_polynomial(self): + p = nn.Parameter(flow.zeros(0)) + opt = flow.optim.SGD([p], lr=5.0) + + sched = WarmupPolynomialLR( + optimizer=opt, + max_iter=30, + warmup_factor=0.001, + warmup_iter=0, + end_learning_rate=1e-4, + power=1.0, + cycle=False, + warmup_method="linear", + ) + + # self.assertEqual(opt.param_groups[0]["lr"], 0.005) + # lrs = [0.005] + lrs = [5.0] # lr_scheduler first invoke result + + def _get_polynomial_lr( + base_lr, max_iters, warmup_iters, end_lr=1e-4, power=1.0, cycle=False + ): + valid_values = [] + decay_steps = max_iters - warmup_iters + for step in range(max_iters - warmup_iters): + if cycle: + if step == 0: + step = 1 + decay_steps = decay_steps * math.ceil(step / decay_steps) + else: + step = min(step, decay_steps) + valid_values.append( + (base_lr - end_lr) * ((1 - step / decay_steps) ** power) + end_lr + ) + return valid_values + + for _ in range(29): + sched.step() # only invoke (max_iter-1), because the first invoke is done when init + lrs.append(opt.param_groups[0]["lr"]) + # self.assertTrue(np.allclose(lrs[:5], [0.005, 1.004, 2.003, 3.002, 4.001])) + valid_intermediate_values = _get_polynomial_lr(base_lr=5.0, max_iters=30, warmup_iters=0) + # self.assertEqual(lrs[5:30], valid_intermediate_values) + self.assertEqual(lrs, valid_intermediate_values) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_trainer.py b/tests/test_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..bb04f4df50afc0d7b3f1b6ce75b5177d83f6861c --- /dev/null +++ b/tests/test_trainer.py @@ -0,0 +1,142 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +import oneflow as flow +from omegaconf import OmegaConf +from oneflow.utils.data import DataLoader, TensorDataset + +sys.path.append(".") +from libai.config import LazyCall, default_argument_parser +from libai.engine import DefaultTrainer, default_setup +from libai.optim import get_default_optimizer_params +from libai.scheduler import WarmupMultiStepLR +from tests.layers.test_trainer_model import build_graph, build_model + + +def setup(args): + """ + Create configs and perform basic setups. + """ + + cfg = OmegaConf.create() + + cfg.train = dict( + output_dir="./demo_output", + train_micro_batch_size=32, + test_micro_batch_size=32, + dist=dict( + data_parallel_size=1, + tensor_parallel_size=1, + pipeline_parallel_size=1, + pipeline_num_layers=4, + ), + start_iter=0, + train_iter=20, + train_epoch=1, + warmup_ratio=0.05, + lr_warmup_fraction=0.01, + lr_decay_iter=6000, + eval_period=1000, + log_period=1, + checkpointer=dict(period=100), + nccl_fusion_threshold_mb=16, + nccl_fusion_max_ops=24, + scheduler=LazyCall(WarmupMultiStepLR)( + warmup_factor=0.001, + # alpha=0.01, + warmup_method="linear", + milestones=[0.1, 0.2], + ), + ) + + cfg.optim = LazyCall(flow.optim.AdamW)( + parameters=LazyCall(get_default_optimizer_params)( + # parameters.model is meant to be set to the model object, before + # instantiating the optimizer. + clip_grad_max_norm=1.0, + clip_grad_norm_type=2.0, + weight_decay_norm=0.0, + weight_decay_bias=0.0, + ), + lr=1e-4, + weight_decay=0.01, + betas=(0.9, 0.999), + do_bias_correction=True, + ) + + cfg.graph = dict( + enabled=True, + ) + + default_setup(cfg, args) + return cfg + + +class DemoTrainer(DefaultTrainer): + @classmethod + def build_model(cls, cfg): + """ + Returns: + flow.nn.Module: + It now calls :func:`libai.layers.build_model`. + Overwrite it if you'd like a different model. + """ + model = build_model(cfg) + return model + + @classmethod + def build_graph(cls, cfg, model, optimizer=None, lr_scheduler=None, is_train=True): + return build_graph(cfg, model, optimizer, lr_scheduler) + + @classmethod + def get_batch(cls, data): + return [ + flow.randn( + 32, + 512, + sbp=flow.sbp.split(0), + placement=flow.placement("cuda", [0]), + ) + ] + + @classmethod + def build_train_loader(cls, cfg, tokenizer=None): + return ( + DataLoader( + TensorDataset(flow.randn(1000)), batch_size=cfg.train.train_micro_batch_size + ), + None, + None, + ) + + @classmethod + def build_test_loader(cls, cfg): + return [] + + +def main(args): + cfg = setup(args) + + trainer = DemoTrainer(cfg) + + # trainer.resume_or_load(resume=args.resume) + return trainer.train() + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + main(args) diff --git a/tests/tokenizer/__init__.py b/tests/tokenizer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/tokenizer/test_tokenization_bert.py b/tests/tokenizer/test_tokenization_bert.py new file mode 100644 index 0000000000000000000000000000000000000000..ea29a5f036e58b6e90e7b1f228628edaf6699dd3 --- /dev/null +++ b/tests/tokenizer/test_tokenization_bert.py @@ -0,0 +1,157 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import unittest + +from libai.tokenizer.tokenization_base import _is_control, _is_punctuation, _is_whitespace +from libai.tokenizer.tokenization_bert import ( + VOCAB_FILES_NAMES, + BasicTokenizer, + BertTokenizer, + WordpieceTokenizer, +) +from tests.tokenizer.test_tokenization_common import TokenizerTesterMixin + + +class BertTokenizationTest(TokenizerTesterMixin, unittest.TestCase): + tokenizer_class = BertTokenizer + + def setUp(self): + super().setUp() + + vocab_tokens = [ + "[UNK]", + "[CLS]", + "[SEP]", + "[PAD]", + "[MASK]", + "want", + "##want", + "##ed", + "wa", + "un", + "runn", + "##ing", + ",", + "low", + "lowest", + ] + self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"]) + with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer: + vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) + + def get_input_output_texts(self, tokenizer): + input_text = "UNwant\u00E9d,running" + output_text = "unwanted, running" + return input_text, output_text + + def test_full_tokenizer(self): + tokenizer = self.tokenizer_class(self.vocab_file) + + tokens = tokenizer.tokenize("UNwant\u00E9d,running") + self.assertListEqual(tokens, ["un", "##want", "##ed", ",", "runn", "##ing"]) + self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), [9, 6, 7, 12, 10, 11]) + + def test_chinese(self): + tokenizer = BasicTokenizer() + + self.assertListEqual( + tokenizer.tokenize("ah\u535A\u63A8zz"), ["ah", "\u535A", "\u63A8", "zz"] + ) + + def test_basic_tokenizer_lower(self): + tokenizer = BasicTokenizer(do_lower_case=True) + + self.assertListEqual( + tokenizer.tokenize(" \tHeLLo!how \n Are yoU? "), + ["hello", "!", "how", "are", "you", "?"], + ) + self.assertListEqual(tokenizer.tokenize("H\u00E9llo"), ["hello"]) + + def test_basic_tokenizer_no_lower(self): + tokenizer = BasicTokenizer(do_lower_case=False) + + self.assertListEqual( + tokenizer.tokenize(" \tHeLLo!how \n Are yoU? "), + ["HeLLo", "!", "how", "Are", "yoU", "?"], + ) + + def test_basic_tokenizer_respects_never_split_tokens(self): + tokenizer = BasicTokenizer(do_lower_case=False, never_split=["[UNK]"]) + + self.assertListEqual( + tokenizer.tokenize(" \tHeLLo!how \n Are yoU? [UNK]"), + ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"], + ) + + def test_wordpiece_tokenizer(self): + vocab_tokens = [ + "[UNK]", + "[CLS]", + "[SEP]", + "want", + "##want", + "##ed", + "wa", + "un", + "runn", + "##ing", + ] + + vocab = {} + for (i, token) in enumerate(vocab_tokens): + vocab[token] = i + tokenizer = WordpieceTokenizer(vocab=vocab, unk_token="[UNK]") + + self.assertListEqual(tokenizer.tokenize(""), []) + + self.assertListEqual( + tokenizer.tokenize("unwanted running"), ["un", "##want", "##ed", "runn", "##ing"] + ) + + self.assertListEqual(tokenizer.tokenize("unwantedX running"), ["[UNK]", "runn", "##ing"]) + + def test_is_whitespace(self): + self.assertTrue(_is_whitespace(" ")) + self.assertTrue(_is_whitespace("\t")) + self.assertTrue(_is_whitespace("\r")) + self.assertTrue(_is_whitespace("\n")) + self.assertTrue(_is_whitespace("\u00A0")) + + self.assertFalse(_is_whitespace("A")) + self.assertFalse(_is_whitespace("-")) + + def test_is_control(self): + self.assertTrue(_is_control("\u0005")) + + self.assertFalse(_is_control("A")) + self.assertFalse(_is_control(" ")) + self.assertFalse(_is_control("\t")) + self.assertFalse(_is_control("\r")) + + def test_is_punctuation(self): + self.assertTrue(_is_punctuation("-")) + self.assertTrue(_is_punctuation("$")) + self.assertTrue(_is_punctuation("`")) + self.assertTrue(_is_punctuation(".")) + + self.assertFalse(_is_punctuation("A")) + self.assertFalse(_is_punctuation(" ")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/tokenizer/test_tokenization_common.py b/tests/tokenizer/test_tokenization_common.py new file mode 100644 index 0000000000000000000000000000000000000000..ac839c669e1b340a77b12050f3ab2b89f52c2340 --- /dev/null +++ b/tests/tokenizer/test_tokenization_common.py @@ -0,0 +1,400 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import inspect +import os +import pickle +import re +import shutil +import tempfile +from typing import Tuple + +from libai.tokenizer import PreTrainedTokenizer +from tests.fixtures.utils import get_fixtures + + +def get_tests_dir(append_path=None): + """ + Args: + append_path: optional path to append to the tests dir path + + Return: + The full path to the `tests` dir, so that the tests can be invoked from anywhere. + Optionally `append_path` is joined after the `tests` dir the former is provided. + + """ + # this function caller's __file__ + caller__file__ = inspect.stack()[1][1] + tests_dir = os.path.abspath(os.path.dirname(caller__file__)) + if append_path: + return os.path.join(tests_dir, append_path) + else: + return tests_dir + + +class TokenizerTesterMixin: + + tokenizer_class = None + + def setUp(self): + self.tokenizers_list = [] + get_fixtures("sample_text.txt") + with open(f"{get_tests_dir()}/../fixtures/sample_text.txt", encoding="utf-8") as f_data: + self._data = f_data.read().replace("\n\n", "\n").strip() + + self.tmpdirname = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tmpdirname) + + def get_input_output_texts(self, tokenizer): + input_txt = self.get_clean_sequence(tokenizer)[0] + return input_txt, input_txt + + def get_clean_sequence( + self, tokenizer, with_prefix_space=False, max_length=20 + ) -> Tuple[str, list]: + toks = [ + (i, tokenizer.decode([i], clean_up_tokenization_spaces=False)) + for i in range(len(tokenizer)) + ] + toks = list(filter(lambda t: re.match(r"^[ a-zA-Z]+$", t[1]), toks)) + toks = list(filter(lambda t: [t[0]] == tokenizer.encode(t[1]), toks)) + if max_length is not None and len(toks) > max_length: + toks = toks[:max_length] + toks_ids = [t[0] for t in toks] + + # Ensure consistency + output_txt = tokenizer.decode(toks_ids, clean_up_tokenization_spaces=False) + if " " not in output_txt and len(toks_ids) > 1: + output_txt = ( + tokenizer.decode([toks_ids[0]], clean_up_tokenization_spaces=False) + + " " + + tokenizer.decode(toks_ids[1:], clean_up_tokenization_spaces=False) + ) + if with_prefix_space: + output_txt = " " + output_txt + output_ids = tokenizer.encode(output_txt) + return output_txt, output_ids + + def get_tokenizers(self, **kwargs): + return [self.get_tokenizer(**kwargs)] + + def get_tokenizer(self, **kwargs) -> PreTrainedTokenizer: + return self.tokenizer_class.from_pretrained(self.tmpdirname, **kwargs) + + def test_tokenizers_common_properties(self): + tokenizers = self.get_tokenizers() + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + attributes_list = [ + "bos_token", + "eos_token", + "unk_token", + "sep_token", + "pad_token", + "cls_token", + "mask_token", + ] + for attr in attributes_list: + self.assertTrue(hasattr(tokenizer, attr)) + self.assertTrue(hasattr(tokenizer, attr + "_id")) + + self.assertTrue(hasattr(tokenizer, "additional_special_tokens")) + self.assertTrue(hasattr(tokenizer, "additional_special_tokens_ids")) + + attributes_list = [ + "init_inputs", + "init_kwargs", + "added_tokens_encoder", + "added_tokens_decoder", + ] + for attr in attributes_list: + self.assertTrue(hasattr(tokenizer, attr)) + + def test_save_and_load_tokenizer(self): + # Now let's start the test + tokenizers = self.get_tokenizers() + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + # Isolate this from the other tests because we save additional tokens/etc + tmpdirname = tempfile.mkdtemp() + + sample_text = " He is very happy, UNwant\u00E9d,running" + before_tokens = tokenizer.encode(sample_text) + before_vocab = tokenizer.get_vocab() + tokenizer.save_pretrained(tmpdirname) + + after_tokenizer = tokenizer.__class__.from_pretrained(tmpdirname) + after_tokens = after_tokenizer.encode(sample_text) + after_vocab = after_tokenizer.get_vocab() + self.assertListEqual(before_tokens, after_tokens) + self.assertDictEqual(before_vocab, after_vocab) + + shutil.rmtree(tmpdirname) + + # Now let's start the test + tokenizers = self.get_tokenizers() + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + # Isolate this from the other tests because we save additional tokens/etc + tmpdirname = tempfile.mkdtemp() + + sample_text = " He is very happy, UNwant\u00E9d,running" + tokenizer.add_tokens(["bim", "bambam"]) + additional_special_tokens = tokenizer.additional_special_tokens + additional_special_tokens.append("new_additional_special_token") + tokenizer.add_special_tokens( + {"additional_special_tokens": additional_special_tokens} + ) + before_tokens = tokenizer.encode(sample_text) + before_vocab = tokenizer.get_vocab() + tokenizer.save_pretrained(tmpdirname) + + after_tokenizer = tokenizer.__class__.from_pretrained(tmpdirname) + after_tokens = after_tokenizer.encode(sample_text) + after_vocab = after_tokenizer.get_vocab() + self.assertListEqual(before_tokens, after_tokens) + self.assertDictEqual(before_vocab, after_vocab) + self.assertIn("bim", after_vocab) + self.assertIn("bambam", after_vocab) + self.assertIn( + "new_additional_special_token", after_tokenizer.additional_special_tokens + ) + + shutil.rmtree(tmpdirname) + + def test_pickle_tokenizer(self): + """Google pickle __getstate__ __setstate__ if you are struggling with this.""" + tokenizers = self.get_tokenizers() + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + self.assertIsNotNone(tokenizer) + + text = "Munich and Berlin are nice cities" + subwords = tokenizer.tokenize(text) + + filename = os.path.join(self.tmpdirname, "tokenizer.bin") + with open(filename, "wb") as handle: + pickle.dump(tokenizer, handle) + + with open(filename, "rb") as handle: + tokenizer_new = pickle.load(handle) + + subwords_loaded = tokenizer_new.tokenize(text) + + self.assertListEqual(subwords, subwords_loaded) + + def test_added_tokens_do_lower_case(self): + tokenizers = self.get_tokenizers(do_lower_case=True) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + if not hasattr(tokenizer, "do_lower_case") or not tokenizer.do_lower_case: + continue + + special_token = tokenizer.all_special_tokens[0] + + text = special_token + " aaaaa bbbbbb low cccccccccdddddddd l " + special_token + text2 = special_token + " AAAAA BBBBBB low CCCCCCCCCDDDDDDDD l " + special_token + + toks0 = tokenizer.tokenize(text) # toks before adding new_toks + + new_toks = [ + "aaaaa bbbbbb", + "cccccccccdddddddd", + "AAAAA BBBBBB", + "CCCCCCCCCDDDDDDDD", + ] + added = tokenizer.add_tokens(new_toks) + self.assertEqual(added, 2) + + toks = tokenizer.tokenize(text) + toks2 = tokenizer.tokenize(text2) + + self.assertEqual(len(toks), len(toks2)) + self.assertListEqual(toks, toks2) + self.assertNotEqual(len(toks), len(toks0)) # toks0 should be longer + + # Check that none of the special tokens are lowercased + sequence_with_special_tokens = ( + "A " + " yEs ".join(tokenizer.all_special_tokens) + " B" + ) + tokenized_sequence = tokenizer.tokenize(sequence_with_special_tokens) + + for special_token in tokenizer.all_special_tokens: + self.assertTrue(special_token in tokenized_sequence) + + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + special_token = tokenizer.all_special_tokens[0] + + text = special_token + " aaaaa bbbbbb low cccccccccdddddddd l " + special_token + text2 = special_token + " AAAAA BBBBBB low CCCCCCCCCDDDDDDDD l " + special_token + + new_toks = [ + "aaaaa bbbbbb", + "cccccccccdddddddd", + "AAAAA BBBBBB", + "CCCCCCCCCDDDDDDDD", + ] + + toks0 = tokenizer.tokenize(text) # toks before adding new_toks + + added = tokenizer.add_tokens(new_toks) + self.assertEqual(added, 4) + + toks = tokenizer.tokenize(text) + toks2 = tokenizer.tokenize(text2) + + self.assertEqual(len(toks), len(toks2)) # Length should still be the same + self.assertNotEqual( + toks[1], toks2[1] + ) # But at least the first non-special tokens should differ + + self.assertNotEqual(len(toks), len(toks0)) # toks0 should be longer + + def test_add_tokens_tokenizer(self): + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + vocab_size = tokenizer.vocab_size + all_size = len(tokenizer) + + self.assertNotEqual(vocab_size, 0) + + # We usually have added tokens from the start in tests + # because our vocab fixtures are smaller than the original vocabs + # let's not assert this self.assertEqual(vocab_size, all_size) + + new_toks = ["aaaaa bbbbbb", "cccccccccdddddddd"] + added_toks = tokenizer.add_tokens(new_toks) + vocab_size_2 = tokenizer.vocab_size + all_size_2 = len(tokenizer) + + self.assertNotEqual(vocab_size_2, 0) + self.assertEqual(vocab_size, vocab_size_2) + self.assertEqual(added_toks, len(new_toks)) + self.assertEqual(all_size_2, all_size + len(new_toks)) + + tokens = tokenizer.encode("aaaaa bbbbbb low cccccccccdddddddd l") + + self.assertGreaterEqual(len(tokens), 4) + self.assertGreater(tokens[0], tokenizer.vocab_size - 1) + self.assertGreater(tokens[-2], tokenizer.vocab_size - 1) + + new_toks_2 = {"eos_token": ">>>>|||<||<<|<<", "pad_token": "<<<<<|||>|>>>>|>"} + added_toks_2 = tokenizer.add_special_tokens(new_toks_2) + vocab_size_3 = tokenizer.vocab_size + all_size_3 = len(tokenizer) + + self.assertNotEqual(vocab_size_3, 0) + self.assertEqual(vocab_size, vocab_size_3) + self.assertEqual(added_toks_2, len(new_toks_2)) + self.assertEqual(all_size_3, all_size_2 + len(new_toks_2)) + + tokens = tokenizer.encode( + ">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l" + ) + + self.assertGreaterEqual(len(tokens), 6) + self.assertGreater(tokens[0], tokenizer.vocab_size - 1) + self.assertGreater(tokens[0], tokens[1]) + self.assertGreater(tokens[-2], tokenizer.vocab_size - 1) + self.assertGreater(tokens[-2], tokens[-3]) + self.assertEqual(tokens[0], tokenizer.eos_token_id) + self.assertEqual(tokens[-2], tokenizer.pad_token_id) + + def test_add_special_tokens(self): + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + input_text, ids = self.get_clean_sequence(tokenizer) + + special_token = "[SPECIAL_TOKEN]" + + tokenizer.add_special_tokens({"cls_token": special_token}) + encoded_special_token = tokenizer.encode(special_token) + self.assertEqual(len(encoded_special_token), 1) + + text = tokenizer.decode( + ids + encoded_special_token, clean_up_tokenization_spaces=False + ) + encoded = tokenizer.encode(text) + + input_encoded = tokenizer.encode(input_text) + special_token_id = tokenizer.encode(special_token) + self.assertEqual(encoded, input_encoded + special_token_id) + + decoded = tokenizer.decode(encoded, skip_special_tokens=True) + self.assertTrue(special_token not in decoded) + + def test_internal_consistency(self): + tokenizers = self.get_tokenizers() + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + input_text, output_text = self.get_input_output_texts(tokenizer) + + tokens = tokenizer.tokenize(input_text) + ids = tokenizer.convert_tokens_to_ids(tokens) + ids_2 = tokenizer.encode(input_text) + self.assertListEqual(ids, ids_2) + + tokens_2 = tokenizer.convert_ids_to_tokens(ids) + self.assertNotEqual(len(tokens_2), 0) + text_2 = tokenizer.decode(ids) + self.assertIsInstance(text_2, str) + + self.assertEqual(text_2, output_text) + + def test_encode_decode_with_spaces(self): + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + + new_toks = ["[ABC]", "[DEF]"] + tokenizer.add_tokens(new_toks) + input = "[ABC] [DEF] [ABC] [DEF]" + encoded = tokenizer.encode(input) + decoded = tokenizer.decode(encoded) + self.assertEqual(decoded, input) + + def test_pretrained_model_lists(self): + weights_list = list(self.tokenizer_class.max_model_input_sizes.keys()) + weights_lists_2 = [] + for file_id, map_list in self.tokenizer_class.pretrained_vocab_files_map.items(): + weights_lists_2.append(list(map_list.keys())) + + for weights_list_2 in weights_lists_2: + self.assertListEqual(weights_list, weights_list_2) + + def test_get_vocab(self): + tokenizers = self.get_tokenizers(do_lower_case=False) + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + vocab = tokenizer.get_vocab() + + self.assertIsInstance(vocab, dict) + self.assertEqual(len(vocab), len(tokenizer)) + + for word, ind in vocab.items(): + self.assertEqual(tokenizer.convert_tokens_to_ids(word), ind) + self.assertEqual(tokenizer.convert_ids_to_tokens(ind), word) + + tokenizer.add_tokens(["asdfasdfasdfasdf"]) + vocab = tokenizer.get_vocab() + self.assertIsInstance(vocab, dict) + self.assertEqual(len(vocab), len(tokenizer)) diff --git a/tests/tokenizer/test_tokenization_gpt2.py b/tests/tokenizer/test_tokenization_gpt2.py new file mode 100644 index 0000000000000000000000000000000000000000..acd11d248e6970fd0ad90e4488e27955c8bdb394 --- /dev/null +++ b/tests/tokenizer/test_tokenization_gpt2.py @@ -0,0 +1,88 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json +import os +import unittest + +from libai.tokenizer.tokenization_gpt2 import VOCAB_FILES_NAMES, GPT2Tokenizer +from tests.tokenizer.test_tokenization_common import TokenizerTesterMixin + + +class GPT2TokenizationTest(TokenizerTesterMixin, unittest.TestCase): + tokenizer_class = GPT2Tokenizer + + def setUp(self): + super().setUp() + + # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt + vocab = [ + "l", + "o", + "w", + "e", + "r", + "s", + "t", + "i", + "d", + "n", + "\u0120", + "\u0120l", + "\u0120n", + "\u0120lo", + "\u0120low", + "er", + "\u0120lowest", + "\u0120newer", + "\u0120wider", + "", + "<|endoftext|>", + ] + vocab_tokens = dict(zip(vocab, range(len(vocab)))) + merges = ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""] + self.special_tokens_map = {"unk_token": ""} + + self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"]) + self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["merges_file"]) + with open(self.vocab_file, "w", encoding="utf-8") as fp: + fp.write(json.dumps(vocab_tokens) + "\n") + with open(self.merges_file, "w", encoding="utf-8") as fp: + fp.write("\n".join(merges)) + + def get_tokenizer(self, **kwargs): + kwargs.update(self.special_tokens_map) + return GPT2Tokenizer.from_pretrained(self.tmpdirname, **kwargs) + + def get_input_output_texts(self, tokenizer): + input_text = "lower newer" + output_text = "lower newer" + return input_text, output_text + + def test_full_tokenizer(self): + tokenizer = GPT2Tokenizer(self.vocab_file, self.merges_file, **self.special_tokens_map) + text = " lower newer" + bpe_tokens = ["\u0120low", "er", "\u0120", "n", "e", "w", "er"] + tokens = tokenizer.tokenize(text) + self.assertListEqual(tokens, bpe_tokens) + + input_tokens = tokens + [tokenizer.unk_token] + input_bpe_tokens = [14, 15, 10, 9, 3, 2, 15, 19] + self.assertListEqual(tokenizer.convert_tokens_to_ids(input_tokens), input_bpe_tokens) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/tokenizer/test_tokenization_roberta.py b/tests/tokenizer/test_tokenization_roberta.py new file mode 100644 index 0000000000000000000000000000000000000000..f6a025253740b32f0621dfee1a80a56052c45d5a --- /dev/null +++ b/tests/tokenizer/test_tokenization_roberta.py @@ -0,0 +1,97 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import unittest + +from libai.tokenizer.tokenization_roberta import VOCAB_FILES_NAMES, RobertaTokenizer +from tests.tokenizer.test_tokenization_common import TokenizerTesterMixin + + +class RobertaTokenizationTest(TokenizerTesterMixin, unittest.TestCase): + tokenizer_class = RobertaTokenizer + + def setUp(self): + super().setUp() + + # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt + vocab = [ + "l", + "o", + "w", + "e", + "r", + "s", + "t", + "i", + "d", + "n", + "\u0120", + "\u0120l", + "\u0120n", + "\u0120lo", + "\u0120low", + "er", + "\u0120lowest", + "\u0120newer", + "\u0120wider", + "", + ] + vocab_tokens = dict(zip(vocab, range(len(vocab)))) + merges = ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""] + self.special_tokens_map = {"unk_token": ""} + + self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"]) + self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["merges_file"]) + with open(self.vocab_file, "w", encoding="utf-8") as fp: + fp.write(json.dumps(vocab_tokens) + "\n") + with open(self.merges_file, "w", encoding="utf-8") as fp: + fp.write("\n".join(merges)) + + def get_tokenizer(self, **kwargs): + kwargs.update(self.special_tokens_map) + return RobertaTokenizer.from_pretrained(self.tmpdirname, **kwargs) + + def get_input_output_texts(self, tokenizer): + input_text = "lower newer" + output_text = "lower newer" + return input_text, output_text + + def test_full_tokenizer(self): + tokenizer = RobertaTokenizer(self.vocab_file, self.merges_file, **self.special_tokens_map) + text = "lower newer" + bpe_tokens = ["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"] + tokens = tokenizer.tokenize(text) + self.assertListEqual(tokens, bpe_tokens) + + input_tokens = tokens + [tokenizer.unk_token] + input_bpe_tokens = [0, 1, 2, 15, 10, 9, 3, 2, 15, 19] + self.assertListEqual(tokenizer.convert_tokens_to_ids(input_tokens), input_bpe_tokens) + + def roberta_dict_integration_testing(self): + tokenizer = self.get_tokenizer() + + self.assertListEqual( + tokenizer.encode("Hello world!", add_special_tokens=False), [0, 31414, 232, 328, 2] + ) + self.assertListEqual( + tokenizer.encode("Hello world! cécé herlolip 418", add_special_tokens=False), + [0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/tokenizer/test_tokenization_t5.py b/tests/tokenizer/test_tokenization_t5.py new file mode 100644 index 0000000000000000000000000000000000000000..7d22eca561c8d66c31aeeed128e4385c3c8e28ca --- /dev/null +++ b/tests/tokenizer/test_tokenization_t5.py @@ -0,0 +1,155 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import shutil +import tempfile +import unittest + +from libai.tokenizer.tokenization_t5 import T5Tokenizer +from tests.fixtures.utils import get_fixtures +from tests.tokenizer.test_tokenization_common import TokenizerTesterMixin + +SPIECE_UNDERLINE = "▁" + +SAMPLE_VOCAB = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../fixtures/test_sentencepiece.model" +) + + +class T5TokenizationTest(TokenizerTesterMixin, unittest.TestCase): + + tokenizer_class = T5Tokenizer + + def setUp(self): + super().setUp() + + # We have a SentencePiece fixture for testing + get_fixtures(SAMPLE_VOCAB) + tokenizer = T5Tokenizer(SAMPLE_VOCAB) + tokenizer.save_pretrained(self.tmpdirname) + + def test_convert_token_and_id(self): + """Test ``_convert_token_to_id`` and ``_convert_id_to_token``.""" + token = "" + token_id = 1 + + self.assertEqual(self.get_tokenizer()._convert_token_to_id(token), token_id) + self.assertEqual(self.get_tokenizer()._convert_id_to_token(token_id), token) + + def test_get_vocab(self): + vocab_keys = list(self.get_tokenizer().get_vocab().keys()) + + self.assertEqual(vocab_keys[0], "") + self.assertEqual(vocab_keys[1], "") + self.assertEqual(vocab_keys[-1], "") + self.assertEqual(len(vocab_keys), 1_101) + + def test_vocab_size(self): + self.assertEqual(self.get_tokenizer().vocab_size, 1_100) + + def test_full_tokenizer(self): + tokenizer = T5Tokenizer(SAMPLE_VOCAB) + + tokens = tokenizer.tokenize("This is a test") + self.assertListEqual(tokens, ["▁This", "▁is", "▁a", "▁t", "est"]) + + self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), [285, 46, 10, 170, 382]) + + tokens = tokenizer.tokenize("I was born in 92000, and this is falsé.") + self.assertListEqual( + tokens, + [ + SPIECE_UNDERLINE + "I", + SPIECE_UNDERLINE + "was", + SPIECE_UNDERLINE + "b", + "or", + "n", + SPIECE_UNDERLINE + "in", + SPIECE_UNDERLINE + "", + "9", + "2", + "0", + "0", + "0", + ",", + SPIECE_UNDERLINE + "and", + SPIECE_UNDERLINE + "this", + SPIECE_UNDERLINE + "is", + SPIECE_UNDERLINE + "f", + "al", + "s", + "é", + ".", + ], + ) + ids = tokenizer.convert_tokens_to_ids(tokens) + self.assertListEqual( + ids, [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4] + ) + + back_tokens = tokenizer.convert_ids_to_tokens(ids) + self.assertListEqual( + back_tokens, + [ + SPIECE_UNDERLINE + "I", + SPIECE_UNDERLINE + "was", + SPIECE_UNDERLINE + "b", + "or", + "n", + SPIECE_UNDERLINE + "in", + SPIECE_UNDERLINE + "", + "", + "2", + "0", + "0", + "0", + ",", + SPIECE_UNDERLINE + "and", + SPIECE_UNDERLINE + "this", + SPIECE_UNDERLINE + "is", + SPIECE_UNDERLINE + "f", + "al", + "s", + "", + ".", + ], + ) + + def test_save_and_load_tokenizer(self): + # Now let's start the test + tokenizers = self.get_tokenizers() + for tokenizer in tokenizers: + with self.subTest(f"{tokenizer.__class__.__name__}"): + # Isolate this from the other tests because we save additional tokens/etc + tmpdirname = tempfile.mkdtemp() + + sample_text = " He is very happy, UNwant\u00E9d,running" + before_tokens = tokenizer.encode(sample_text) + before_vocab = tokenizer.get_vocab() + tokenizer.save_pretrained(tmpdirname) + + after_tokenizer = tokenizer.__class__.from_pretrained(tmpdirname) + after_tokens = after_tokenizer.encode(sample_text) + after_vocab = after_tokenizer.get_vocab() + self.assertListEqual(before_tokens, after_tokens) + self.assertDictEqual(before_vocab, after_vocab) + + shutil.rmtree(tmpdirname) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/tools/test_preprocess.sh b/tests/tools/test_preprocess.sh new file mode 100644 index 0000000000000000000000000000000000000000..feeeb44349936975debe771b234f77d69bea2ebe --- /dev/null +++ b/tests/tools/test_preprocess.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +IMPL=lazy +KEYS=text + +python tools/preprocess_data.py \ + --input path/to/test_sample_cn.json \ + --json-keys ${KEYS} \ + --vocab-file path/to/bert-base-chinese-vocab.txt \ + --dataset-impl ${IMPL} \ + --tokenizer-name BertTokenizer \ + --do-lower-case \ + --do-chinese-wwm \ + --split-sentences \ + --output-prefix cn_samples_${IMPL} \ + --workers 1 \ + --log-interval 2 \ No newline at end of file diff --git a/tests/tools/test_trainer.sh b/tests/tools/test_trainer.sh new file mode 100644 index 0000000000000000000000000000000000000000..8c53df79ab15b681be96543800a69a2d58ab6edf --- /dev/null +++ b/tests/tools/test_trainer.sh @@ -0,0 +1,20 @@ +#! /bin/bash + +source $1 + +_DEVICE_NUM_PER_NODE=2 +_MASTER_ADDR=127.0.0.1 +_NUM_NODES=1 +_NODE_RANK=0 + +run_cmd="python3 -m oneflow.distributed.launch \ + --nproc_per_node $_DEVICE_NUM_PER_NODE \ + --nnodes $_NUM_NODES \ + --node_rank $_NODE_RANK \ + --master_addr $_MASTER_ADDR \ + tests/test_trainer.py" + +echo ${run_cmd} +eval ${run_cmd} + +set +x \ No newline at end of file diff --git a/tools/infer.sh b/tools/infer.sh new file mode 100644 index 0000000000000000000000000000000000000000..1c209a5414a0205e33ca2adc002e1eccd1875612 --- /dev/null +++ b/tools/infer.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +FILE=$1 +GPUS=$2 +NODE=${NODE:-1} +NODE_RANK=${NODE_RANK:-0} +ADDR=${ADDR:-127.0.0.1} +PORT=${PORT:-12345} + +python3 -m oneflow.distributed.launch \ +--nproc_per_node $GPUS --nnodes $NODE --node_rank $NODE_RANK --master_addr $ADDR --master_port $PORT \ +$FILE ${@:3} + diff --git a/tools/preprocess_data.py b/tools/preprocess_data.py new file mode 100644 index 0000000000000000000000000000000000000000..fdc1a3ddf415e6f765746cdb79e77b88bbb5fa7d --- /dev/null +++ b/tools/preprocess_data.py @@ -0,0 +1,265 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Processing data for pretraining.""" + +import argparse +import json +import multiprocessing +import os +import sys +import time + +import oneflow as flow +from omegaconf import OmegaConf + +from libai.config import LazyCall + +try: + import nltk + + nltk_available = True +except ImportError: + nltk_available = False + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) + +from libai import tokenizer +from libai.data.data_utils import indexed_dataset +from libai.tokenizer import build_tokenizer + + +# https://stackoverflow.com/questions/33139531/preserve-empty-lines-with-nltks-punkt-tokenizer +class CustomLanguageVars(nltk.tokenize.punkt.PunktLanguageVars): + + _period_context_fmt = r""" + \S* # some word material + %(SentEndChars)s # a potential sentence ending + \s* # <-- THIS is what I changed + (?=(?P + %(NonWord)s # either other punctuation + | + (?P\S+) # <-- Normally you would have \s+ here + ))""" + + +class IdentitySplitter(object): + def tokenize(self, *text): + return text + + +class Encoder(object): # split sentence, tokenize + def __init__(self, args, cfg): + self.args = args + self.cfg = cfg + + def initializer(self): + # Use Encoder class as a container for global data + Encoder.tokenizer = build_tokenizer(self.cfg) + if self.args.split_sentences: + if not nltk_available: + print("NLTK is not available to split sentences.") + exit() + splitter = nltk.load("tokenizers/punkt/english.pickle") + if self.args.keep_newlines: + # this prevents punkt from eating newlines after sentences + Encoder.splitter = nltk.tokenize.punkt.PunktSentenceTokenizer( + train_text=splitter._params, lang_vars=CustomLanguageVars() + ) + else: + Encoder.splitter = splitter + + else: + Encoder.splitter = IdentitySplitter() + + def encode(self, json_line): + data = json.loads(json_line) + ids = {} + for key in self.args.json_keys: + text = data[key] + doc_ids = [] + for sentence in Encoder.splitter.tokenize(text): + sentence_ids = Encoder.tokenizer.encode(sentence) + if len(sentence_ids) > 0: + doc_ids.append(sentence_ids) + if ( + len(doc_ids) > 0 and self.args.append_eod + ): # append eod token when at the enc of document + doc_ids[-1].append(Encoder.tokenizer.eod) + ids[key] = doc_ids + return ids, len(json_line) + + +def get_args(): + parser = argparse.ArgumentParser() + group = parser.add_argument_group(title="input data") + group.add_argument("--input", type=str, required=True, help="Path to input JSON") + group.add_argument( + "--json-keys", + nargs="+", + default=["text"], + help="space separate listed of keys to extract from json", + ) + group.add_argument( + "--split-sentences", action="store_true", help="Split documents into sentences." + ) + group.add_argument( + "--keep-newlines", + action="store_true", + help="Keep newlines between sentences when splitting.", + ) + + group = parser.add_argument_group(title="tokenizer") + group.add_argument( + "--tokenizer-name", + type=str, + required=True, + choices=["BertTokenizer", "GPT2Tokenizer", "T5Tokenizer", "RobertaTokenizer"], + help="What type of tokenizer to use.", + ) + group.add_argument("--vocab-file", type=str, default=None, help="Path to the vocab file") + group.add_argument( + "--merges-file", + type=str, + default=None, + help="Path to the BPE merge file (if necessary).", + ) + group.add_argument("--do-lower-case", action="store_true", help="Whether to do lower case.") + group.add_argument("--extra-ids", type=int, default=0, help="Number of extra ids.") + group.add_argument( + "--append-eod", + action="store_true", + help="Append an token to the end of a document.", + ) + group.add_argument( + "--do-chinese-wwm", action="store_true", help="Whether to do whole word mask for Chinese." + ) + + group = parser.add_argument_group(title="output data") + group.add_argument( + "--output-prefix", + type=str, + required=True, + help="Path to binary output file without suffix", + ) + group.add_argument( + "--dataset-impl", type=str, default="mmap", choices=["lazy", "cached", "mmap"] + ) + + group = parser.add_argument_group(title="runtime") + group.add_argument( + "--workers", type=int, default=1, help="Number of worker processes to launch" + ) + group.add_argument( + "--log-interval", + type=int, + default=100, + help="Interval between progress updates", + ) + args = parser.parse_args() + + if args.tokenizer_name.startswith("Bert"): + if not args.split_sentences: + print("Bert tokenizer detected, are you sure you don't want to split sentences?") + + return args + + +def parse_args_to_config(args): + + tokenization = OmegaConf.create() + + tokenization.tokenizer = LazyCall(getattr(tokenizer, args.tokenizer_name))( + vocab_file="bert-base-chinese-vocab.txt", + do_lower_case=True, + do_chinese_wwm=True, + ) + + tokenization.tokenizer.vocab_file = args.vocab_file + tokenization.tokenizer.merges_file = args.merges_file + tokenization.tokenizer.do_lower_case = args.do_lower_case + tokenization.tokenizer.extra_id = args.extra_ids + tokenization.tokenizer.do_chinese_wwm = args.do_chinese_wwm + tokenization.append_eod = args.append_eod + + return tokenization + + +def main(): + args = get_args() + cfg = parse_args_to_config(args) + startup_start = time.time() + + print("Opening", args.input) + fin = open(args.input, "r", encoding="utf-8") + + if nltk_available and args.split_sentences: + print("Start downloading punkt data...") + # Download url: http://www.nltk.org/nltk_data/ + nltk.download("punkt", quiet=True) + print("End download") + + encoder = Encoder(args, cfg) + tokenizer = build_tokenizer(cfg) + pool = multiprocessing.Pool(args.workers, initializer=encoder.initializer) + encoded_docs = pool.imap(encoder.encode, fin, 25) + + level = "document" + if args.split_sentences: + level = "sentence" + + print(f"Vocab size: {tokenizer.vocab_size}") + print(f"Output prefix: {args.output_prefix}") + output_bin_files = {} + output_idx_files = {} + builders = {} + for key in args.json_keys: + output_bin_files[key] = "{}_{}_{}.bin".format(args.output_prefix, key, level) + output_idx_files[key] = "{}_{}_{}.idx".format(args.output_prefix, key, level) + builders[key] = indexed_dataset.make_builder( + output_bin_files[key], impl=args.dataset_impl, vocab_size=len(tokenizer) + ) + + startup_end = time.time() + proc_start = time.time() + total_bytes_processed = 0 + print("Time to startup:", startup_end - startup_start) + + for i, (doc, bytes_processed) in enumerate(encoded_docs, start=1): + total_bytes_processed += bytes_processed + for key, sentences in doc.items(): + if len(sentences) == 0: + continue + for sentence in sentences: + builders[key].add_item( + flow.tensor(sentence, dtype=flow.int32) + ) # write data into .bin file + builders[key].end_document() + if i % args.log_interval == 0: + current = time.time() + elapsed = current - proc_start + mbs = total_bytes_processed / elapsed / 1024 / 1024 + print( + f"Processed {i} documents", + f"({i/elapsed} docs/s, {mbs} MB/s).", + file=sys.stderr, + ) + + for key in args.json_keys: + builders[key].finalize(output_idx_files[key]) # write data into .idx file + + +if __name__ == "__main__": + main() diff --git a/tools/train.sh b/tools/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..6adb9d6969f9f7eaa19477825865e647600e5ee1 --- /dev/null +++ b/tools/train.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +FILE=$1 +CONFIG=$2 +GPUS=$3 +NODE=${NODE:-1} +NODE_RANK=${NODE_RANK:-0} +ADDR=${ADDR:-127.0.0.1} +PORT=${PORT:-12345} + +export ONEFLOW_FUSE_OPTIMIZER_UPDATE_CAST=true + +echo "NODE_RANK=$NODE_RANK" + +python3 -m oneflow.distributed.launch \ +--nproc_per_node $GPUS --nnodes $NODE --node_rank $NODE_RANK --master_addr $ADDR --master_port $PORT \ +$FILE --config-file $CONFIG ${@:4} + diff --git a/tools/train_net.py b/tools/train_net.py new file mode 100644 index 0000000000000000000000000000000000000000..4e6a1d0b82ead6f494083deaf57e2d37433e45e1 --- /dev/null +++ b/tools/train_net.py @@ -0,0 +1,71 @@ +# coding=utf-8 +# Copyright 2021 The OneFlow Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import random +import sys + +import numpy as np +import oneflow as flow + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) +from libai.config import LazyConfig, default_argument_parser, try_get_key +from libai.engine import DefaultTrainer, default_setup +from libai.utils.checkpoint import Checkpointer + +logger = logging.getLogger("libai." + __name__) + + +def main(args): + cfg = LazyConfig.load(args.config_file) + cfg = LazyConfig.apply_overrides(cfg, args.opts) + default_setup(cfg, args) + + seed_for_rank = cfg.train.seed + flow.env.get_rank() + flow.manual_seed(seed_for_rank) + flow.cuda.manual_seed(seed_for_rank) + np.random.seed(seed_for_rank) + random.seed(seed_for_rank) + + if args.fast_dev_run: + cfg.train.train_epoch = 0 + cfg.train.train_iter = 20 + cfg.train.evaluation.eval_period = 10 + cfg.train.log_period = 1 + + if args.eval_only: + tokenizer = None + if try_get_key(cfg, "tokenization") is not None: + tokenizer = DefaultTrainer.build_tokenizer(cfg) + model = DefaultTrainer.build_model(cfg) + Checkpointer(model, save_dir=cfg.train.output_dir).resume_or_load( + cfg.train.load_weight, resume=args.resume + ) + if try_get_key(cfg, "train.graph.enabled", default=False): + model = DefaultTrainer.build_graph(cfg, model, is_train=False) + test_loader = DefaultTrainer.build_test_loader(cfg, tokenizer) + if len(test_loader) == 0: + logger.info("No dataset in dataloader.test, please set dataset for dataloader.test") + _ = DefaultTrainer.test(cfg, test_loader, model) + return + + trainer = DefaultTrainer(cfg) + return trainer.train() + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + main(args)