GPT2 base on megatron-deepspeed

megatron-deepspeed_dtk22.10/tests/conftest.py

+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# tests directory-specific settings - this file is run automatically
+# by pytest before any tests are run
+
+import sys
+import warnings
+from os.path import abspath, dirname, join
+
+
+# allow having multiple repository checkouts and not needing to remember to rerun
+# 'pip install -e .[dev]' when switching between checkouts and running tests.
+git_repo_path = abspath(join(dirname(dirname(__file__))))
+sys.path.insert(1, git_repo_path)
+
+# silence FutureWarning warnings in tests since often we can't act on them until
+# they become normal warnings - i.e. the tests still need to test the current functionality
+warnings.simplefilter(action="ignore", category=FutureWarning)
+
+
+def pytest_sessionfinish(session, exitstatus):
+    # If no tests are collected, pytest exists with code 5, which makes the CI fail.
+    if exitstatus == 5:
+        session.exitstatus = 0

megatron-deepspeed_dtk22.10/tests/data/gpt2/README.md

+Dataset used for testing.
+
+`ag_news_prompt*`: manually generated from dataset available at https://huggingface.co/datasets/TimeRobber/ag_news_classify_question_first_100
\ No newline at end of file

megatron-deepspeed_dtk22.10/tests/data/gpt2/generate_ag_news_mtf_dataset.sh

+python -c "from datasets import load_dataset; load_dataset('TimeRobber/ag_news_classify_question_first_100', split='train').to_json('ag_news_classify_question_first_100.jsonl')"
+
+python tools/preprocess_data.py \
+    --input ag_news_classify_question_first_100.jsonl \
+    --output-prefix tests/data/gpt2/ag_news_prompt \
+    --dataset-impl mmap \
+    --json-key targets \
+    --tokenizer-type PretrainedFromHF \
+    --tokenizer-name-or-path bigscience/tokenizer \
+    --append-eod \
+    --workers 8
+
+python tools/preprocess_data.py \
+    --input ag_news_classify_question_first_100.jsonl \
+    --output-prefix tests/data/gpt2/ag_news_prompt \
+    --dataset-impl mmap \
+    --json-key inputs \
+    --tokenizer-type PretrainedFromHF \
+    --tokenizer-name-or-path bigscience/tokenizer \
+    --workers 8
+
+rm ag_news_classify_question_first_100.jsonl

megatron-deepspeed_dtk22.10/tests/data/gpt2/gpt2-tiny-merges.txt

+#version: 0.2 - Trained by `huggingface/tokenizers`
+Ġ t
+Ġ a
+h e
+i n
+r e
+o n
+Ġt he
+e r
+Ġ s
+a t
+Ġ w
+Ġ o
+e n
+Ġ c
+i t
+i s
+a n
+o r
+e s
+Ġ b
+e d
+Ġ f
+in g
+Ġ p
+o u
+Ġa n
+a l
+a r
+Ġt o
+Ġ m
+Ġo f
+Ġ in
+Ġ d
+Ġ h
+Ġan d
+i c
+a s
+l e
+Ġt h
+i on
+o m
+l l
+en t
+Ġ n
+Ġ l
+s t
+Ġ re
+v e
+Ġ e
+r o
+l y
+Ġb e
+Ġ g
+Ġ T
+c t
+Ġ S
+i d
+o t
+Ġ I
+u t
+e t
+Ġ A
+Ġ is
+Ġ on
+i m
+a m
+o w
+a y
+a d
+s e
+Ġth at
+Ġ C
+i g
+Ġf or
+a c
+Ġ y
+v er
+u r
+Ġ u
+l d
+Ġs t
+Ġ M
+' s
+Ġ he
+Ġ it
+at ion
+it h
+i r
+c e
+Ġy ou
+i l
+Ġ B
+Ġw h
+o l
+Ġ P
+Ġw ith
+Ġ 1
+t er
+c h
+Ġa s
+Ġw e
+Ġ (
+n d
+i ll
+Ġ D
+i f
+Ġ 2
+a g
+er s
+k e
+Ġ "
+Ġ H
+e m
+Ġc on
+Ġ W
+Ġ R
+he r
+Ġw as
+Ġ r
+o d
+Ġ F
+u l
+at e
+Ġa t
+r i
+p p
+o re
+ĠT he
+Ġs e
+u s
+Ġp ro
+Ġh a
+u m
+Ġa re
+Ġd e
+a in
+an d
+Ġo r
+ig h
+es t
+is t
+a b
+r om
+Ġ N
+t h
+Ġc om
+Ġ G
+u n
+o p
+0 0
+Ġ L
+Ġn ot
+es s
+Ġe x
+Ġ v
+re s
+Ġ E
+e w
+it y
+an t
+Ġb y
+e l
+o s
+or t
+o c
+q u
+Ġf rom
+Ġha ve
+Ġs u
+i ve
+ou ld
+Ġs h
+Ġth is
+n t
+r a
+p e
+igh t
+ar t
+m ent
+Ġa l
+u st
+en d
+- -
+al l
+Ġ O
+ac k
+Ġc h
+Ġ le
+i es
+re d
+ar d
+â Ģ
+ou t
+Ġ J
+Ġa b
+e ar
+i v
+al ly
+ou r
+o st
+g h
+p t
+Ġp l
+as t
+Ġc an
+a k
+om e
+u d
+T he
+Ġh is
+Ġd o
+Ġg o
+Ġh as
+g e
+' t
+Ġ U
+r ou
+Ġs a
+Ġ j
+Ġb ut
+Ġw or
+Ġa ll
+e ct
+Ġ k
+am e
+Ġw ill
+o k
+Ġw he
+Ġthe y
+id e
+0 1
+f f
+ic h
+p l
+t her
+Ġt r
+. .
+Ġin t
+i e
+u re
+ag e
+Ġn e
+i al
+a p
+in e
+ic e
+Ġm e
+Ġo ut
+an s
+on e
+on g
+ion s
+Ġwh o
+Ġ K
+Ġu p
+Ġthe ir
+Ġa d
+Ġ 3
+Ġu s
+at ed
+ou s
+Ġm ore
+u e
+o g
+ĠS t
+in d
+i ke
+Ġs o
+im e
+p er
+. "
+b er
+i z
+a ct
+Ġon e
+Ġsa id
+Ġ -
+a re
+Ġyou r
+c c
+ĠT h
+Ġc l
+e p
+a ke
+ab le
+i p
+Ġcon t
+Ġwh ich
+i a
+Ġ im
+Ġab out
+Ġwe re
+ver y
+u b
+Ġh ad
+Ġ en
+Ġcom p
+, "
+ĠI n
+Ġu n
+Ġa g
+i re
+ac e
+a u
+ar y
+Ġw ould
+as s
+r y
+Ġ âĢ
+c l
+o ok
+e re
+s o
+Ġ V
+ig n
+i b
+Ġof f
+Ġt e
+v en
+Ġ Y
+i le
+o se
+it e
+or m
+Ġ2 01
+Ġre s
+Ġm an
+Ġp er
+Ġo ther
+or d
+ul t
+Ġbe en
+Ġl ike
+as e
+an ce
+k s
+ay s
+ow n
+en ce
+Ġd is
+ct ion
+Ġan y
+Ġa pp
+Ġs p
+in t
+res s
+ation s
+a il
+Ġ 4
+ic al
+Ġthe m
+Ġhe r
+ou nt
+ĠC h
+Ġa r
+Ġ if
+Ġthe re
+Ġp e
+Ġy ear
+a v
+Ġm y
+Ġs ome
+Ġwhe n
+ou gh
+ac h
+Ġth an
+r u
+on d
+ic k
+Ġo ver
+ve l
+Ġ qu
+Ċ Ċ
+Ġs c
+re at
+re e
+ĠI t
+ou nd
+p ort
+Ġal so
+Ġp art
+f ter
+Ġk n
+Ġbe c
+Ġt ime
+en s
+Ġ 5
+op le
+Ġwh at
+Ġn o
+d u
+m er
+an g
+Ġn ew
+-- --
+Ġg et
+or y
+it ion
+ing s
+Ġj ust
+Ġint o
+Ġ 0
+ent s
+o ve
+t e
+Ġpe ople
+Ġp re
+Ġit s
+Ġre c
+Ġt w
+i an
+ir st
+ar k
+or s
+Ġwor k
+ad e
+o b
+Ġs he
+Ġo ur
+w n
+in k
+l ic
+Ġ1 9
+ĠH e
+is h
+nd er
+au se
+Ġh im
+on s
+Ġ [
+Ġ ro
+f orm
+i ld
+at es
+ver s
+Ġon ly
+o ll
+Ġs pe
+c k
+e ll
+am p
+Ġa cc
+Ġb l
+i ous
+ur n
+f t
+o od
+Ġh ow
+he d
+Ġ '
+Ġa fter
+a w
+Ġat t
+o v
+n e
+Ġpl ay
+er v
+ic t
+Ġc ould
+it t
+Ġa m
+Ġf irst
+Ġ 6
+Ġa ct
+Ġ $
+e c
+h ing
+u al
+u ll
+Ġcom m
+o y
+o ld
+c es
+at er
+Ġf e
+Ġbe t
+w e
+if f
+Ġtw o
+oc k
+Ġb ack
+) .
+id ent
+Ġu nder
+rou gh
+se l
+x t
+Ġm ay
+rou nd
+Ġp o
+p h
+is s
+Ġd es
+Ġm ost
+Ġd id
+Ġad d
+j ect
+Ġin c
+f ore
+Ġp ol
+on t
+Ġag ain
+cl ud
+ter n
+Ġkn ow
+Ġne ed
+Ġcon s
+Ġc o
+Ġ .
+Ġw ant
+Ġse e
+Ġ 7
+n ing
+i ew
+ĠTh is
+c ed
+Ġe ven
+Ġin d
+t y
+ĠW e
+at h
+Ġthe se
+Ġp r
+Ġu se
+Ġbec ause
+Ġf l
+n g
+Ġn ow
+ĠâĢ ĵ
+c om
+is e
+Ġm ake
+Ġthe n
+ow er
+Ġe very
+ĠU n
+Ġse c
+os s
+u ch
+Ġe m
+Ġ =
+ĠR e
+i ed
+r it
+Ġin v
+le ct
+Ġsu pp
+at ing
+Ġl ook
+m an
+pe ct
+Ġ 8
+ro w
+Ġb u
+Ġwhe re
+if ic
+Ġyear s
+i ly
+Ġd iff
+Ġsh ould
+Ġre m
+T h
+I n
+Ġe v
+d ay
+' re
+ri b
+Ġre l
+s s
+Ġde f
+Ġr ight
+Ġs y
+) ,
+l es
+00 0
+he n
+Ġth rough
+ĠT r
+_ _
+Ġw ay
+Ġd on
+Ġ ,
+Ġ1 0
+as ed
+Ġas s
+ub lic
+Ġre g
+ĠA nd
+i x
+Ġ very
+Ġin clud
+ot her
+Ġim p
+ot h
+Ġsu b
+ĠâĢ Ķ
+Ġbe ing
+ar g
+ĠW h
+= =
+ib le
+Ġdo es
+an ge
+r am
+Ġ 9
+er t
+p s
+it ed
+ation al
+Ġb r
+Ġd own
+Ġman y
+ak ing
+Ġc all
+ur ing
+it ies
+Ġp h
+ic s
+al s
+Ġde c
+at ive
+en er
+Ġbe fore
+il ity
+Ġwe ll
+Ġm uch
+ers on
+Ġth ose
+Ġsu ch
+Ġ ke
+Ġ end
+ĠB ut
+as on
+t ing
+Ġl ong
+e f
+Ġth ink
+y s
+Ġbe l
+Ġs m
+it s
+a x
+Ġo wn
+Ġpro v
+Ġs et
+if e
+ment s
+b le
+w ard
+Ġsh ow
+Ġp res
+m s
+om et
+Ġo b
+Ġs ay
+ĠS h
+t s
+f ul
+Ġe ff
+Ġg u
+Ġin st
+u nd
+re n
+c ess
+Ġ ent
+ĠY ou
+Ġgo od
+Ġst art
+in ce
+Ġm ade
+t t
+st em
+ol og
+u p
+Ġ |
+um p
+Ġhe l
+ver n
+ul ar
+u ally
+Ġa c
+Ġm on
+Ġl ast
+Ġ2 00
+1 0
+Ġst ud
+u res
+ĠA r
+sel f
+ar s
+mer ic
+u es
+c y
+Ġm in
+oll ow
+Ġc ol
+i o
+Ġm od
+Ġc ount
+ĠC om
+he s
+Ġf in
+a ir
+i er
+âĢ Ķ
+re ad
+an k
+at ch
+e ver
+Ġst r
+Ġpo int
+or k
+ĠN ew
+Ġs ur
+o ol
+al k
+em ent
+Ġus ed
+ra ct
+we en
+Ġs ame
+ou n
+ĠA l
+c i
+Ġdiff ere
+Ġwh ile
+---- ----
+Ġg ame
+ce pt
+Ġs im
+.. .
+Ġin ter
+e k
+Ġre port
+Ġpro du
+Ġst ill
+l ed
+a h
+Ġhe re
+Ġwor ld
+Ġth ough
+Ġn um
+ar ch
+im es
+al e
+ĠS e
+ĠI f
+/ /
+ĠL e
+Ġre t
+Ġre f
+Ġtr ans
+n er
+ut ion
+ter s
+Ġt ake
+ĠC l
+Ġcon f
+w ay
+a ve
+Ġgo ing
+Ġs l
+u g
+ĠA meric
+Ġspe c
+Ġh and
+Ġbet ween
+ist s
+ĠD e
+o ot
+I t
+Ġe ar
+Ġagain st
+Ġh igh
+g an
+a z
+at her
+Ġex p
+Ġo p
+Ġin s
+Ġg r
+Ġhel p
+Ġre qu
+et s
+in s
+ĠP ro
+is m
+Ġf ound
+l and
+at a
+us s
+am es
+Ġp erson
+Ġg reat
+p r
+Ġs ign
+ĠA n
+' ve
+Ġs omet
+Ġs er
+h ip
+Ġr un
+Ġ :
+Ġt er
+ire ct
+Ġf ollow
+Ġd et
+ic es
+Ġf ind
+1 2
+Ġm em
+Ġc r
+e red
+e x
+Ġex t
+ut h
+en se
+c o
+Ġte am
+v ing
+ou se
+as h
+at t
+v ed
+Ġsy stem
+ĠA s
+d er
+iv es
+m in
+Ġle ad
+ĠB l
+c ent
+Ġa round
+Ġgo vern
+Ġc ur
+vel op
+an y
+Ġc our
+al th
+ag es
+iz e
+Ġc ar
+od e
+Ġl aw
+Ġre ad
+' m
+c on
+Ġre al
+Ġsupp ort
+Ġ1 2
+.. ..
+Ġre ally
+n ess
+Ġf act
+Ġd ay
+Ġb oth
+y ing
+Ġs erv
+ĠF or
+Ġth ree
+Ġw om
+Ġm ed
+od y
+ĠThe y
+5 0
+Ġex per
+t on
+Ġe ach
+ak es
+Ġc he
+Ġc re
+in es
+Ġre p
+1 9
+g g
+ill ion
+Ġg rou
+ut e
+i k
+W e
+g et
+E R
+Ġm et
+Ġs ays
+o x
+Ġd uring
+er n
+iz ed
+a red
+Ġf am
+ic ally
+Ġha pp
+ĠI s
+Ġch ar
+m ed
+v ent
+Ġg ener
+i ent
+p le
+i et
+re nt
+1 1
+v es
+pt ion
+Ġ2 0
+form ation
+Ġc or
+Ġoff ic
+ie ld
+Ġto o
+is ion
+Ġin f
+Ġ Z
+t he
+o ad
+Ġp ublic
+Ġpro g
+r ic
+* *
+Ġw ar
+Ġp ower
+v iew
+Ġf ew
+Ġl oc
+Ġdiffere nt
+Ġst ate
+Ġhe ad
+' ll
+Ġp oss
+Ġst at
+re t
+ant s
+Ġv al
+Ġis s
+Ġc le
+i vers
+an c
+Ġex pl
+Ġan other
+Ġ Q
+Ġa v
+th ing
+n ce
+W h
+Ġch ild
+Ġs ince
+i red
+l ess
+Ġl ife
+Ġde velop
+itt le
+Ġde p
+Ġp ass
+ã ĥ
+Ġt urn
+or n
+Th is
+b ers
+ro ss
+ĠA d
+Ġf r
+Ġres p
+Ġsec ond
+o h
+Ġ /
+Ġdis c
+Ġ &
+Ġsomet hing
+Ġcomp le
+Ġ ed
+Ġf il
+Ġmon th
+a j
+u c
+Ġgovern ment
+Ġwith out
+Ġle g
+Ġd ist
+Ġp ut
+Ġqu est
+an n
+Ġpro t
+2 0
+Ġne ver
+i ence
+Ġle vel
+Ġar t
+Ġth ings
+Ġm ight
+Ġeff ect
+Ġcont ro
+Ġc ent
+Ġ1 8
+Ġall ow
+Ġbel ie
+ch ool
+ot t
+Ġinc re
+Ġfe el
+Ġres ult
+Ġl ot
+Ġf un
+ot e
+Ġt y
+ere st
+Ġcont in
+Ġus ing
+Ġb ig
+2 01
+Ġas k
+Ġb est
+Ġ )
+I N
+Ġo pp
+3 0
+Ġnum ber
+in ess
+S t
+le ase
+Ġc a
+Ġm ust
+Ġd irect
+Ġg l
+Ġ <
+Ġop en
+Ġp ost
+Ġcom e
+Ġse em
+ord ing
+Ġwe ek
+ate ly
+it al
+Ġe l
+ri end
+Ġf ar
+Ġt ra
+in al
+Ġp ri
+ĠU S
+Ġpl ace
+Ġfor m
+Ġto ld
+" :
+ain s
+at ure
+ĠTr ump
+Ġst and
+Ġ #
+id er
+ĠF r
+Ġne xt
+Ġs oc
+Ġp ur
+Ġle t
+Ġl ittle
+Ġh um
+Ġ i
+r on
+1 5
+Ġ1 5
+Ġcomm un
+Ġm ark
+ĠThe re
+Ġw r
+ĠTh at
+Ġin formation
+w ays
+Ġb us
+a pp
+Ġinv est
+m e
+Ġh ard
+ain ed
+e ad
+Ġim port
+Ġapp ro
+Ġt est
+Ġt ri
+Ġre st
+os ed
+Ġf ull
+Ġc are
+ĠS p
+Ġc ase
+O N
+Ġs k
+Ġl ess
+Ġ +
+Ġpart ic
+ĠP l
+ab ly
+u ck
+is hed
+ch n
+b e
+Ġl ist
+at or
+Ġto p
+Ġad v
+ĠB e
+ru ct
+Ġd em
+r ation
+l ing
+g y
+re en
+g er
+Ġh ome
+Ġle ft
+Ġbet ter
+Ġd ata
+Ġ1 1
+Ġatt ack
+Ġpro ble
+l ine
+ard s
+Ġbe h
+r al
+ĠH ow
+ĠS he
+ar ge
+Ġ --
+: //
+Ġb ro
+ĠP h
+at s
+Ġbu ild
+w w
+id ed
+a im
+as es
+en cy
+Ġm ain
+in ed
+Ġinclud ing
+Ġ {
+Ġg ot
+Ġint erest
+Ġke ep
+Ġ X
+Ġe as
+ain ing
+Ġcl ass
+âĢ ¦
+ĠN o
+Ġv ar
+Ġsm all
+amp le
+A T
+Ġ ide
+ĠS o
+Ġre ce
+Ġpol it
+Ġm ov
+Ġpl an
+Ġper cent
+iv ing
+Ġc amp
+Ġp ay
+1 4
+s c
+is ed
+Ġu nt
+one y
+pl oy
+== ==
+Ġdid n
+ĠI nd
+el s
+ert ain
+Ġp os
+__ __
+i ver
+Ġpro cess
+Ġprog ram
+if ied
+ĠR ep
+1 6
+u ro
+olog y
+at ter
+in a
+Ġn ame
+ĠA ll
+Ġf our
+Ġret urn
+v ious
+b s
+Ġcall ed
+Ġm ove
+ĠS c
+ir d
+Ġgrou p
+Ġb re
+Ġm en
+Ġc ap
+t en
+e e
+Ġd ri
+le g
+he re
+uth or
+Ġp at
+Ġcur rent
+id es
+Ġp op
+t o
+ent ion
+Ġal ways
+Ġm il
+Ġwom en
+Ġ1 6
+Ġo ld
+iv en
+ra ph
+ĠO r
+r or
+ent ly
+Ġn ear
+ĠE x
+re am
+s h
+Ġ1 4
+Ġf ree
+iss ion
+st and
+ĠC on
+al ity
+us ed
+1 3
+Ġdes ign
+Ġch ange
+Ġch ang
+Ġb o
+Ġv is
+em ber
+Ġb ook
+read y
+Ġk ill
+2 5
+pp ed
+Ġa way
+Ġab le
+Ġcount ry
+Ġcon st
+ar n
+Ġor der
+A R
+i or
+i um
+or th
+1 8
+ail able
+Ġs w
+Ġm illion
+Ġ1 3
+at ic
+t ed
+ĠG o
+Ġo per
+en g
+Ġth ing
+aj or
+con om
+ĠCom m
+Ġwh y
+u red
+ur al
+Ġs chool
+b y
+ĠM ar
+Ġa ff
+Ġd ays
+Ġan n
+us h
+an e
+I f
+e g
+Ġpro f
+Ġhe alth
+ou th
+B ut
+ion al
+. ,
+Ġs ol
+Ġal ready
+Ġ3 0
+Ġchar act
+H e
+Ġf riend
+E S
+i ans
+ic le
+' d
+ĠO n
+Ġle ast
+Ġp rom
+Ġd r
+Ġh ist
+it her
+Ġ est
+i qu
+1 7
+s on
+Ġte ll
+Ġt alk
+oh n
+o int
+le ction
+A N
+Ġunt il
+au gh
+Ġl ater
+Ġ ve
+Ġv iew
+end ing
+iv ed
+Ġwor d
+w are
+Ġc ost
+Ġen ough
+Ġg ive
+ĠUn ited
+Ġte chn
+are nt
+O R
+Ġp ar
+ĠD r
+Ġ201 6
+r ist
+er ing
+Ġ Â
+Ġl arge
+s ide
+ac y
+cc ess
+Ġw in
+Ġimport ant
+Ġ19 9
+Ġdoes n
+Ġ1 7
+Ġbus iness
+Ġcle ar
+Ġre se
+" ,
+ur y
+Ġe qu
+as ter
+al f
+ĠAmeric an
+n ect
+Ġex pect
+ivers ity
+Ġo cc
+ĠF l
+Ġk ind
+Ġme an
+Ġp ast
+Ġde v
+Ġb as
+le t
+ra ft
+Ġor gan
+Ġde l
+Ġper form
+Ġst ory
+Ġse ason
+ĠC ol
+Ġcl aim
+Ġc ame
+Ġwith in
+Ġl ine
+Ġpro ject
+ĠA t
+Ġcontro l
+end ed
+ĠS y
+Ġa ir
+iz ation
+Ġ *
+le y
+Ġm oney
+id d
+Y ou
+f or
+Ġfam ily
+Ġm aking
+Ġb it
+Ġpol ice
+Ġhapp en
+Ġ vers
+on y
+u ff
+ĠW hen
+Ġs it
+ide o
+l f
+is on
+Ġsu re
+g in
+Ġapp ear
+Ġl ight
+Ġ es
+o f
+Ġw ater
+Ġt imes
+n ot
+Ġg row
+Ġcomp any
+ĠT e
+ow s
+Ġm ar
+our ce
+i ol
+ar m
+b r
+Ġex ample
+Ġcon c
+Ġf ore
+ĠT o
+p ro
+E N
+ri es
+Ġ2 5
+ĠC an
+ne y
+Ġact ually
+Ġe ver
+ur ity
+ak en
+ap s
+Ġt ax
+Ġm ajor
+am a
+Ġof ten
+er al
+Ġhum an
+Ġj ob
+is ter
+Ġav ailable
+oc r
+en n
+a id
+iv id
+Ġrec ord
+? "
+Ġs ing
+ĠA m
+id ence
+Ġnew s
+st er
+Ġe conom
+Ġfollow ing
+ĠB r
+is ing
+Ġh our
+m ost
+um ent
+Ġse x
+Ġdes c
+Ġbec ome
+ĠE d
+Ġto ok
+Ġha ving
+Ġprodu ct
+a ult
+A s
+ar ing
+Ġme ans
+Ġh op
+un e
+Ġch o
+Ġc ertain
+Ġn on
+Ġde al
+2 4
+le ment
+oc i
+en e
+Ġs ide
+ĠP r
+ĠM ay
+Ġre ason
+u ed
+c hed
+ul ation
+Ġe lect
+Ġoffic ial
+Ġposs ible
+Ġh old
+and s
+ot s
+Ġc ity
+or ies
+Ġse ver
+Ġchild ren
+Ġon ce
+Ġact iv
+l er
+Ġn ight
+it ions
+ĠJ ohn
+a pe
+pl ay
+Ġd one
+Ġl im
+Ġwork ing
+ĠP res
+or ld
+e b
+ĠC o
+Ġb ody
+ail s
+ut es
+ĠM r
+Ġwhe ther
+Ġa uthor
+ro p
+Ġpro per
+Ġse en
+) ;
+Ġf ac
+ĠS u
+Ġcon d
+it ing
+Ġcour se
+Ġ }
+-------- --------
+a ign
+Ġev ent
+Ġen g
+Ġp ot
+Ġin tern
+i am
+Ġsh ort
+em pt
+ã Ĥ
+ĠG od
+il ar
+8 0
+Ġor ig
+I S
+our n
+ab ility
+it ive
+Ġd am
+Ġ1 00
+Ġp ress
+Ġdo ing
+Ġprot ect
+r ing
+Ġthough t
+Ġquest ion
+re w
+ĠW ar
+Ġsever al
+ĠSt ate
+Ġg iven
+Ġf und
+ĠT w
+Ġw ent
+an ces
+w ork
+p or
+m y
+4 0
+Ġar g
+art ment
+ust om
+Ġpol ic
+Ġme et
+Ġc reat
+2 2
+ĠSt ates
+Ġg ames
+ra w
+ut ure
+Ġunder stand
+ur s
+ĠO b
+l ish
+s y
+Ġm akes
+Ġw on
+ag on
+Ġh tt
+Ġl ove
+ent ial
+Ġcomple te
+p ar
+ĠI m
+A L
+Ġacc ount
+Â ł
+ore d
+ver t
+Ġ ident
+Ġ201 5
+Ġother s
+ĠM in
+i ber
+ver age
+The re
+ition al
+d d
+Ġpro b
+Ġyou ng
+Ġal ong
+Ġacc ording
+Ġy et
+Ġmem bers
+ĠWh at
+o id
+ĠM an
+A nd
+Ġam ong
+a i
+Ġem ploy
+ĠR es
+Ġ >
+Ġinv ol
+Ġl ow
+a f
+ĠC ar
+Ġh ig
+ĠO ne
+ĠS ec
+in ation
+Ġlike ly
+Ġan t
+ag ed
+ĠR uss
+Ġb en
+Ġre le
+F or
+b ack
+ĠN ot
+Ġpres ident
+b all
+Ġacc ess
+ivid ual
+ĠD em
+ĠE uro
+6 0
+Ġkn own
+ir l
+ĠG r
+Ġear ly
+u se
+iet y
+âĢ ĵ
+Ġf ight
+Ġs ent
+Ġto day
+Ġmark et
+" .
+Ġb ased
+Ġstr ong
+ur ther
+Ġde b
+m ber
+Ġproble m
+Ġde ath
+Ġsoc ial
+im ate
+A S
+ort un
+Ġcamp aign
+er y
+C h
+Ġe y
+i ally
+Ġm us
+w h
+p os
+Ġ er
+Ġsa f
+Ġmonth s
+ir on
+Ġv iol
+Ġf ive
+Ġst re
+Ġplay ers
+in c
+al d
+y ear
+a un
+Ġsu ccess
+Ġpres ent
+ere nce
+Ġ201 4
+Ġsu gg
+Ġpartic ular
+Ġtr y
+Ġsugg est
+ĠCh rist
+on es
+Ġpri v
+2 3
+Ġc rit
+Ġl and
+Ġloc al
+if y
+2 9
+Ġa ut
+E D
+ĠG u
+Ġm ult
+Ġpolit ical
+Ġask ed
+Ġfor mer
+it ter
+ri pt
+Ġcl ose
+Ġp ract
+ĠY ork
+Ġget ting
+Ġac ross
+Ġcom b
+Ġbelie ve
+Ġ z
+Ġto get
+Ġtoget her
+ĠC ent
+ir c
+Ġind ividual
+ĠM c
+2 7
+is k
+ĠE ng
+Ġf ace
+Ġ2 4
+Ġval ue
+Ġare a
+e v
+Ġw rit
+ĠPres ident
+Ġv ot
+Ġke y
+Ġm om
+p ut
+Ġany thing
+Ġexper ience
+att le
+Ġm ind
+a ff
+om m
+Ġf uture
+g ed
+Ġc ut
+Ġto t
+it ch
+Ġv ideo
+Ġinvest ig
+Ġn et
+ĠM y
+r ict
+i en
+. )
+Ġimp ro
+th ough
+ward s
+Ġcon nect
+ĠM ed
+sel ves
+ens ive
+m b
+o ber
+at ors
+A n
+Ġ5 0
+Ġre du
+res ent
+Ġab ove
+Ġf re
+ĠEuro pe
+s w
+Ġam ount
+ĠA pp
+Ġe ither
+Ġmil it
+Ġan al
+Ġf ail
+ĠE n
+al es
+Ġspec ial
+Ġbl ack
+I T
+c her
+Ġlook ing
+Ġf ire
+y n
+Ġal most
+o on
+Ġstud y
+Ġm iss
+c hes
+ro wn
+Ġt re
+Ġcommun ity
+Ġmed ia
+Ġf ood
+Ġcom es
+ĠUn iversity
+Ġsing le
+Wh at
+u ly
+Ġh alf
+ag ue
+h od
+ĠRep ublic
+Ġstart ed
+Ġqu ick
+ot o
+b ook
+Ġiss ue
+it or
+Ġel se
+Ġcons ider
+2 6
+ro du
+Ġt aken
+2 8
+9 9
+ĠW ith
+Ġtr ue
+Ġw a
+Ġtr ad
+Ġag o
+Ġm ess
+ie f
+Ġadd ed
+o ke
+Ġb ad
+Ġf av
+3 3
+Ġsim ilar
+as k
+ĠD on
+Ġcharact er
+ort s
+ĠH ouse
+Ġreport ed
+Ġty pe
+v al
+i od
+ĠHow ever
+Ġt arg
+Ġent ire
+pp ing
+Ġhist ory
+Ġl ive
+ff ic
+.... ....
+ed eral
+Ġtr ying
+Ġdisc uss
+ĠH ar
+ac es
+l ished
+Ġse lf
+os p
+re st
+Ġro om
+el t
+Ġf all
+ol ution
+Ġe t
+Ġ x
+Ġis n
+Ġide a
+b o
+Ġs ound
+ĠD ep
+Ġsome one
+ci ally
+ull y
+Ġf oc
+Ġob ject
+if t
+ap er
+Ġplay er
+Ġr ather
+Ġserv ice
+as hing
+ĠD o
+ĠP art
+ru g
+m on
+p ly
+Ġm or
+Ġnot hing
+Ġprov ide
+I C
+un g
+Ġpart y
+Ġex ist
+Ġm ag
+7 0
+Ġr ul
+Ġh ouse
+Ġbeh ind
+Ġhow ever
+ĠW orld
+Ġs um
+Ġapp lic
+Ġ ;
+Ġfun ction
+g r
+ĠP ol
+Ġfr ont
+2 00
+Ġser ies
+Ġt em
+Ġty p
+ill s
+Ġo pt
+Ġpoint s
+Ġbel ow
+itt ed
+Ġspec ific
+Ġ201 7
+um b
+Ġr a
+Ġpre vious
+Ġpre t
+re me
+Ġc ustom
+Ġcour t
+ĠM e
+Ġre pl
+Ġwho le
+g o
+c er
+Ġt reat
+ĠA ct
+Ġprob ably
+Ġle arn
+end er
+ĠA ss
+Ġvers ion
+n ow
+Ġche ck
+ĠC al
+R E
+min ist
+O n
+our ces
+Ġben ef
+Ġd oc
+Ġdet er
+Ġen c
+Ġsu per
+Ġadd ress
+Ġv ict
+Ġ201 3
+Ġme as
+t r
+Ġf ield
+W hen
+Ġsign ific
+u ge
+Ġfe at
+Ġcomm on
+l oad
+Ġbe gin
+Ġbr ing
+Ġa ction
+er man
+Ġdesc rib
+Ġind ust
+Ġwant ed
+ri ed
+m ing
+Ġatt empt
+4 5
+f er
+Ġd ue
+ress ion
+# #
+Ġsh all
+Ġs ix
+o o
+Ġst ep
+Ġp ub
+Ġhim self
+Ġ2 3
+Ġc op
+Ġd est
+Ġst op
+A C
+ib ility
+Ġl ab
+ic ult
+Ġhour s
+Ġcre ate
+Ġf urther
+ĠAmeric a
+ĠC ity
+Ġd ou
+he ad
+S T
+ĠN orth
+c ing
+Ġn ational
+u le
+ĠIn st
+Ġt aking
+ĠQ u
+ir t
+Ġre d
+Ġrese arch
+v iron
+ĠG e
+Ġbre ak
+an a
+Ġsp ace
+ater ial
+Ġrec ent
+ĠA b
+Ġgener al
+Ġh it
+Ġper iod
+Ġevery thing
+ive ly
+Ġph ys
+Ġsay ing
+an ks
+Ġc ou
+Ġc ult
+ac ed
+e al
+u ation
+Ġc oun
+l u
+Ġinclud e
+Ġpos ition
+ĠA fter
+ĠCan ad
+ĠE m
+Ġim m
+ĠR ed
+Ġp ick
+Ġcom pl
+Ġm atter
+re g
+e xt
+ang u
+is c
+o le
+a ut
+Ġcomp et
+e ed
+f ect
+Ġ2 1
+ĠS en
+ĠThe se
+as ing
+Ġcan not
+Ġin it
+Ġrel ations
+ac hed
+Ġb ar
+Ġ4 0
+ĠT H
+Ġ201 2
+Ġv ol
+Ġg round
+Ġsec urity
+Ġup d
+il t
+3 5
+Ġconc ern
+ĠJ ust
+Ġwh ite
+Ġseem s
+ĠH er
+pe cially
+i ents
+Ġann oun
+Ġf ig
+ight s
+Ġst ri
+l ike
+id s
+Ġs us
+Ġw atch
+Ġ â
+Ġw ind
+ĠC ont
+Ġit self
+Ġm ass
+A l
+y le
+iqu e
+ĠN ational
+Ġab s
+Ġp ack
+Ġout side
+Ġan im
+Ġp ain
+et er
+Ġman ag
+du ct
+og n
+Ġ ]
+ĠSe pt
+se c
+o ff
+ĠJ an
+Ġf oot
+ad es
+Ġth ird
+Ġm ot
+Ġev idence
+int on
+Ġth reat
+a pt
+pl es
+c le
+Ġl o
+Ġde cl
+Ġit em
+med i
+Ġrep resent
+om b
+am er
+Ġsignific ant
+og raph
+s u
+Ġc al
+i res
+00 00
+I D
+A M
+Ġsim ply
+Ġlong er
+Ġf ile
+O T
+c he
+S o
+ate g
+or g
+ĠH is
+Ġen er
+Ġd om
+Ġup on
+il i
+": "
+Ġthem selves
+Ġcom ing
+Ġqu ite
+Ġdiff icult
+ĠB ar
+il ities
+re l
+end s
+c ial
+6 4
+Ġwom an
+ra p
+y r
+Ġne cess
+ip s
+Ġte xt
+Ġrequ ire
+Ġmilit ary
+Ġre view
+Ġresp ons
+7 5
+Ġsub ject
+Ġinst ead
+Ġiss ues
+Ġg en
+" ,"
+Ġmin utes
+Ġwe ap
+r ay
+am ed
+t ime
+b l
+H ow
+Ġc ode
+ĠS m
+Ġhig her
+ĠSt e
+r is
+Ġp age
+Ġstud ents
+ĠIn tern
+Ġmet hod
+ĠA ug
+ĠP er
+ĠA g
+Ġpolic y
+ĠS w
+Ġex ec
+Ġac cept
+um e
+rib ut
+Ġword s
+Ġfin al
+Ġchang es
+ĠDem ocr
+Ġfriend s
+Ġres pect
+Ġe p
+Ġcomp an
+iv il
+Ġdam age
+** **
+og le
+viron ment
+Ġne g
+ent al
+Ġa p
+Ġtot al
+iv al
+! "
+l im
+Ġneed s
+Ġag re
+Ġdevelop ment
+Ġa ge
+ip le
+2 1
+Ġresult s
+ĠA f
+S h
+Ġg un
+ĠOb ama
+ro ll
+Ġ @
+Ġright s
+ĠB rit
+Ġrun ning
+Ġwas n
+Ġp ort
+Ġr ate
+Ġpret ty
+Ġtarg et
+Ġsa w
+Ġc irc
+Ġwor ks
+ic ro
+al t
+o ver
+ww w
+Th at
+l ier
+Ġevery one
+ud e
+Ġp ie
+idd le
+ra el
+Ġr ad
+Ġbl ock
+Ġw alk
+T o
+ã ģ
+n es
+ĠA ust
+a ul
+ro te
+ĠS outh
+ess ion
+op h
+Ġshow s
+Ġs ite
+Ġj o
+Ġr isk
+cl us
+l t
+Ġin j
+id ing
+ĠS pe
+Ġch all
+ir m
+Ġ2 2
+itt ing
+st r
+Ġh y
+L E
+ke y
+Ġbe gan
+at ur
+ashing ton
+l am
+ĠD av
+b it
+Ġs ize
+ĠP ar
+3 8
+ourn al
+f ace
+Ġdec ision
+Ġl arg
+Ġj ud
+re ct
+Ġcontin ue
+ĠO ct
+ove red
+ĠI nt
+==== ====
+Ġp arent
+ĠW ill
+Ġeas y
+Ġd rug
+ang er
+Ġs ense
+Ġd i
+id ay
+Ġener gy
+ist ic
+Ġass oci
+ar ter
+ob al
+e ks
+ĠE l
+ur ch
+Ġg irl
+o e
+it le
+Ġ2 8
+ĠC he
+Ġrequ est
+Ġso on
+Ġh ost
+k y
+Ġst ates
+om es
+Ġm aterial
+le x
+Ġmom ent
+Ġan sw
+on se
+Ġes pecially
+Ġn orm
+Ġserv ices
+p ite
+r an
+Ġro le
+4 4
+) :
+Ġc red
+C l
+____ ____
+Ġm at
+Ġl og
+ĠCl inton
+O U
+Ġoff ice
+Ġ2 6
+Ġch arg
+Ġtr ack
+m a
+Ġhe art
+Ġb all
+Ġperson al
+Ġbuild ing
+n a
+s et
+b ody
+ĠBl ack
+Ġincre ase
+itt en
+Ġneed ed
+3 6
+3 2
+= "
+Ġl ost
+Ġbec ame
+Ġgrou ps
+ĠM us
+Ġw rote
+ĠP e
+Ġpro p
+j oy
+Ã ©
+ĠWh ite
+Ġde ad
+. '
+Ġhtt p
+Ġwe bs
+O S
+Ġins ide
+Ġwr ong
+Ġstat ement
+Ġ ...
+y l
+Ġfil m
+Ġmus ic
+Ġsh are
+ific ation
+Ġre lease
+Ġfor ward
+Ġst ay
+Ġcomp ut
+it te
+s er
+Ġorig inal
+Ġc ard
+Ġc and
+Ġd iv
+at ural
+Ġfav or
+O M
+Ġc ases
+us es
+Ġse ction
+Ġle ave
+g ing
+ov ed
+ĠW ashington
+3 9
+ĠG l
+Ġrequ ired
+act ion
+ap an
+o or
+it er
+ĠK ing
+Ġcount ries
+ĠG erman
+ll ing
+Ġ2 7
+3 4
+Ġquest ions
+Ġpr im
+Ġc ell
+Ġsh oot
+Ġany one
+ĠW est
+Ġaff ect
+ep end
+Ġon line
+ĠIs rael
+ĠSept ember
+Ġab ility
+Ġcont ent
+is es
+Ġre ve
+Ġl aun
+Ġind ic
+Ġfor ce
+c ast
+Ġso ld
+av ing
+f l
+Ġso ft
+Ġcompan ies
+ce ed
+Ġart icle
+Ġa ud
+Ġre v
+Ġed uc
+Ġplay ing
+0 5
+Ġhe ld
+ct or
+Ġrele ased
+Ġf ederal
+3 7
+Ġad minist
+Ġinter view
+Ġinst all
+Ġrece ived
+Ġs ource
+u k
+P h
+Ġser ious
+Ġcre ated
+Ġc ause
+Ġim medi
+Ġdef in
+u el
+ĠDep artment
+ct ions
+ĠC our
+ĠN ow
+z e
+it es
+it ution
+Ġl ate
+Ġspe ak
+n ers
+Ġleg al
+ar i
+ĠC or
+Ġwe eks
+Ġmod el
+Ġp red
+Ġex act
+B C
+ĠB y
+IN G
+os ing
+Ġt akes
+Ġreg ard
+Ġopp ortun
+Ġpr ice
+Ġ19 8
+ĠA pr
+f ully
+Ġor d
+Ġproble ms
+ru ction
+h am
+ĠC ount
+le ge
+Ġlead ers
+E T
+le v
+Ġde ep
+olog ical
+es e
+h aps
+ĠS ome
+Ġp ers
+Ġcont ract
+Ġrelations hip
+s p
+ou d
+Ġb ase
+4 8
+m it
+A d
+anc ial
+Ġcons um
+Ġpot ential
+Ġl angu
+re m
+et h
+Ġrel ig
+ress ed
+6 6
+Ġl ink
+Ġl ower
+ay er
+ĠJ une
+Ġf em
+un t
+er c
+ur d
+Ġcont act
+Ġ ill
+Ġm other
+Ġest ab
+h tt
+ĠM arch
+ĠB ro
+ĠCh ina
+Ġ2 9
+Ġs qu
+Ġprov ided
+Ġa verage
+as ons
+Ġ201 1
+Ġex am
+l in
+5 5
+n ed
+Ġper fect
+Ġt ou
+al se
+u x
+Ġbu y
+Ġsh ot
+Ġcol lect
+Ġph ot
+Ġplay ed
+Ġsur pr
+Ġofficial s
+Ġsim ple
+av y
+Ġindust ry
+Ġhand s
+g round
+Ġp ull
+Ġr ound
+Ġus er
+Ġr ange
+u ary
+Ġpriv ate
+op s
+e es
+Ġw ays
+ĠM ich
+Ġve h
+Ġex cept
+Ġter ms
+im um
+pp er
+I ON
+ore s
+ĠDr agon
+ou l
+Ġd en
+Ġperform ance
+Ġb ill
+c il
+4 7
+Ġen vironment
+Ġex c
+ad d
+Ġwor th
+Ġp ict
+Ġch ance
+Ġ201 8
+b or
+Ġspe ed
+ict ion
+Ġal leg
+ĠJ apan
+at ory
+re et
+Ġm atch
+ĠI I
+Ġst ru
+ord er
+Ġst e
+Ġl iving
+Ġst ruct
+in o
+Ġse par
+her n
+Ġresp onse
+Ġen joy
+Ġv ia
+A D
+um ents
+ace book
+Ġmem ber
+ib r
+iz ing
+Ġto ol
+ĠM on
+ĠWh ile
+h ood
+ĠA ng
+ĠD ef
+Ġoff er
+T r
+a ur
+Ġturn ed
+ĠJ uly
+d own
+an ced
+Ġrec ently
+ĠE ar
+Ġc e
+ĠSt ar
+ĠC ong
+rough t
+Ġbl ood
+Ġhop e
+Ġcom ment
+ain t
+Ġar ri
+il es
+Ġpartic ip
+ough t
+ri ption
+0 8
+4 9
+Ġg ave
+Ġse lect
+Ġkill ed
+sy ch
+Ġgo es
+i j
+Ġc oll
+Ġimp act
+at ives
+ĠS er
+0 9
+ĠAug ust
+Ġb oy
+d e
+ĠD es
+Ġf elt
+U S
+Ġexpect ed
+Ġim age
+ĠM ark
+cc ording
+o ice
+E C
+ĠM ag
+en ed
+h old
+ĠP ost
+Ġpre vent
+N o
+Ġinvol ved
+Ġey es
+Ġquick ly
+A t
+un k
+Ġbeh av
+Ġ ur
+Ġl ed
+c ome
+e y
+Ġcand id
+Ġear lier
+Ġfoc us
+et y
+P ro
+led ge
+ix ed
+ill ed
+Ġpop ular
+A P
+Ġset t
+l ight
+Ġvar ious
+in ks
+Ġlevel s
+Ġro ad
+ell ig
+ab les
+he l
+itte e
+ĠG ener
+y pe
+Ġhe ard
+ic les
+Ġm is
+Ġus ers
+ĠS an
+Ġimpro ve
+Ġf ather
+Ġse arch
+The y
+v il
+Ġprof ess
+Ġkn ew
+Ġl oss
+Ġev ents
+6 5
+Ġb illion
+0 7
+0 2
+ĠNew s
+ĠA M
+Ġco ver
+w here
+ens ion
+Ġb ott
+Ġare as
+en ces
+op e
+ĠTw itter
+a el
+Ġget s
+ĠGo ogle
+Ġs n
+i ant
+Ġv ote
+Ġnear ly
+Ġinclud ed
+Ġrec ogn
+z z
+m m
+al ed
+Ġhappen ed
+0 4
+Ġh ot
+Ġwho se
+Ġc ivil
+Ġsu ff
+o es
+it iz
+ĠSy ri
+Ġresp ond
+Ġh on
+Ġfeat ures
+Ġeconom ic
+ĠApr il
+r im
+Ġtechn ology
+Ġo ption
+ag ing
+Ġpur ch
+R e
+Ġl at
+ch ie
+is l
+Ġrec omm
+u f
+Ġtr aining
+Ġeffect s
+Ġf ast
+Ġ201 0
+Ġocc ur
+Ġwebs ite
+Ġem ail
+Ġs ens
+e ch
+Ġo il
+Ġinf lu
+Ġcurrent ly
+ĠS ch
+ĠAd d
+Ġgo al
+Ġsc ient
+Ġcon v
+1 00
+em y
+Ġdec ided
+Ġtra vel
+Ġm ention
+L L
+0 3
+Ġe lection
+Ġph one
+Ġlook s
+Ġsit uation
+Ġc y
+Ġh or
+b ed
+ĠCour t
+a ily
+av es
+Ġqu ality
+ĠCom p
+w ise
+Ġt able
+Ġst aff
+ĠW ind
+et t
+Ġtri ed
+ide red
+Ġadd ition
+Ġb ox
+Ġl ack
+ar ily
+Ġw ide
+Ġm id
+Ġbo ard
+ys is
+Ġant i
+h a
+Ġd ig
+en ing
+Ġd ro
+C on
+6 8
+Ġsl ow
+b ased
+se qu
+Ġp ath
+E x
+ak er
+Ġwork ed
+Ġp en
+Ġeng ine
+Ġlook ed
+ĠSu per
+ĠS erv
+Ġvict im
+U n
+Ġproper ty
+Ġint rodu
+Ġexec ut
+ĠP M
+L e
+Ġcol or
+ĠM ore
+Ġ6 0
+Ġnet work
+Ġd ate
+c ul
+id ge
+Ġext ra
+3 1
+Ġs le
+6 7
+Ġw ond
+Ġreport s
+j ust
+ĠAust ral
+Ġcap ital
+Ġen s
+Ġcomm and
+Ġallow ed
+Ġpre p
+Ġca pt
+h ib
+Ġnum bers
+ch an
+Ġf air
+m p
+om s
+Ġre ach
+W ith
+t ain
+Ġbro ad
+Ġcou ple
+ec ause
+ly ing
+ĠF eb
+Ġsc reen
+Ġl ives
+Ġpri or
+ĠCong ress
+A r
+Ġappro ach
+Ġe mer
+ar ies
+ĠD is
+s erv
+ĠN e
+Ġbu ilt
+c ies
+Ġre pe
+Ġrul es
+for ce
+ĠP al
+Ġfin ancial
+Ġcons idered
+ĠCh ar
+n ces
+ĠI S
+Ġb rought
+Ġb i
+i ers
+ĠS im
+O P
+Ġproduct s
+Ġvis it
+Ġdoc ument
+Ġcon duct
+Ġcomplete ly
+in ing
+ĠCal if
+ib ly
+Ġwr itten
+ĠT V
+em ents
+Ġd raw
+O ne
+Ġpub lished
+Ġsec ret
+r ain
+he t
+ĠF acebook
+ond ay
+ĠU p
+Ġsex ual
+Ġth ous
+ĠP at
+Ġ ess
+Ġstand ard
+Ġar m
+g es
+ect ion
+Ġf ell
+Ġfore ign
+an i
+ĠFr iday
+Ġreg ular
+in ary
+Ġincre ased
+Ġus ually
+Ġdem on
+Ġd ark
+Ġadd itional
+ro l
+ĠO f
+Ġprodu ction
+! !
+und red
+Ġintern ational
+id ents
+ĠF ree
+rou p
+Ġr ace
+Ġm ach
+Ġh uge
+A ll
+le ar
+ove mber
+Ġto wn
+Ġatt ention
+ĠO ff
+y ond
+ĠThe n
+f ield
+Ġter ror
+ra z
+ĠB o
+Ġmeet ing
+ĠP ark
+Ġar rest
+Ġf ear
+Ġa w
+ĠV al
+or ing
+' ,
+Ġext reme
+ar r
+Ġwork ers
+A fter
+Ġ3 1
+n et
+am ent
+Ġdirect ly
+Ġpop ulation
+ub e
+ĠOct ober
+ĠI N
+ĠJan uary
+5 9
+ĠDav id
+Ġc ross
+ce mber
+ĠF irst
+Ġmess age
+ir it
+Ġn ation
+Ġp oll
+is ions
+Ġansw er
+n y
+is ode
+Ġcar ry
+ĠRuss ia
+Ġhe ar
+eng th
+ro y
+Ġn atural
+in ally
+Ġdo g
+m itted
+Ġtr ade
+Ġsub st
+Ġmult iple
+ĠAf ric
+Ġf ans
+Ġs ort
+Ġgl obal
+ic ation
+ĠW ed
+ar a
+Ġa chie
+Ġlangu age
+ve y
+Ġt al
+Ġnecess ary
+Ġdet ails
+Ġs en
+ĠS und
+ĠRe g
+ĠR ec
+0 6
+Ġs il
+ress ive
+Ġmed ical
+un ch
+orn ia
+Ġu nd
+f ort
+oc ks
+ĠM onday
+ues day
+c raft
+7 7
+ur t
+Ġ ver
+ĠH ill
+Ġrece ive
+Ġmor ning
+es tern
+Ġb ank
+Ġs at
+ir th
+ĠH igh
+Ġdev ice
+ĠTH E
+ĠCent er
+Ġsaf e
+Ġp le
+ĠCanad a
+Ġsystem s
+Ġass ist
+Ġsur v
+Ġb attle
+ĠS oc
+vert is
+S he
+Ġp aper
+Ġgrow th
+Ġc ast
+S c
+Ġpl ans
+ll ed
+Ġpart s
+Ġw all
+Ġmove ment
+Ġpract ice
+im ately
+Ġdis play
+Ġsomet imes
+om p
+ĠP aul
+ĠY es
+k ing
+5 8
+o ly
+Ġs on
+Ġav oid
+ok es
+ĠJ ew
+Ġto wards
+as c
+Ġ //
+ĠK ore
+Ġtalk ing
+Ġcor rect
+Ġsp ent
+ic ks
+i able
+e ared
+Ġter m
+Ġwant s
+om ing
+Ġ ut
+Ġdou b
+Ġfor ces
+Ġp lease
+6 9
+ĠN ovember
+at form
+ond on
+Ġon es
+Ġimmedi ately
+ĠRuss ian
+ĠM et
+Ġde g
+Ġparent s
+C H
+ĠAmeric ans
+al y
+ĠM od
+Ġsh own
+Ġcond itions
+Ġst uff
+Ġre b
+ĠY our
+Ġinclud es
+n own
+ĠS am
+Ġexper ien
+m ission
+ĠE ven
+augh t
+Ġannoun ced
+ĠRepublic an
+Ġdeter min
+Ġdescrib ed
+ĠCount y
+( )
+Ġdo or
+Ġchang ed
+Ġne igh
+ĠH ere
+Ġcle an
+Ġp an
+ĠDe cember
+ĠEurope an
+ir ing
+ap ter
+Ġcl ub
+ĠT uesday
+Ġp aid
+ĠN et
+Ġattack s
+Ġcharact ers
+Ġal one
+Ġdirect or
+d om
+Ġ3 5
+Ġl oad
+Ġr out
+ĠCalif ornia
+Ġfin ally
+Ġr ac
+Ġcont r
+Ġexact ly
+res h
+p ri
+ĠIs lam
+Ġn ature
+Ġcare er
+Ġlat est
+Ġcon vers
+ĠS l
+p ose
+ci ent
+ĠIn c
+iv ity
+8 8
+ĠA tt
+ĠM or
+nes day
+Ġwe ight
+k en
+Ġnot e
+Ġteam s
+Ġ \
+air s
+ĠG reen
+Ġh undred
+on ent
+Ġstre ng
+Ġcons ist
+ic ated
+Ġreg ul
+Ġl ic
+ast ic
+Ġt en
+urs day
+ellig ence
+ous ly
+ĠU K
+B I
+Ġcost s
+Ġind epend
+ĠA P
+Ġnorm al
+Ġh om
+Ġob vious
+Ġs we
+Ġst ar
+Ġread y
+ac her
+Ġimp lement
+g est
+Ġs ong
+ĠG et
+ĠL ab
+Ġinterest ing
+us ing
+Ġg iving
+ĠSund ay
+Ġet c
+Ġm iddle
+Ġrem ember
+r ight
+os ition
+ut ions
+Ġm ax
+4 6
+Ġyour self
+Ġdem and
+Ġtreat ment
+Ġd anger
+ĠC ons
+Ġgu y
+ĠBrit ish
+Ġphys ical
+Ġrel ated
+Ġrem ain
+Ġcould n
+Ġref er
+Ġc itiz
+b ox
+EN T
+bo ard
+Ġin n
+I G
+er o
+ĠSt reet
+osp ital
+ren ch
+cher s
+Ġst ra
+O L
+ag er
+ĠA N
+Ġeas ily
+I A
+en ge
+in y
+Ġcl os
+ock ed
+Ġus es
+ĠC oun
+I m
+u ild
+? ?
+m ore
+Ġan g
+Ġwr ite
+ol ute
+5 7
+Ġlead er
+Ġread ing
+< /
+Ġaut om
+est s
+4 3
+Ġleg isl
+ĠG old
+Ġdesign ed
+ĠS T
+ĠLe g
+a res
+Ġbe aut
+ĠT ex
+Ġappear s
+Ġstru gg
+ĠR om
+Ġ 00
+Ġcho ice
+Ġparticular ly
+ĠF rom
+op er
+ĠL ondon
+ann ed
+Ġallow s
+ob ile
+Ġdiffere nce
+âĢ ¢
+ĠV iew
+ĠWed nesday
+Ġal though
+Ġrel ative
+Ġapplic ation
+ate ver
+Ġare n
+Ġmy self
+Ġim ag
+Ġdis e
+Ġsoc iety
+Ġfre qu
+ĠEng lish
+Ġpo or
+ĠD ay
+Ġwrit ing
+Ġse ven
+Ġstart ing
+Ġb ud
+Ġpr int
+ĠTr ans
+uf act
+ĠSt ud
+n ew
+Ġcr im
+Ġg ives
+Ġco ol
+a e
+i ance
+ĠGener al
+Ġthink ing
+Ġsa ve
+Ġlim ited
+ĠPart y
+Ġmean ing
+p en
+ow ers
+ĠJ ack
+E M
+Ġn ice
+ru pt
+Ġg as
+Ġe ight
+Ġfe et
+Ġeff ort
+Ġ ign
+ic it
+B l
+co in
+Ġop in
+Ġbr ain
+Wh ile
+he st
+ĠTh ursday
+Ġwould n
+augh ter
+Ġtou ch
+le ments
+Ġstud ies
+Ġcent er
+c ont
+or ge
+Ġcomput er
+Ġinvestig ation
+P l
+or ks
+Ġ200 8
+Ġincre asing
+Ġst ore
+Ġcom ments
+Ġb al
+m en
+Ġdo ll
+Ġl iber
+Ġw ife
+Ġlaw s
+atur day
+it ness
+Ġmod ern
+ĠS k
+Ġadminist ration
+Ġopportun ity
+Ġs al
+Ġpower ful
+M y
+Ġclaim s
+ĠEar th
+ord s
+Ġt itle
+Ġes c
+n ame
+N ot
+om en
+Ġbe yond
+Ġc amer
+Ġse ll
+it ute
+ear ch
+Ġapp l
+im ent
+4 2
+ĠAr t
+Ġun f
+Ġviol ence
+ur g
+ĠE ast
+Ġcomp ared
+Ġopt ions
+Ġthrough out
+Ġv s
+ig r
+. [
+ac hes
+7 8
+Ġfil es
+F L
+E L
+ar ian
+ĠJ ames
+ĠA ir
+an ch
+Ġdet ail
+Ġpie ce
+P S
+Ġn amed
+Ġeduc ation
+Ġdri ve
+Ġitem s
+Ġstud ent
+ic ed
+: :
+ic o
+Ġth row
+Ġsc ene
+Ġcomple x
+Ġ200 9
+Ġpre c
+ĠB re
+7 9
+Ġcon cept
+Ġstat us
+am ing
+Ġd ied
+Ġknow ledge
+Ġbegin ning
+O D
+ru ary
+Ġcertain ly
+Ġgu ys
+Ġsl ight
+in n
+ound s
+Ġf ine
+Ġf at
+ic ations
+Ġper haps
+ĠA nt
+Ġinc ome
+Ġhtt ps
+Ġmajor ity
+port s
+st on
+Ġgreat er
+Ġfe ed
+ent ially
+Ġsaf ety
+Ġun ique
+and om
+Ġg one
+Ġshow ed
+Ġhist or
+Ġcoun ter
+i us
+id a
+Ġlead ing
+i pe
+Ġs end
+ĠDon ald
+er ve
+Ġdef ense
+ines e
+Ġy es
+ĠF ire
+ĠMus lim
+ra q
+Ġcontin ued
+os h
+Ġprov ides
+Ġpr ison
+ĠP re
+Ġhapp y
+Ġeconom y
+Ġtr ust
+ag s
+ĠG ame
+Ġweap ons
+um an
+ĠC le
+it ation
+Ġanal ysis
+ĠT imes
+Ġsc ience
+- >
+Ġfig ure
+Ġdis app
+ent y
+Ġsoft ware
+Ġu lt
+Ġoffic ers
+N ew
+I s
+Ġrem ains
+ĠInd ia
+Ġp sych
+ri ef
+Ġc at
+es c
+Ġob serv
+Ġst age
+ĠD ark
+Ġent er
+ch ange
+Ġpass ed
+Ġdes pite
+ĠO ut
+Ġmov ie
+r s
+Ġv oice
+m ine
+ĠPl ay
+Ġto ward
+ĠT er
+Ġreg ion
+Ġval ues
+or ters
+Ġm ount
+Ġoffic er
+ĠO ther
+b an
+Ġh ous
+w ood
+ro om
+I V
+ĠS un
+se e
+ĠO ver
+ro g
+9 0
+Ġl ay
+ĠT ur
+a wn
+Ġpress ure
+ĠS ub
+Ġbook s
+ed om
+ĠS and
+A A
+ag o
+Ġre asons
+f ord
+Ġactiv ity
+U T
+N ow
+ĠSen ate
+ce ll
+n ight
+Ġcall s
+in ter
+Ġlet ter
+ĠR ob
+ĠJ e
+Ġcho ose
+ĠL aw
+G et
+B e
+Ġro b
+Ġtyp es
+Ġpl atform
+Ġqu arter
+R A
+ĠT ime
+Ġmay be
+ĠC r
+9 5
+p re
+Ġmov ing
+Ġl if
+Ġgo ld
+Ġs om
+Ġpat ients
+Ġtr uth
+ĠK e
+ur ance
+ant ly
+m ar
+Ġchar ge
+ĠG reat
+Ġce le
+---------------- ----------------
+Ġro ck
+ro id
+an cy
+Ġcred it
+a ud
+B y
+ĠE very
+Ġmov ed
+ing er
+rib ution
+Ġn ames
+Ġstra ight
+ĠHe alth
+ĠW ell
+Ġfe ature
+Ġr ule
+Ġsc he
+in ated
+ĠMich ael
+ber g
+4 1
+il ed
+b and
+Ġcl ick
+ĠAng el
+on ents
+Â Ń
+ĠI raq
+ĠS aturday
+Ġa ware
+p art
+Ġpat tern
+O W
+ĠL et
+Ġgr ad
+ign ed
+Ġassoci ated
+Ġst yle
+n o
+i ation
+a ith
+il ies
+Ġst ories
+ur ation
+Ġindividual s
+ĠâĢ ¦
+m iss
+ĠAss oci
+ish ing
+ab y
+Ġsum mer
+ĠB en
+Ġ3 2
+Ġar ch
+ut y
+ĠTex as
+h ol
+Ġfull y
+Ġm ill
+Ġfollow ed
+ĠB ill
+ĠInd ian
+ĠSec ret
+ĠB el
+ĠFeb ruary
+Ġjob s
+Ġseem ed
+ĠGo vern
+i pped
+Ġreal ity
+Ġl ines
+Ġp ark
+Ġmeas ure
+ĠO ur
+I M
+Ġbro ther
+Ġgrow ing
+Ġb an
+Ġest im
+Ġc ry
+ĠS chool
+Ġme chan
+ĠO F
+ĠWind ows
+Ġr ates
+ĠO h
+Ġpos itive
+Ġcult ure
+ist ics
+ic a
+Ġh ar
+y a
+ite ly
+i pp
+Ġm ap
+en cies
+ĠWill iam
+I I
+ak ers
+5 6
+ĠM art
+ĠR em
+Ġal tern
+it ude
+Ġco ach
+row d
+D on
+Ġk ids
+Ġj ournal
+Ġcor por
+Ġf alse
+Ġwe b
+Ġsle ep
+Ġcont ain
+Ġst o
+Ġb ed
+iver se
+ĠR ich
+ĠCh inese
+Ġp un
+Ġme ant
+k nown
+Ġnot ice
+Ġfavor ite
+a ven
+Ġcond ition
+Ġpur pose
+) )
+Ġorgan ization
+Ġchall eng
+Ġman ufact
+Ġsus p
+ĠA c
+Ġcrit ic
+un es
+uc lear
+Ġm er
+vent ion
+Ġ8 0
+Ġm ist
+ĠU s
+ĠT or
+htt p
+ol f
+Ġlarg er
+Ġadv ant
+Ġrese ar
+Ġact ions
+m l
+Ġke pt
+Ġa im
+, '
+c ol
+Ġbenef its
+if ying
+Ġact ual
+ĠIntern ational
+Ġveh icle
+Ġch ief
+Ġeff orts
+ĠLe ague
+ĠM ost
+Ġwa it
+Ġad ult
+Ġover all
+Ġspe ech
+Ġhigh ly
+Ġfem ale
+Ġer ror
+Ġeffect ive
+5 4
+Ġenc our
+w ell
+Ġfail ed
+Ġcons erv
+Ġprogram s
+Ġt rou
+Ġa head
+5 00
+vertis ement
+I P
+ĠF ound
+p ir
+Ġ %
+Ġcr ime
+and er
+Ġloc ation
+ĠI ran
+Ġbehav ior
+az ing
+Ġr are
+Ġem b
+Ġca used
+Ġsh ip
+Ġact ive
+Ġcont ribut
+Ġg reen
+Ġac qu
+Ġref lect
+ven ue
+Ġf irm
+Ġb irth
+] .
+Ġclear ly
+Ġem ot
+Ġag ency
+ri age
+Ġmem ory
+9 8
+S A
+ĠSe e
+ac ing
+C C
+Ġbig gest
+Ġr ap
+Ġbas ic
+Ġb and
+e at
+Ġsus pect
+ĠM ac
+Ġ9 0
+m ark
+ist an
+Ġsp read
+am s
+k i
+as y
+ra v
+ĠR ober
+Ġdemon str
+r ated
+Ġabs olute
+Ġpl aces
+Ġim pl
+ibr ary
+Ġc ards
+Ġdest roy
+Ġv irt
+ve re
+Ġapp eared
+y an
+p oint
+Ġbe g
+Ġtem per
+s pe
+ant ed
+ear s
+ĠD irect
+Ġl ength
+Ġbl og
+am b
+Ġint eg
+Ġres ources
+ac c
+if ul
+Ġsp ot
+Ġfor ced
+Ġthous ands
+ĠMin ister
+Ġqu al
+ĠF rench
+at ically
+Ġgener ally
+Ġdr ink
+Ġth us
+I L
+od es
+Ġappro pri
+ĠRe ad
+Ġwh om
+Ġey e
+Ġcol lege
+Ġ4 5
+ire ction
+Ġens ure
+Ġapp arent
+id ers
+Ġrelig ious
+Ġmin or
+ol ic
+Ġt ro
+ĠWh y
+rib ute
+m et
+Ġprim ary
+Ġdevelop ed
+Ġpe ace
+Ġsk in
+st e
+av a
+Ġbl ue
+Ġfam ilies
+Ġ ir
+Ġapp ly
+Ġin form
+ĠSm ith
+C T
+i i
+Ġlim it
+Ġres ist
+........ ........
+um n
+Ġconf lic
+Ġtw e
+ud d
+ĠT om
+Ġl iter
+qu e
+b on
+Ġha ir
+Ġevent ually
+Ġp us
+Ġhelp ed
+Ġag g
+or ney
+ĠApp le
+Ġf it
+ĠS ur
+Ġpre m
+Ġs ales
+Ġsecond s
+Ġstreng th
+Ġfeel ing
+¿ ½
+Ġt our
+Ġknow s
+o om
+Ġex erc
+Ġsom ew
+ï ¿½
+> >
+Ġsp okes
+Ġide as
+Ġreg ist
+so ft
+ĠD el
+ĠP C
+Ġpro pos
+Ġlaun ch
+Ġbott om
+T H
+ĠP lease
+v est
+it z
+ĠIn ter
+Ġsc ript
+Ġr at
+ar ning
+Ġ il
+ĠJ er
+ĠA re
+Ġwh atever
+ok en
+ci ence
+Ġmod e
+Ġag ree
+Ġs ources
+Ġinit ial
+Ġrest rict
+Ġwond er
+us ion
+## ##
+ĠS il
+vil le
+Ġb urn
+t w
+as ion
+ĠÂ £
+Ġn or
+u ing
+Ġre ached
+Ġs un
+Ġc ateg
+ig ration
+Ġc ook
+Ġprom ot
+Ġm ale
+Ġcl imate
+Ġf ix
+Ġalleg ed
+U R
+all ed
+Ġim ages
+C ont
+ot a
+Ġschool s
+i os
+Ġd rop
+Ġst ream
+ĠM o
+Ġprevious ly
+al ing
+Ġp et
+Ġdou ble
+Ġ( @
+ann el
+Ġdef ault
+t ies
+Ġr ank
+ĠD ec
+ĠCoun cil
+Ġweap on
+Ġst ock
+Ġanal y
+ĠSt r
+Ġpict ure
+ĠPol ice
+f erence
+Ġcent ury
+Ġcitiz ens
+Ġon to
+Ġexp and
+Ġhe ro
+ĠS ol
+Ġw ild
+Ġupd ate
+Ġcustom ers
+r ont
+d ef
+Ġl ik
+Ġcrim inal
+ĠChrist ian
+S P
+7 6
+Ġle aving
+Ġother wise
+ĠD ist
+Ġbas is
+5 2
+5 3
+ic ip
+ĠB er
+Ġrecomm end
+Ġfl oor
+Ġc rowd
+ol es
+Ġ7 0
+Ġcent ral
+ĠE v
+Ġd ream
+Ġdown load
+Ġconf ir
+ĠTh om
+Ġwind ow
+Ġhapp ens
+Ġun it
+Ġt end
+Ġs pl
+Ġbec omes
+Ġfight ing
+Ġpred ict
+ĠP ress
+ĠP ower
+Ġhe avy
+ak ed
+Ġf an
+or ter
+ate gy
+B A
+iz es
+Ġsp end
+H ere
+Ġ200 7
+Ġad op
+ĠH am
+Ġfoot ball
+ĠP ort
+od ay
+5 1
+amp ions
+Ġtrans fer
+h t
+Ġ3 8
+ter m
+ac ity
+Ġb ur
+] ,
+tern al
+r ig
+b ut
+Ġthere fore
+ĠB ecause
+res p
+re y
+Ġm ission
+S ome
+Ġnot ed
+Ġass um
+Ġdise ase
+Ġed it
+Ġprog ress
+r d
+ĠB rown
+oc al
+Ġadd ing
+Ġra ised
+ĠAn y
+Ġt ick
+Ġsee ing
+ĠPe ople
+Ġagre ement
+Ġser ver
+Ġw at
+Ġdeb ate
+Ġsupp osed
+il ing
+Ġlarg est
+Ġsuccess ful
+ĠP ri
+ĠDemocr atic
+Ġj ump
+ĠSyri a
+Ġown ers
+Ġoff ers
+Ġshoot ing
+Ġeff ic
+se y
+Ġha ven
+ver se
+te red
+ĠL ight
+im al
+ĠB ig
+Ġdef end
+Ġbe at
+Ġrecord s
+% )
+Ġsc en
+Ġemploy ees
+Ġdev ices
+he m
+Ġcom mer
+ĠM ex
+Ġbenef it
+ĠPro f
+Ġil leg
+Ġsur face
+ĠAl so
+Ġh arm
+ing ly
+w ide
+ĠA lex
+Ġsh ut
+ĠC ur
+Ġl ose
+p m
+Ġchall enge
+se mb
+Ġst ation
+Ġint elligence
+Ġacc ur
+ĠFl or
+Ġrequ ires
+ĠM al
+b um
+Ġh ospital
+Ġsp irit
+Ġoff ered
+Ġprodu ce
+ĠComm un
+Ġcreat ing
+Ġcr is
+s pect
+Ġend ed
+Ġd aily
+Ġvot ers
+land s
+i as
+i h
+on a
+Ġsm art
+ĠOff ice
+ĠL ord
+ri al
+ĠIntern et
+Ġcirc um
+Ġextreme ly
+' .
+Ġopin ion
+ĠM il
+Ġg ain
+B S
+ĠF in
+y p
+Ġuse ful
+Ġbud get
+Ġcom fort
+is f
+Ġback ground
+el ine
+Ġep isode
+Ġen emy
+Ġtri al
+Ġestab lish
+d ate
+ĠC ap
+Ġcontin ues
+Ġshow ing
+ĠUn ion
+w ith
+Ġpost ed
+ĠSy stem
+Ġe at
+ri an
+Ġr ise
+ĠGerman y
+il s
+Ġsign ed
+Ġv ill
+Ġgr and
+m or
+ĠEng land
+Ġproject s
+um ber
+Ġconf erence
+z a
+Ġrespons ible
+ĠAr ab
+Ġlearn ed
+âĢĶ âĢĶ
+i pping
+ĠGe orge
+O C
+Ġreturn ed
+ĠAustral ia
+Ġb rief
+Q u
+Ġbr and
+ill ing
+ab led
+Ġhig hest
+Ġtr ain
+ĠComm ission
+wh ile
+Ġn om
+cept ion
+Ġm ut
+ĠBl ue
+Ġinc ident
+v ant
+8 6
+ĠI D
+Ġn uclear
+7 4
+ĠL ike
+ĠR E
+ĠM icro
+l i
+m ail
+Ġcharg es
+8 9
+Ġad just
+ad o
+Ġear th
+N A
+Ġpr ices
+P A
+Ġd raft
+Ġrun s
+Ġcandid ate
+ens es
+Ġmanag ement
+ĠPh il
+ĠM iss
+Ġte ach
+g ram
+Ġunderstand ing
+a it
+ic ago
+A dd
+ĠE p
+sec ut
+Ġsepar ate
+Ġinst ance
+Ġe th
+Ġun less
+**** ****
+ĠF ore
+in ate
+Ġoper ations
+S p
+Ġf aith
+g ar
+ĠCh urch
+ron ic
+Ġconf ig
+os ure
+Ġactiv ities
+Ġtrad itional
+Ġ3 6
+Ġd irection
+Ġmach ine
+Ġsur round
+Ġp ush
+un ction
+ĠE U
+Ġeas ier
+Ġarg ument
+G B
+Ġm icro
+Ġsp ending
+iz ations
+Ġthe ory
+ad ow
+Ġcall ing
+ĠL ast
+Ġd er
+Ġinflu ence
+Ġcomm it
+Ġph oto
+Ġun c
+ist ry
+g n
+ast e
+ack s
+Ġdis p
+ad y
+d o
+ĠG ood
+Ġ `
+Ġw ish
+Ġreve aled
+Âł Âł
+l ig
+Ġen force
+ĠComm ittee
+Ġche m
+Ġmil es
+Ġinterest ed
+Ġsol ution
+ic y
+in ct
+Ġ- >
+ĠD et
+Ġrem oved
+Ġcomp ar
+e ah
+Ġpl ant
+ĠS ince
+Ġachie ve
+Ġadvant age
+Ġslight ly
+b ing
+Ġpl aced
+u nder
+201 5
+ĠM ad
+Ġt im
+os es
+Ġc ru
+ĠR ock
+Ġmost ly
+Ġneg ative
+Ġset ting
+Ġprodu ced
+Ġm ur
+Ġconnect ion
+ĠM er
+Ġdri ver
+Ġexecut ive
+Ġass ault
+Ġb orn
+ĠV er
+t ained
+Ġstruct ure
+Ġredu ce
+Ġdec ades
+Ġd ed
+u ke
+ĠM any
+idd en
+Ġle ague
+S e
+Ġjo in
+Ġdis co
+Ġd ie
+c ks
+act ions
+Ġass ess
+ag n
+Ġgo als
+our s
+I R
+Ġsen ior
+ill er
+m od
+ip ment
+oc ol
+u y
+ĠQ ue
+Ġpart ies
+ir gin
+Ġle arning
+it able
+Ġstre et
+Ġcamer a
+A pp
+Ġsk ills
+b re
+c ious
+Ġcele br
+ĠFr anc
+Ġexist ing
+Ġwill ing
+l or
+Ġ id
+ĠSp ace
+Ġcrit ical
+ĠL a
+ortun ately
+Ġser ve
+Ġc old
+Ġspec ies
+T S
+Ġanim als
+ĠB ay
+Ġold er
+ĠU nder
+est ic
+ĠT re
+Ġte acher
+Ġpre fer
+v is
+Ġth read
+ĠM att
+Ġmanag er
+ãĥ »
+Ġprofess ional
+ĠV ol
+Ġnot es
+The se
+ul a
+Ġf resh
+ent ed
+u zz
+ed y
+clus ion
+ĠR el
+Ġdoub t
+E O
+Ġopen ed
+ĠB it
+Ad vertisement
+Ġgu ess
+ĠU N
+Ġse qu
+Ġexpl ain
+ott en
+Ġatt ract
+ak s
+Ġstr ing
+Ġcont ext
+oss ible
+ĠRepublic ans
+Ġsol id
+Ġc ities
+Ġask ing
+Ġr andom
+u ps
+ur ies
+ar ant
+dd en
+g l
+ĠFlor ida
+Ġdep end
+ĠSc ott
+Ġ3 3
+Ġi T
+ic on
+Ġmention ed
+Ġ2 000
+Ġclaim ed
+Ġdefin itely
+ul f
+Ġc ore
+Ġopen ing
+ĠCon st
+wh ich
+ĠT ra
+A G
+7 2
+Ġbelie ved
+ad a
+Ġ4 8
+ĠSec urity
+yr ight
+ĠP et
+ĠL ou
+Ġhold ing
+======== ========
+Ġ ice
+Ġb row
+Ġauthor ities
+h ost
+w ord
+Ġsc ore
+ĠD iv
+Ġcell s
+Ġtrans l
+Ġneigh bor
+Ġrem ove
+u ct
+Ġdist rict
+ĠA ccording
+Ġwor se
+Ġconcern s
+Ġpresident ial
+Ġpolic ies
+ĠH all
+7 3
+Ġh us
+A Y
+Ġ200 6
+ĠJ ud
+Ġindepend ent
+ĠJust ice
+ili ar
+pr int
+igh ter
+Ġprotect ion
+z en
+Ġsu dden
+h ouse
+ĠJ es
+P R
+ĠIn f
+Ġb ul
+Ġ _
+ĠServ ice
+ĠP R
+Ġstr ategy
+ff ect
+Ġgirl s
+Ġmiss ing
+oy al
+ĠTe am
+ul ated
+Ġd at
+Ġpolit ics
+ab or
+A ccording
+Ġspe ll
+Ġg raph
+ort hern
+T C
+A b
+Ġlab or
+is her
+Ġk ick
+ĠiT unes
+Ġstep s
+pos es
+Ġsmall er
+E n
+ber t
+Ġro ll
+Ġresear chers
+Ġcl osed
+Ġtrans port
+Ġlaw y
+________ ________
+ĠCh icago
+Ġas pect
+Ġn one
+Ġmar riage
+9 6
+Ġe lements
+ĠF re
+ĠS al
+Ġd ram
+F C
+t op
+e qu
+Ġhe aring
+Ġsupport ed
+Ġtest ing
+co hol
+Ġmass ive
+Ġst ick
+Ġgu ard
+is co
+ph one
+F rom
+How ever
+Ġb order
+Ġcop y
+ograph y
+l ist
+7 1
+Ġown er
+cl ass
+ru it
+r ate
+ĠO nce
+Ġdig ital
+Ġt ask
+ER S
+Ġinc red
+t es
++ +
+ĠFr ance
+Ġb reat
+ow l
+Ġiss ued
+ĠW estern
+Ġdet ect
+Ġpart ners
+Ġsh ared
+ĠC all
+Ġcan cer
+ac he
+rib e
+Ġexpl ained
+Ġhe at
+{ "
+Ġinvest ment
+ĠB ook
+Ġw ood
+Ġtool s
+ĠAl though
+Ġbelie f
+Ġcris is
+Ġg e
+ĠM P
+Ġoper ation
+ty pe
+~ ~
+g a
+Ġcont ains
+ant a
+Ġexp ress
+ĠG roup
+ĠJ ournal
+k a
+Ġam b
+ĠUS A
+Ġfind ing
+Ġfund ing
+h ow
+Ġestab lished
+ide os
+Ġdeg ree
+Ġdanger ous
+ang ing
+Ġfre edom
+pp ort
+out hern
+Ġch urch
+Ġc atch
+ĠTw o
+Ġpres ence
+ĠGu ard
+U p
+Ġauthor ity
+ĠPro ject
+Ġbut ton
+Ġcon sequ
+Ġval id
+Ġwe ak
+Ġstart s
+Ġref erence
+ĠM em
+" )
+U N
+or age
+ĠO pen
+Ġcol lection
+y m
+g ency
+Ġbeaut iful
+ro s
+Ġtell s
+Ġwa iting
+n el
+Ġprov iding
+ĠDemocr ats
+Ġd aughter
+Ġm aster
+Ġpur poses
+ĠJapan ese
+Ġequ al
+Ġturn s
+Ġdoc uments
+Ġwatch ing
+R es
+Ġr an
+201 4
+Ġre ject
+ĠKore a
+Ġvictim s
+Le vel
+ere nces
+Ġw itness
+Ġ3 4
+Ġre form
+com ing
+Ġocc up
+Ġc aught
+Ġtra ffic
+ad ing
+Ġmod els
+ar io
+Ġserv ed
+Ġb atter
+u ate
+ĠSecret ary
+Ġagre ed
+Ġtr uly
+yn am
+ĠR et
+Ġun its
+ĠRes earch
+h and
+az ine
+ĠM ike
+Ġvar iety
+ot al
+Ġam azing

megatron-deepspeed_dtk22.10/tests/data/gpt2/gpt2-tiny-vocab.json

+{"!":0,"\"":1,"#":2,"$":3,"%":4,"&":5,"'":6,"(":7,")":8,"*":9,"+":10,",":11,"-":12,".":13,"/":14,"0":15,"1":16,"2":17,"3":18,"4":19,"5":20,"6":21,"7":22,"8":23,"9":24,":":25,";":26,"<":27,"=":28,">":29,"?":30,"@":31,"A":32,"B":33,"C":34,"D":35,"E":36,"F":37,"G":38,"H":39,"I":40,"J":41,"K":42,"L":43,"M":44,"N":45,"O":46,"P":47,"Q":48,"R":49,"S":50,"T":51,"U":52,"V":53,"W":54,"X":55,"Y":56,"Z":57,"[":58,"\\":59,"]":60,"^":61,"_":62,"`":63,"a":64,"b":65,"c":66,"d":67,"e":68,"f":69,"g":70,"h":71,"i":72,"j":73,"k":74,"l":75,"m":76,"n":77,"o":78,"p":79,"q":80,"r":81,"s":82,"t":83,"u":84,"v":85,"w":86,"x":87,"y":88,"z":89,"{":90,"|":91,"}":92,"~":93,"¡":94,"¢":95,"£":96,"¤":97,"¥":98,"¦":99,"§":100,"¨":101,"©":102,"ª":103,"«":104,"¬":105,"®":106,"¯":107,"°":108,"±":109,"²":110,"³":111,"´":112,"µ":113,"¶":114,"·":115,"¸":116,"¹":117,"º":118,"»":119,"¼":120,"½":121,"¾":122,"¿":123,"À":124,"Á":125,"Â":126,"Ã":127,"Ä":128,"Å":129,"Æ":130,"Ç":131,"È":132,"É":133,"Ê":134,"Ë":135,"Ì":136,"Í":137,"Î":138,"Ï":139,"Ð":140,"Ñ":141,"Ò":142,"Ó":143,"Ô":144,"Õ":145,"Ö":146,"×":147,"Ø":148,"Ù":149,"Ú":150,"Û":151,"Ü":152,"Ý":153,"Þ":154,"ß":155,"à":156,"á":157,"â":158,"ã":159,"ä":160,"å":161,"æ":162,"ç":163,"è":164,"é":165,"ê":166,"ë":167,"ì":168,"í":169,"î":170,"ï":171,"ð":172,"ñ":173,"ò":174,"ó":175,"ô":176,"õ":177,"ö":178,"÷":179,"ø":180,"ù":181,"ú":182,"û":183,"ü":184,"ý":185,"þ":186,"ÿ":187,"Ā":188,"ā":189,"Ă":190,"ă":191,"Ą":192,"ą":193,"Ć":194,"ć":195,"Ĉ":196,"ĉ":197,"Ċ":198,"ċ":199,"Č":200,"č":201,"Ď":202,"ď":203,"Đ":204,"đ":205,"Ē":206,"ē":207,"Ĕ":208,"ĕ":209,"Ė":210,"ė":211,"Ę":212,"ę":213,"Ě":214,"ě":215,"Ĝ":216,"ĝ":217,"Ğ":218,"ğ":219,"Ġ":220,"ġ":221,"Ģ":222,"ģ":223,"Ĥ":224,"ĥ":225,"Ħ":226,"ħ":227,"Ĩ":228,"ĩ":229,"Ī":230,"ī":231,"Ĭ":232,"ĭ":233,"Į":234,"į":235,"İ":236,"ı":237,"IJ":238,"ij":239,"Ĵ":240,"ĵ":241,"Ķ":242,"ķ":243,"ĸ":244,"Ĺ":245,"ĺ":246,"Ļ":247,"ļ":248,"Ľ":249,"ľ":250,"Ŀ":251,"ŀ":252,"Ł":253,"ł":254,"Ń":255,"Ġt":256,"Ġa":257,"he":258,"in":259,"re":260,"on":261,"Ġthe":262,"er":263,"Ġs":264,"at":265,"Ġw":266,"Ġo":267,"en":268,"Ġc":269,"it":270,"is":271,"an":272,"or":273,"es":274,"Ġb":275,"ed":276,"Ġf":277,"ing":278,"Ġp":279,"ou":280,"Ġan":281,"al":282,"ar":283,"Ġto":284,"Ġm":285,"Ġof":286,"Ġin":287,"Ġd":288,"Ġh":289,"Ġand":290,"ic":291,"as":292,"le":293,"Ġth":294,"ion":295,"om":296,"ll":297,"ent":298,"Ġn":299,"Ġl":300,"st":301,"Ġre":302,"ve":303,"Ġe":304,"ro":305,"ly":306,"Ġbe":307,"Ġg":308,"ĠT":309,"ct":310,"ĠS":311,"id":312,"ot":313,"ĠI":314,"ut":315,"et":316,"ĠA":317,"Ġis":318,"Ġon":319,"im":320,"am":321,"ow":322,"ay":323,"ad":324,"se":325,"Ġthat":326,"ĠC":327,"ig":328,"Ġfor":329,"ac":330,"Ġy":331,"ver":332,"ur":333,"Ġu":334,"ld":335,"Ġst":336,"ĠM":337,"'s":338,"Ġhe":339,"Ġit":340,"ation":341,"ith":342,"ir":343,"ce":344,"Ġyou":345,"il":346,"ĠB":347,"Ġwh":348,"ol":349,"ĠP":350,"Ġwith":351,"Ġ1":352,"ter":353,"ch":354,"Ġas":355,"Ġwe":356,"Ġ(":357,"nd":358,"ill":359,"ĠD":360,"if":361,"Ġ2":362,"ag":363,"ers":364,"ke":365,"Ġ\"":366,"ĠH":367,"em":368,"Ġcon":369,"ĠW":370,"ĠR":371,"her":372,"Ġwas":373,"Ġr":374,"od":375,"ĠF":376,"ul":377,"ate":378,"Ġat":379,"ri":380,"pp":381,"ore":382,"ĠThe":383,"Ġse":384,"us":385,"Ġpro":386,"Ġha":387,"um":388,"Ġare":389,"Ġde":390,"ain":391,"and":392,"Ġor":393,"igh":394,"est":395,"ist":396,"ab":397,"rom":398,"ĠN":399,"th":400,"Ġcom":401,"ĠG":402,"un":403,"op":404,"00":405,"ĠL":406,"Ġnot":407,"ess":408,"Ġex":409,"Ġv":410,"res":411,"ĠE":412,"ew":413,"ity":414,"ant":415,"Ġby":416,"el":417,"os":418,"ort":419,"oc":420,"qu":421,"Ġfrom":422,"Ġhave":423,"Ġsu":424,"ive":425,"ould":426,"Ġsh":427,"Ġthis":428,"nt":429,"ra":430,"pe":431,"ight":432,"art":433,"ment":434,"Ġal":435,"ust":436,"end":437,"--":438,"all":439,"ĠO":440,"ack":441,"Ġch":442,"Ġle":443,"ies":444,"red":445,"ard":446,"âĢ":447,"out":448,"ĠJ":449,"Ġab":450,"ear":451,"iv":452,"ally":453,"our":454,"ost":455,"gh":456,"pt":457,"Ġpl":458,"ast":459,"Ġcan":460,"ak":461,"ome":462,"ud":463,"The":464,"Ġhis":465,"Ġdo":466,"Ġgo":467,"Ġhas":468,"ge":469,"'t":470,"ĠU":471,"rou":472,"Ġsa":473,"Ġj":474,"Ġbut":475,"Ġwor":476,"Ġall":477,"ect":478,"Ġk":479,"ame":480,"Ġwill":481,"ok":482,"Ġwhe":483,"Ġthey":484,"ide":485,"01":486,"ff":487,"ich":488,"pl":489,"ther":490,"Ġtr":491,"..":492,"Ġint":493,"ie":494,"ure":495,"age":496,"Ġne":497,"ial":498,"ap":499,"ine":500,"ice":501,"Ġme":502,"Ġout":503,"ans":504,"one":505,"ong":506,"ions":507,"Ġwho":508,"ĠK":509,"Ġup":510,"Ġtheir":511,"Ġad":512,"Ġ3":513,"Ġus":514,"ated":515,"ous":516,"Ġmore":517,"ue":518,"og":519,"ĠSt":520,"ind":521,"ike":522,"Ġso":523,"ime":524,"per":525,".\"":526,"ber":527,"iz":528,"act":529,"Ġone":530,"Ġsaid":531,"Ġ-":532,"are":533,"Ġyour":534,"cc":535,"ĠTh":536,"Ġcl":537,"ep":538,"ake":539,"able":540,"ip":541,"Ġcont":542,"Ġwhich":543,"ia":544,"Ġim":545,"Ġabout":546,"Ġwere":547,"very":548,"ub":549,"Ġhad":550,"Ġen":551,"Ġcomp":552,",\"":553,"ĠIn":554,"Ġun":555,"Ġag":556,"ire":557,"ace":558,"au":559,"ary":560,"Ġwould":561,"ass":562,"ry":563,"ĠâĢ":564,"cl":565,"ook":566,"ere":567,"so":568,"ĠV":569,"ign":570,"ib":571,"Ġoff":572,"Ġte":573,"ven":574,"ĠY":575,"ile":576,"ose":577,"ite":578,"orm":579,"Ġ201":580,"Ġres":581,"Ġman":582,"Ġper":583,"Ġother":584,"ord":585,"ult":586,"Ġbeen":587,"Ġlike":588,"ase":589,"ance":590,"ks":591,"ays":592,"own":593,"ence":594,"Ġdis":595,"ction":596,"Ġany":597,"Ġapp":598,"Ġsp":599,"int":600,"ress":601,"ations":602,"ail":603,"Ġ4":604,"ical":605,"Ġthem":606,"Ġher":607,"ount":608,"ĠCh":609,"Ġar":610,"Ġif":611,"Ġthere":612,"Ġpe":613,"Ġyear":614,"av":615,"Ġmy":616,"Ġsome":617,"Ġwhen":618,"ough":619,"ach":620,"Ġthan":621,"ru":622,"ond":623,"ick":624,"Ġover":625,"vel":626,"Ġqu":627,"ĊĊ":628,"Ġsc":629,"reat":630,"ree":631,"ĠIt":632,"ound":633,"port":634,"Ġalso":635,"Ġpart":636,"fter":637,"Ġkn":638,"Ġbec":639,"Ġtime":640,"ens":641,"Ġ5":642,"ople":643,"Ġwhat":644,"Ġno":645,"du":646,"mer":647,"ang":648,"Ġnew":649,"----":650,"Ġget":651,"ory":652,"ition":653,"ings":654,"Ġjust":655,"Ġinto":656,"Ġ0":657,"ents":658,"ove":659,"te":660,"Ġpeople":661,"Ġpre":662,"Ġits":663,"Ġrec":664,"Ġtw":665,"ian":666,"irst":667,"ark":668,"ors":669,"Ġwork":670,"ade":671,"ob":672,"Ġshe":673,"Ġour":674,"wn":675,"ink":676,"lic":677,"Ġ19":678,"ĠHe":679,"ish":680,"nder":681,"ause":682,"Ġhim":683,"ons":684,"Ġ[":685,"Ġro":686,"form":687,"ild":688,"ates":689,"vers":690,"Ġonly":691,"oll":692,"Ġspe":693,"ck":694,"ell":695,"amp":696,"Ġacc":697,"Ġbl":698,"ious":699,"urn":700,"ft":701,"ood":702,"Ġhow":703,"hed":704,"Ġ'":705,"Ġafter":706,"aw":707,"Ġatt":708,"ov":709,"ne":710,"Ġplay":711,"erv":712,"ict":713,"Ġcould":714,"itt":715,"Ġam":716,"Ġfirst":717,"Ġ6":718,"Ġact":719,"Ġ$":720,"ec":721,"hing":722,"ual":723,"ull":724,"Ġcomm":725,"oy":726,"old":727,"ces":728,"ater":729,"Ġfe":730,"Ġbet":731,"we":732,"iff":733,"Ġtwo":734,"ock":735,"Ġback":736,").":737,"ident":738,"Ġunder":739,"rough":740,"sel":741,"xt":742,"Ġmay":743,"round":744,"Ġpo":745,"ph":746,"iss":747,"Ġdes":748,"Ġmost":749,"Ġdid":750,"Ġadd":751,"ject":752,"Ġinc":753,"fore":754,"Ġpol":755,"ont":756,"Ġagain":757,"clud":758,"tern":759,"Ġknow":760,"Ġneed":761,"Ġcons":762,"Ġco":763,"Ġ.":764,"Ġwant":765,"Ġsee":766,"Ġ7":767,"ning":768,"iew":769,"ĠThis":770,"ced":771,"Ġeven":772,"Ġind":773,"ty":774,"ĠWe":775,"ath":776,"Ġthese":777,"Ġpr":778,"Ġuse":779,"Ġbecause":780,"Ġfl":781,"ng":782,"Ġnow":783,"ĠâĢĵ":784,"com":785,"ise":786,"Ġmake":787,"Ġthen":788,"ower":789,"Ġevery":790,"ĠUn":791,"Ġsec":792,"oss":793,"uch":794,"Ġem":795,"Ġ=":796,"ĠRe":797,"ied":798,"rit":799,"Ġinv":800,"lect":801,"Ġsupp":802,"ating":803,"Ġlook":804,"man":805,"pect":806,"Ġ8":807,"row":808,"Ġbu":809,"Ġwhere":810,"ific":811,"Ġyears":812,"ily":813,"Ġdiff":814,"Ġshould":815,"Ġrem":816,"Th":817,"In":818,"Ġev":819,"day":820,"'re":821,"rib":822,"Ġrel":823,"ss":824,"Ġdef":825,"Ġright":826,"Ġsy":827,"),":828,"les":829,"000":830,"hen":831,"Ġthrough":832,"ĠTr":833,"__":834,"Ġway":835,"Ġdon":836,"Ġ,":837,"Ġ10":838,"ased":839,"Ġass":840,"ublic":841,"Ġreg":842,"ĠAnd":843,"ix":844,"Ġvery":845,"Ġinclud":846,"other":847,"Ġimp":848,"oth":849,"Ġsub":850,"ĠâĢĶ":851,"Ġbeing":852,"arg":853,"ĠWh":854,"==":855,"ible":856,"Ġdoes":857,"ange":858,"ram":859,"Ġ9":860,"ert":861,"ps":862,"ited":863,"ational":864,"Ġbr":865,"Ġdown":866,"Ġmany":867,"aking":868,"Ġcall":869,"uring":870,"ities":871,"Ġph":872,"ics":873,"als":874,"Ġdec":875,"ative":876,"ener":877,"Ġbefore":878,"ility":879,"Ġwell":880,"Ġmuch":881,"erson":882,"Ġthose":883,"Ġsuch":884,"Ġke":885,"Ġend":886,"ĠBut":887,"ason":888,"ting":889,"Ġlong":890,"ef":891,"Ġthink":892,"ys":893,"Ġbel":894,"Ġsm":895,"its":896,"ax":897,"Ġown":898,"Ġprov":899,"Ġset":900,"ife":901,"ments":902,"ble":903,"ward":904,"Ġshow":905,"Ġpres":906,"ms":907,"omet":908,"Ġob":909,"Ġsay":910,"ĠSh":911,"ts":912,"ful":913,"Ġeff":914,"Ġgu":915,"Ġinst":916,"und":917,"ren":918,"cess":919,"Ġent":920,"ĠYou":921,"Ġgood":922,"Ġstart":923,"ince":924,"Ġmade":925,"tt":926,"stem":927,"olog":928,"up":929,"Ġ|":930,"ump":931,"Ġhel":932,"vern":933,"ular":934,"ually":935,"Ġac":936,"Ġmon":937,"Ġlast":938,"Ġ200":939,"10":940,"Ġstud":941,"ures":942,"ĠAr":943,"self":944,"ars":945,"meric":946,"ues":947,"cy":948,"Ġmin":949,"ollow":950,"Ġcol":951,"io":952,"Ġmod":953,"Ġcount":954,"ĠCom":955,"hes":956,"Ġfin":957,"air":958,"ier":959,"âĢĶ":960,"read":961,"ank":962,"atch":963,"ever":964,"Ġstr":965,"Ġpoint":966,"ork":967,"ĠNew":968,"Ġsur":969,"ool":970,"alk":971,"ement":972,"Ġused":973,"ract":974,"ween":975,"Ġsame":976,"oun":977,"ĠAl":978,"ci":979,"Ġdiffere":980,"Ġwhile":981,"--------":982,"Ġgame":983,"cept":984,"Ġsim":985,"...":986,"Ġinter":987,"ek":988,"Ġreport":989,"Ġprodu":990,"Ġstill":991,"led":992,"ah":993,"Ġhere":994,"Ġworld":995,"Ġthough":996,"Ġnum":997,"arch":998,"imes":999,"ale":1000,"ĠSe":1001,"ĠIf":1002,"//":1003,"ĠLe":1004,"Ġret":1005,"Ġref":1006,"Ġtrans":1007,"ner":1008,"ution":1009,"ters":1010,"Ġtake":1011,"ĠCl":1012,"Ġconf":1013,"way":1014,"ave":1015,"Ġgoing":1016,"Ġsl":1017,"ug":1018,"ĠAmeric":1019,"Ġspec":1020,"Ġhand":1021,"Ġbetween":1022,"ists":1023,"ĠDe":1024,"oot":1025,"It":1026,"Ġear":1027,"Ġagainst":1028,"Ġhigh":1029,"gan":1030,"az":1031,"ather":1032,"Ġexp":1033,"Ġop":1034,"Ġins":1035,"Ġgr":1036,"Ġhelp":1037,"Ġrequ":1038,"ets":1039,"ins":1040,"ĠPro":1041,"ism":1042,"Ġfound":1043,"land":1044,"ata":1045,"uss":1046,"ames":1047,"Ġperson":1048,"Ġgreat":1049,"pr":1050,"Ġsign":1051,"ĠAn":1052,"'ve":1053,"Ġsomet":1054,"Ġser":1055,"hip":1056,"Ġrun":1057,"Ġ:":1058,"Ġter":1059,"irect":1060,"Ġfollow":1061,"Ġdet":1062,"ices":1063,"Ġfind":1064,"12":1065,"Ġmem":1066,"Ġcr":1067,"ered":1068,"ex":1069,"Ġext":1070,"uth":1071,"ense":1072,"co":1073,"Ġteam":1074,"ving":1075,"ouse":1076,"ash":1077,"att":1078,"ved":1079,"Ġsystem":1080,"ĠAs":1081,"der":1082,"ives":1083,"min":1084,"Ġlead":1085,"ĠBl":1086,"cent":1087,"Ġaround":1088,"Ġgovern":1089,"Ġcur":1090,"velop":1091,"any":1092,"Ġcour":1093,"alth":1094,"ages":1095,"ize":1096,"Ġcar":1097,"ode":1098,"Ġlaw":1099,"Ġread":1100,"'m":1101,"con":1102,"Ġreal":1103,"Ġsupport":1104,"Ġ12":1105,"....":1106,"Ġreally":1107,"ness":1108,"Ġfact":1109,"Ġday":1110,"Ġboth":1111,"ying":1112,"Ġserv":1113,"ĠFor":1114,"Ġthree":1115,"Ġwom":1116,"Ġmed":1117,"ody":1118,"ĠThey":1119,"50":1120,"Ġexper":1121,"ton":1122,"Ġeach":1123,"akes":1124,"Ġche":1125,"Ġcre":1126,"ines":1127,"Ġrep":1128,"19":1129,"gg":1130,"illion":1131,"Ġgrou":1132,"ute":1133,"ik":1134,"We":1135,"get":1136,"ER":1137,"Ġmet":1138,"Ġsays":1139,"ox":1140,"Ġduring":1141,"ern":1142,"ized":1143,"ared":1144,"Ġfam":1145,"ically":1146,"Ġhapp":1147,"ĠIs":1148,"Ġchar":1149,"med":1150,"vent":1151,"Ġgener":1152,"ient":1153,"ple":1154,"iet":1155,"rent":1156,"11":1157,"ves":1158,"ption":1159,"Ġ20":1160,"formation":1161,"Ġcor":1162,"Ġoffic":1163,"ield":1164,"Ġtoo":1165,"ision":1166,"Ġinf":1167,"ĠZ":1168,"the":1169,"oad":1170,"Ġpublic":1171,"Ġprog":1172,"ric":1173,"**":1174,"Ġwar":1175,"Ġpower":1176,"view":1177,"Ġfew":1178,"Ġloc":1179,"Ġdifferent":1180,"Ġstate":1181,"Ġhead":1182,"'ll":1183,"Ġposs":1184,"Ġstat":1185,"ret":1186,"ants":1187,"Ġval":1188,"Ġiss":1189,"Ġcle":1190,"ivers":1191,"anc":1192,"Ġexpl":1193,"Ġanother":1194,"ĠQ":1195,"Ġav":1196,"thing":1197,"nce":1198,"Wh":1199,"Ġchild":1200,"Ġsince":1201,"ired":1202,"less":1203,"Ġlife":1204,"Ġdevelop":1205,"ittle":1206,"Ġdep":1207,"Ġpass":1208,"ãĥ":1209,"Ġturn":1210,"orn":1211,"This":1212,"bers":1213,"ross":1214,"ĠAd":1215,"Ġfr":1216,"Ġresp":1217,"Ġsecond":1218,"oh":1219,"Ġ/":1220,"Ġdisc":1221,"Ġ&":1222,"Ġsomething":1223,"Ġcomple":1224,"Ġed":1225,"Ġfil":1226,"Ġmonth":1227,"aj":1228,"uc":1229,"Ġgovernment":1230,"Ġwithout":1231,"Ġleg":1232,"Ġdist":1233,"Ġput":1234,"Ġquest":1235,"ann":1236,"Ġprot":1237,"20":1238,"Ġnever":1239,"ience":1240,"Ġlevel":1241,"Ġart":1242,"Ġthings":1243,"Ġmight":1244,"Ġeffect":1245,"Ġcontro":1246,"Ġcent":1247,"Ġ18":1248,"Ġallow":1249,"Ġbelie":1250,"chool":1251,"ott":1252,"Ġincre":1253,"Ġfeel":1254,"Ġresult":1255,"Ġlot":1256,"Ġfun":1257,"ote":1258,"Ġty":1259,"erest":1260,"Ġcontin":1261,"Ġusing":1262,"Ġbig":1263,"201":1264,"Ġask":1265,"Ġbest":1266,"Ġ)":1267,"IN":1268,"Ġopp":1269,"30":1270,"Ġnumber":1271,"iness":1272,"St":1273,"lease":1274,"Ġca":1275,"Ġmust":1276,"Ġdirect":1277,"Ġgl":1278,"Ġ<":1279,"Ġopen":1280,"Ġpost":1281,"Ġcome":1282,"Ġseem":1283,"ording":1284,"Ġweek":1285,"ately":1286,"ital":1287,"Ġel":1288,"riend":1289,"Ġfar":1290,"Ġtra":1291,"inal":1292,"Ġpri":1293,"ĠUS":1294,"Ġplace":1295,"Ġform":1296,"Ġtold":1297,"\":":1298,"ains":1299,"ature":1300,"ĠTrump":1301,"Ġstand":1302,"Ġ#":1303,"ider":1304,"ĠFr":1305,"Ġnext":1306,"Ġsoc":1307,"Ġpur":1308,"Ġlet":1309,"Ġlittle":1310,"Ġhum":1311,"Ġi":1312,"ron":1313,"15":1314,"Ġ15":1315,"Ġcommun":1316,"Ġmark":1317,"ĠThere":1318,"Ġwr":1319,"ĠThat":1320,"Ġinformation":1321,"ways":1322,"Ġbus":1323,"app":1324,"Ġinvest":1325,"me":1326,"Ġhard":1327,"ained":1328,"ead":1329,"Ġimport":1330,"Ġappro":1331,"Ġtest":1332,"Ġtri":1333,"Ġrest":1334,"osed":1335,"Ġfull":1336,"Ġcare":1337,"ĠSp":1338,"Ġcase":1339,"ON":1340,"Ġsk":1341,"Ġless":1342,"Ġ+":1343,"Ġpartic":1344,"ĠPl":1345,"ably":1346,"uck":1347,"ished":1348,"chn":1349,"be":1350,"Ġlist":1351,"ator":1352,"Ġtop":1353,"Ġadv":1354,"ĠBe":1355,"ruct":1356,"Ġdem":1357,"ration":1358,"ling":1359,"gy":1360,"reen":1361,"ger":1362,"Ġhome":1363,"Ġleft":1364,"Ġbetter":1365,"Ġdata":1366,"Ġ11":1367,"Ġattack":1368,"Ġproble":1369,"line":1370,"ards":1371,"Ġbeh":1372,"ral":1373,"ĠHow":1374,"ĠShe":1375,"arge":1376,"Ġ--":1377,"://":1378,"Ġbro":1379,"ĠPh":1380,"ats":1381,"Ġbuild":1382,"ww":1383,"ided":1384,"aim":1385,"ases":1386,"ency":1387,"Ġmain":1388,"ined":1389,"Ġincluding":1390,"Ġ{":1391,"Ġgot":1392,"Ġinterest":1393,"Ġkeep":1394,"ĠX":1395,"Ġeas":1396,"aining":1397,"Ġclass":1398,"âĢ¦":1399,"ĠNo":1400,"Ġvar":1401,"Ġsmall":1402,"ample":1403,"AT":1404,"Ġide":1405,"ĠSo":1406,"Ġrece":1407,"Ġpolit":1408,"Ġmov":1409,"Ġplan":1410,"Ġpercent":1411,"iving":1412,"Ġcamp":1413,"Ġpay":1414,"14":1415,"sc":1416,"ised":1417,"Ġunt":1418,"oney":1419,"ploy":1420,"====":1421,"Ġdidn":1422,"ĠInd":1423,"els":1424,"ertain":1425,"Ġpos":1426,"____":1427,"iver":1428,"Ġprocess":1429,"Ġprogram":1430,"ified":1431,"ĠRep":1432,"16":1433,"uro":1434,"ology":1435,"atter":1436,"ina":1437,"Ġname":1438,"ĠAll":1439,"Ġfour":1440,"Ġreturn":1441,"vious":1442,"bs":1443,"Ġcalled":1444,"Ġmove":1445,"ĠSc":1446,"ird":1447,"Ġgroup":1448,"Ġbre":1449,"Ġmen":1450,"Ġcap":1451,"ten":1452,"ee":1453,"Ġdri":1454,"leg":1455,"here":1456,"uthor":1457,"Ġpat":1458,"Ġcurrent":1459,"ides":1460,"Ġpop":1461,"to":1462,"ention":1463,"Ġalways":1464,"Ġmil":1465,"Ġwomen":1466,"Ġ16":1467,"Ġold":1468,"iven":1469,"raph":1470,"ĠOr":1471,"ror":1472,"ently":1473,"Ġnear":1474,"ĠEx":1475,"ream":1476,"sh":1477,"Ġ14":1478,"Ġfree":1479,"ission":1480,"stand":1481,"ĠCon":1482,"ality":1483,"used":1484,"13":1485,"Ġdesign":1486,"Ġchange":1487,"Ġchang":1488,"Ġbo":1489,"Ġvis":1490,"ember":1491,"Ġbook":1492,"ready":1493,"Ġkill":1494,"25":1495,"pped":1496,"Ġaway":1497,"Ġable":1498,"Ġcountry":1499,"Ġconst":1500,"arn":1501,"Ġorder":1502,"AR":1503,"ior":1504,"ium":1505,"orth":1506,"18":1507,"ailable":1508,"Ġsw":1509,"Ġmillion":1510,"Ġ13":1511,"atic":1512,"ted":1513,"ĠGo":1514,"Ġoper":1515,"eng":1516,"Ġthing":1517,"ajor":1518,"conom":1519,"ĠComm":1520,"Ġwhy":1521,"ured":1522,"ural":1523,"Ġschool":1524,"by":1525,"ĠMar":1526,"Ġaff":1527,"Ġdays":1528,"Ġann":1529,"ush":1530,"ane":1531,"If":1532,"eg":1533,"Ġprof":1534,"Ġhealth":1535,"outh":1536,"But":1537,"ional":1538,".,":1539,"Ġsol":1540,"Ġalready":1541,"Ġ30":1542,"Ġcharact":1543,"He":1544,"Ġfriend":1545,"ES":1546,"ians":1547,"icle":1548,"'d":1549,"ĠOn":1550,"Ġleast":1551,"Ġprom":1552,"Ġdr":1553,"Ġhist":1554,"ither":1555,"Ġest":1556,"iqu":1557,"17":1558,"son":1559,"Ġtell":1560,"Ġtalk":1561,"ohn":1562,"oint":1563,"lection":1564,"AN":1565,"Ġuntil":1566,"augh":1567,"Ġlater":1568,"Ġve":1569,"Ġview":1570,"ending":1571,"ived":1572,"Ġword":1573,"ware":1574,"Ġcost":1575,"Ġenough":1576,"Ġgive":1577,"ĠUnited":1578,"Ġtechn":1579,"arent":1580,"OR":1581,"Ġpar":1582,"ĠDr":1583,"Ġ2016":1584,"rist":1585,"ering":1586,"ĠÂ":1587,"Ġlarge":1588,"side":1589,"acy":1590,"ccess":1591,"Ġwin":1592,"Ġimportant":1593,"Ġ199":1594,"Ġdoesn":1595,"Ġ17":1596,"Ġbusiness":1597,"Ġclear":1598,"Ġrese":1599,"\",":1600,"ury":1601,"Ġequ":1602,"aster":1603,"alf":1604,"ĠAmerican":1605,"nect":1606,"Ġexpect":1607,"iversity":1608,"Ġocc":1609,"ĠFl":1610,"Ġkind":1611,"Ġmean":1612,"Ġpast":1613,"Ġdev":1614,"Ġbas":1615,"let":1616,"raft":1617,"Ġorgan":1618,"Ġdel":1619,"Ġperform":1620,"Ġstory":1621,"Ġseason":1622,"ĠCol":1623,"Ġclaim":1624,"Ġcame":1625,"Ġwithin":1626,"Ġline":1627,"Ġproject":1628,"ĠAt":1629,"Ġcontrol":1630,"ended":1631,"ĠSy":1632,"Ġair":1633,"ization":1634,"Ġ*":1635,"ley":1636,"Ġmoney":1637,"idd":1638,"You":1639,"for":1640,"Ġfamily":1641,"Ġmaking":1642,"Ġbit":1643,"Ġpolice":1644,"Ġhappen":1645,"Ġvers":1646,"ony":1647,"uff":1648,"ĠWhen":1649,"Ġsit":1650,"ideo":1651,"lf":1652,"ison":1653,"Ġsure":1654,"gin":1655,"Ġappear":1656,"Ġlight":1657,"Ġes":1658,"of":1659,"Ġwater":1660,"Ġtimes":1661,"not":1662,"Ġgrow":1663,"Ġcompany":1664,"ĠTe":1665,"ows":1666,"Ġmar":1667,"ource":1668,"iol":1669,"arm":1670,"br":1671,"Ġexample":1672,"Ġconc":1673,"Ġfore":1674,"ĠTo":1675,"pro":1676,"EN":1677,"ries":1678,"Ġ25":1679,"ĠCan":1680,"ney":1681,"Ġactually":1682,"Ġever":1683,"urity":1684,"aken":1685,"aps":1686,"Ġtax":1687,"Ġmajor":1688,"ama":1689,"Ġoften":1690,"eral":1691,"Ġhuman":1692,"Ġjob":1693,"ister":1694,"Ġavailable":1695,"ocr":1696,"enn":1697,"aid":1698,"ivid":1699,"Ġrecord":1700,"?\"":1701,"Ġsing":1702,"ĠAm":1703,"idence":1704,"Ġnews":1705,"ster":1706,"Ġeconom":1707,"Ġfollowing":1708,"ĠBr":1709,"ising":1710,"Ġhour":1711,"most":1712,"ument":1713,"Ġsex":1714,"Ġdesc":1715,"Ġbecome":1716,"ĠEd":1717,"Ġtook":1718,"Ġhaving":1719,"Ġproduct":1720,"ault":1721,"As":1722,"aring":1723,"Ġmeans":1724,"Ġhop":1725,"une":1726,"Ġcho":1727,"Ġcertain":1728,"Ġnon":1729,"Ġdeal":1730,"24":1731,"lement":1732,"oci":1733,"ene":1734,"Ġside":1735,"ĠPr":1736,"ĠMay":1737,"Ġreason":1738,"ued":1739,"ched":1740,"ulation":1741,"Ġelect":1742,"Ġofficial":1743,"Ġpossible":1744,"Ġhold":1745,"ands":1746,"ots":1747,"Ġcity":1748,"ories":1749,"Ġsever":1750,"Ġchildren":1751,"Ġonce":1752,"Ġactiv":1753,"ler":1754,"Ġnight":1755,"itions":1756,"ĠJohn":1757,"ape":1758,"play":1759,"Ġdone":1760,"Ġlim":1761,"Ġworking":1762,"ĠPres":1763,"orld":1764,"eb":1765,"ĠCo":1766,"Ġbody":1767,"ails":1768,"utes":1769,"ĠMr":1770,"Ġwhether":1771,"Ġauthor":1772,"rop":1773,"Ġproper":1774,"Ġseen":1775,");":1776,"Ġfac":1777,"ĠSu":1778,"Ġcond":1779,"iting":1780,"Ġcourse":1781,"Ġ}":1782,"----------------":1783,"aign":1784,"Ġevent":1785,"Ġeng":1786,"Ġpot":1787,"Ġintern":1788,"iam":1789,"Ġshort":1790,"empt":1791,"ãĤ":1792,"ĠGod":1793,"ilar":1794,"80":1795,"Ġorig":1796,"IS":1797,"ourn":1798,"ability":1799,"itive":1800,"Ġdam":1801,"Ġ100":1802,"Ġpress":1803,"Ġdoing":1804,"Ġprotect":1805,"ring":1806,"Ġthought":1807,"Ġquestion":1808,"rew":1809,"ĠWar":1810,"Ġseveral":1811,"ĠState":1812,"Ġgiven":1813,"Ġfund":1814,"ĠTw":1815,"Ġwent":1816,"ances":1817,"work":1818,"por":1819,"my":1820,"40":1821,"Ġarg":1822,"artment":1823,"ustom":1824,"Ġpolic":1825,"Ġmeet":1826,"Ġcreat":1827,"22":1828,"ĠStates":1829,"Ġgames":1830,"raw":1831,"uture":1832,"Ġunderstand":1833,"urs":1834,"ĠOb":1835,"lish":1836,"sy":1837,"Ġmakes":1838,"Ġwon":1839,"agon":1840,"Ġhtt":1841,"Ġlove":1842,"ential":1843,"Ġcomplete":1844,"par":1845,"ĠIm":1846,"AL":1847,"Ġaccount":1848,"Âł":1849,"ored":1850,"vert":1851,"Ġident":1852,"Ġ2015":1853,"Ġothers":1854,"ĠMin":1855,"iber":1856,"verage":1857,"There":1858,"itional":1859,"dd":1860,"Ġprob":1861,"Ġyoung":1862,"Ġalong":1863,"Ġaccording":1864,"Ġyet":1865,"Ġmembers":1866,"ĠWhat":1867,"oid":1868,"ĠMan":1869,"And":1870,"Ġamong":1871,"ai":1872,"Ġemploy":1873,"ĠRes":1874,"Ġ>":1875,"Ġinvol":1876,"Ġlow":1877,"af":1878,"ĠCar":1879,"Ġhig":1880,"ĠOne":1881,"ĠSec":1882,"ination":1883,"Ġlikely":1884,"Ġant":1885,"aged":1886,"ĠRuss":1887,"Ġben":1888,"Ġrele":1889,"For":1890,"back":1891,"ĠNot":1892,"Ġpresident":1893,"ball":1894,"Ġaccess":1895,"ividual":1896,"ĠDem":1897,"ĠEuro":1898,"60":1899,"Ġknown":1900,"irl":1901,"ĠGr":1902,"Ġearly":1903,"use":1904,"iety":1905,"âĢĵ":1906,"Ġfight":1907,"Ġsent":1908,"Ġtoday":1909,"Ġmarket":1910,"\".":1911,"Ġbased":1912,"Ġstrong":1913,"urther":1914,"Ġdeb":1915,"mber":1916,"Ġproblem":1917,"Ġdeath":1918,"Ġsocial":1919,"imate":1920,"AS":1921,"ortun":1922,"Ġcampaign":1923,"ery":1924,"Ch":1925,"Ġey":1926,"ially":1927,"Ġmus":1928,"wh":1929,"pos":1930,"Ġer":1931,"Ġsaf":1932,"Ġmonths":1933,"iron":1934,"Ġviol":1935,"Ġfive":1936,"Ġstre":1937,"Ġplayers":1938,"inc":1939,"ald":1940,"year":1941,"aun":1942,"Ġsuccess":1943,"Ġpresent":1944,"erence":1945,"Ġ2014":1946,"Ġsugg":1947,"Ġparticular":1948,"Ġtry":1949,"Ġsuggest":1950,"ĠChrist":1951,"ones":1952,"Ġpriv":1953,"23":1954,"Ġcrit":1955,"Ġland":1956,"Ġlocal":1957,"ify":1958,"29":1959,"Ġaut":1960,"ED":1961,"ĠGu":1962,"Ġmult":1963,"Ġpolitical":1964,"Ġasked":1965,"Ġformer":1966,"itter":1967,"ript":1968,"Ġclose":1969,"Ġpract":1970,"ĠYork":1971,"Ġgetting":1972,"Ġacross":1973,"Ġcomb":1974,"Ġbelieve":1975,"Ġz":1976,"Ġtoget":1977,"Ġtogether":1978,"ĠCent":1979,"irc":1980,"Ġindividual":1981,"ĠMc":1982,"27":1983,"isk":1984,"ĠEng":1985,"Ġface":1986,"Ġ24":1987,"Ġvalue":1988,"Ġarea":1989,"ev":1990,"Ġwrit":1991,"ĠPresident":1992,"Ġvot":1993,"Ġkey":1994,"Ġmom":1995,"put":1996,"Ġanything":1997,"Ġexperience":1998,"attle":1999,"Ġmind":2000,"aff":2001,"omm":2002,"Ġfuture":2003,"ged":2004,"Ġcut":2005,"Ġtot":2006,"itch":2007,"Ġvideo":2008,"Ġinvestig":2009,"Ġnet":2010,"ĠMy":2011,"rict":2012,"ien":2013,".)":2014,"Ġimpro":2015,"though":2016,"wards":2017,"Ġconnect":2018,"ĠMed":2019,"selves":2020,"ensive":2021,"mb":2022,"ober":2023,"ators":2024,"An":2025,"Ġ50":2026,"Ġredu":2027,"resent":2028,"Ġabove":2029,"Ġfre":2030,"ĠEurope":2031,"sw":2032,"Ġamount":2033,"ĠApp":2034,"Ġeither":2035,"Ġmilit":2036,"Ġanal":2037,"Ġfail":2038,"ĠEn":2039,"ales":2040,"Ġspecial":2041,"Ġblack":2042,"IT":2043,"cher":2044,"Ġlooking":2045,"Ġfire":2046,"yn":2047,"Ġalmost":2048,"oon":2049,"Ġstudy":2050,"Ġmiss":2051,"ches":2052,"rown":2053,"Ġtre":2054,"Ġcommunity":2055,"Ġmedia":2056,"Ġfood":2057,"Ġcomes":2058,"ĠUniversity":2059,"Ġsingle":2060,"What":2061,"uly":2062,"Ġhalf":2063,"ague":2064,"hod":2065,"ĠRepublic":2066,"Ġstarted":2067,"Ġquick":2068,"oto":2069,"book":2070,"Ġissue":2071,"itor":2072,"Ġelse":2073,"Ġconsider":2074,"26":2075,"rodu":2076,"Ġtaken":2077,"28":2078,"99":2079,"ĠWith":2080,"Ġtrue":2081,"Ġwa":2082,"Ġtrad":2083,"Ġago":2084,"Ġmess":2085,"ief":2086,"Ġadded":2087,"oke":2088,"Ġbad":2089,"Ġfav":2090,"33":2091,"Ġsimilar":2092,"ask":2093,"ĠDon":2094,"Ġcharacter":2095,"orts":2096,"ĠHouse":2097,"Ġreported":2098,"Ġtype":2099,"val":2100,"iod":2101,"ĠHowever":2102,"Ġtarg":2103,"Ġentire":2104,"pping":2105,"Ġhistory":2106,"Ġlive":2107,"ffic":2108,"........":2109,"ederal":2110,"Ġtrying":2111,"Ġdiscuss":2112,"ĠHar":2113,"aces":2114,"lished":2115,"Ġself":2116,"osp":2117,"rest":2118,"Ġroom":2119,"elt":2120,"Ġfall":2121,"olution":2122,"Ġet":2123,"Ġx":2124,"Ġisn":2125,"Ġidea":2126,"bo":2127,"Ġsound":2128,"ĠDep":2129,"Ġsomeone":2130,"cially":2131,"ully":2132,"Ġfoc":2133,"Ġobject":2134,"ift":2135,"aper":2136,"Ġplayer":2137,"Ġrather":2138,"Ġservice":2139,"ashing":2140,"ĠDo":2141,"ĠPart":2142,"rug":2143,"mon":2144,"ply":2145,"Ġmor":2146,"Ġnothing":2147,"Ġprovide":2148,"IC":2149,"ung":2150,"Ġparty":2151,"Ġexist":2152,"Ġmag":2153,"70":2154,"Ġrul":2155,"Ġhouse":2156,"Ġbehind":2157,"Ġhowever":2158,"ĠWorld":2159,"Ġsum":2160,"Ġapplic":2161,"Ġ;":2162,"Ġfunction":2163,"gr":2164,"ĠPol":2165,"Ġfront":2166,"200":2167,"Ġseries":2168,"Ġtem":2169,"Ġtyp":2170,"ills":2171,"Ġopt":2172,"Ġpoints":2173,"Ġbelow":2174,"itted":2175,"Ġspecific":2176,"Ġ2017":2177,"umb":2178,"Ġra":2179,"Ġprevious":2180,"Ġpret":2181,"reme":2182,"Ġcustom":2183,"Ġcourt":2184,"ĠMe":2185,"Ġrepl":2186,"Ġwhole":2187,"go":2188,"cer":2189,"Ġtreat":2190,"ĠAct":2191,"Ġprobably":2192,"Ġlearn":2193,"ender":2194,"ĠAss":2195,"Ġversion":2196,"now":2197,"Ġcheck":2198,"ĠCal":2199,"RE":2200,"minist":2201,"On":2202,"ources":2203,"Ġbenef":2204,"Ġdoc":2205,"Ġdeter":2206,"Ġenc":2207,"Ġsuper":2208,"Ġaddress":2209,"Ġvict":2210,"Ġ2013":2211,"Ġmeas":2212,"tr":2213,"Ġfield":2214,"When":2215,"Ġsignific":2216,"uge":2217,"Ġfeat":2218,"Ġcommon":2219,"load":2220,"Ġbegin":2221,"Ġbring":2222,"Ġaction":2223,"erman":2224,"Ġdescrib":2225,"Ġindust":2226,"Ġwanted":2227,"ried":2228,"ming":2229,"Ġattempt":2230,"45":2231,"fer":2232,"Ġdue":2233,"ression":2234,"##":2235,"Ġshall":2236,"Ġsix":2237,"oo":2238,"Ġstep":2239,"Ġpub":2240,"Ġhimself":2241,"Ġ23":2242,"Ġcop":2243,"Ġdest":2244,"Ġstop":2245,"AC":2246,"ibility":2247,"Ġlab":2248,"icult":2249,"Ġhours":2250,"Ġcreate":2251,"Ġfurther":2252,"ĠAmerica":2253,"ĠCity":2254,"Ġdou":2255,"head":2256,"ST":2257,"ĠNorth":2258,"cing":2259,"Ġnational":2260,"ule":2261,"ĠInst":2262,"Ġtaking":2263,"ĠQu":2264,"irt":2265,"Ġred":2266,"Ġresearch":2267,"viron":2268,"ĠGe":2269,"Ġbreak":2270,"ana":2271,"Ġspace":2272,"aterial":2273,"Ġrecent":2274,"ĠAb":2275,"Ġgeneral":2276,"Ġhit":2277,"Ġperiod":2278,"Ġeverything":2279,"ively":2280,"Ġphys":2281,"Ġsaying":2282,"anks":2283,"Ġcou":2284,"Ġcult":2285,"aced":2286,"eal":2287,"uation":2288,"Ġcoun":2289,"lu":2290,"Ġinclude":2291,"Ġposition":2292,"ĠAfter":2293,"ĠCanad":2294,"ĠEm":2295,"Ġimm":2296,"ĠRed":2297,"Ġpick":2298,"Ġcompl":2299,"Ġmatter":2300,"reg":2301,"ext":2302,"angu":2303,"isc":2304,"ole":2305,"aut":2306,"Ġcompet":2307,"eed":2308,"fect":2309,"Ġ21":2310,"ĠSen":2311,"ĠThese":2312,"asing":2313,"Ġcannot":2314,"Ġinit":2315,"Ġrelations":2316,"ached":2317,"Ġbar":2318,"Ġ40":2319,"ĠTH":2320,"Ġ2012":2321,"Ġvol":2322,"Ġground":2323,"Ġsecurity":2324,"Ġupd":2325,"ilt":2326,"35":2327,"Ġconcern":2328,"ĠJust":2329,"Ġwhite":2330,"Ġseems":2331,"ĠHer":2332,"pecially":2333,"ients":2334,"Ġannoun":2335,"Ġfig":2336,"ights":2337,"Ġstri":2338,"like":2339,"ids":2340,"Ġsus":2341,"Ġwatch":2342,"Ġâ":2343,"Ġwind":2344,"ĠCont":2345,"Ġitself":2346,"Ġmass":2347,"Al":2348,"yle":2349,"ique":2350,"ĠNational":2351,"Ġabs":2352,"Ġpack":2353,"Ġoutside":2354,"Ġanim":2355,"Ġpain":2356,"eter":2357,"Ġmanag":2358,"duct":2359,"ogn":2360,"Ġ]":2361,"ĠSept":2362,"sec":2363,"off":2364,"ĠJan":2365,"Ġfoot":2366,"ades":2367,"Ġthird":2368,"Ġmot":2369,"Ġevidence":2370,"inton":2371,"Ġthreat":2372,"apt":2373,"ples":2374,"cle":2375,"Ġlo":2376,"Ġdecl":2377,"Ġitem":2378,"medi":2379,"Ġrepresent":2380,"omb":2381,"amer":2382,"Ġsignificant":2383,"ograph":2384,"su":2385,"Ġcal":2386,"ires":2387,"0000":2388,"ID":2389,"AM":2390,"Ġsimply":2391,"Ġlonger":2392,"Ġfile":2393,"OT":2394,"che":2395,"So":2396,"ateg":2397,"org":2398,"ĠHis":2399,"Ġener":2400,"Ġdom":2401,"Ġupon":2402,"ili":2403,"\":\"":2404,"Ġthemselves":2405,"Ġcoming":2406,"Ġquite":2407,"Ġdifficult":2408,"ĠBar":2409,"ilities":2410,"rel":2411,"ends":2412,"cial":2413,"64":2414,"Ġwoman":2415,"rap":2416,"yr":2417,"Ġnecess":2418,"ips":2419,"Ġtext":2420,"Ġrequire":2421,"Ġmilitary":2422,"Ġreview":2423,"Ġrespons":2424,"75":2425,"Ġsubject":2426,"Ġinstead":2427,"Ġissues":2428,"Ġgen":2429,"\",\"":2430,"Ġminutes":2431,"Ġweap":2432,"ray":2433,"amed":2434,"time":2435,"bl":2436,"How":2437,"Ġcode":2438,"ĠSm":2439,"Ġhigher":2440,"ĠSte":2441,"ris":2442,"Ġpage":2443,"Ġstudents":2444,"ĠIntern":2445,"Ġmethod":2446,"ĠAug":2447,"ĠPer":2448,"ĠAg":2449,"Ġpolicy":2450,"ĠSw":2451,"Ġexec":2452,"Ġaccept":2453,"ume":2454,"ribut":2455,"Ġwords":2456,"Ġfinal":2457,"Ġchanges":2458,"ĠDemocr":2459,"Ġfriends":2460,"Ġrespect":2461,"Ġep":2462,"Ġcompan":2463,"ivil":2464,"Ġdamage":2465,"****":2466,"ogle":2467,"vironment":2468,"Ġneg":2469,"ental":2470,"Ġap":2471,"Ġtotal":2472,"ival":2473,"!\"":2474,"lim":2475,"Ġneeds":2476,"Ġagre":2477,"Ġdevelopment":2478,"Ġage":2479,"iple":2480,"21":2481,"Ġresults":2482,"ĠAf":2483,"Sh":2484,"Ġgun":2485,"ĠObama":2486,"roll":2487,"Ġ@":2488,"Ġrights":2489,"ĠBrit":2490,"Ġrunning":2491,"Ġwasn":2492,"Ġport":2493,"Ġrate":2494,"Ġpretty":2495,"Ġtarget":2496,"Ġsaw":2497,"Ġcirc":2498,"Ġworks":2499,"icro":2500,"alt":2501,"over":2502,"www":2503,"That":2504,"lier":2505,"Ġeveryone":2506,"ude":2507,"Ġpie":2508,"iddle":2509,"rael":2510,"Ġrad":2511,"Ġblock":2512,"Ġwalk":2513,"To":2514,"ãģ":2515,"nes":2516,"ĠAust":2517,"aul":2518,"rote":2519,"ĠSouth":2520,"ession":2521,"oph":2522,"Ġshows":2523,"Ġsite":2524,"Ġjo":2525,"Ġrisk":2526,"clus":2527,"lt":2528,"Ġinj":2529,"iding":2530,"ĠSpe":2531,"Ġchall":2532,"irm":2533,"Ġ22":2534,"itting":2535,"str":2536,"Ġhy":2537,"LE":2538,"key":2539,"Ġbegan":2540,"atur":2541,"ashington":2542,"lam":2543,"ĠDav":2544,"bit":2545,"Ġsize":2546,"ĠPar":2547,"38":2548,"ournal":2549,"face":2550,"Ġdecision":2551,"Ġlarg":2552,"Ġjud":2553,"rect":2554,"Ġcontinue":2555,"ĠOct":2556,"overed":2557,"ĠInt":2558,"========":2559,"Ġparent":2560,"ĠWill":2561,"Ġeasy":2562,"Ġdrug":2563,"anger":2564,"Ġsense":2565,"Ġdi":2566,"iday":2567,"Ġenergy":2568,"istic":2569,"Ġassoci":2570,"arter":2571,"obal":2572,"eks":2573,"ĠEl":2574,"urch":2575,"Ġgirl":2576,"oe":2577,"itle":2578,"Ġ28":2579,"ĠChe":2580,"Ġrequest":2581,"Ġsoon":2582,"Ġhost":2583,"ky":2584,"Ġstates":2585,"omes":2586,"Ġmaterial":2587,"lex":2588,"Ġmoment":2589,"Ġansw":2590,"onse":2591,"Ġespecially":2592,"Ġnorm":2593,"Ġservices":2594,"pite":2595,"ran":2596,"Ġrole":2597,"44":2598,"):":2599,"Ġcred":2600,"Cl":2601,"________":2602,"Ġmat":2603,"Ġlog":2604,"ĠClinton":2605,"OU":2606,"Ġoffice":2607,"Ġ26":2608,"Ġcharg":2609,"Ġtrack":2610,"ma":2611,"Ġheart":2612,"Ġball":2613,"Ġpersonal":2614,"Ġbuilding":2615,"na":2616,"set":2617,"body":2618,"ĠBlack":2619,"Ġincrease":2620,"itten":2621,"Ġneeded":2622,"36":2623,"32":2624,"=\"":2625,"Ġlost":2626,"Ġbecame":2627,"Ġgroups":2628,"ĠMus":2629,"Ġwrote":2630,"ĠPe":2631,"Ġprop":2632,"joy":2633,"é":2634,"ĠWhite":2635,"Ġdead":2636,".'":2637,"Ġhttp":2638,"Ġwebs":2639,"OS":2640,"Ġinside":2641,"Ġwrong":2642,"Ġstatement":2643,"Ġ...":2644,"yl":2645,"Ġfilm":2646,"Ġmusic":2647,"Ġshare":2648,"ification":2649,"Ġrelease":2650,"Ġforward":2651,"Ġstay":2652,"Ġcomput":2653,"itte":2654,"ser":2655,"Ġoriginal":2656,"Ġcard":2657,"Ġcand":2658,"Ġdiv":2659,"atural":2660,"Ġfavor":2661,"OM":2662,"Ġcases":2663,"uses":2664,"Ġsection":2665,"Ġleave":2666,"ging":2667,"oved":2668,"ĠWashington":2669,"39":2670,"ĠGl":2671,"Ġrequired":2672,"action":2673,"apan":2674,"oor":2675,"iter":2676,"ĠKing":2677,"Ġcountries":2678,"ĠGerman":2679,"lling":2680,"Ġ27":2681,"34":2682,"Ġquestions":2683,"Ġprim":2684,"Ġcell":2685,"Ġshoot":2686,"Ġanyone":2687,"ĠWest":2688,"Ġaffect":2689,"epend":2690,"Ġonline":2691,"ĠIsrael":2692,"ĠSeptember":2693,"Ġability":2694,"Ġcontent":2695,"ises":2696,"Ġreve":2697,"Ġlaun":2698,"Ġindic":2699,"Ġforce":2700,"cast":2701,"Ġsold":2702,"aving":2703,"fl":2704,"Ġsoft":2705,"Ġcompanies":2706,"ceed":2707,"Ġarticle":2708,"Ġaud":2709,"Ġrev":2710,"Ġeduc":2711,"Ġplaying":2712,"05":2713,"Ġheld":2714,"ctor":2715,"Ġreleased":2716,"Ġfederal":2717,"37":2718,"Ġadminist":2719,"Ġinterview":2720,"Ġinstall":2721,"Ġreceived":2722,"Ġsource":2723,"uk":2724,"Ph":2725,"Ġserious":2726,"Ġcreated":2727,"Ġcause":2728,"Ġimmedi":2729,"Ġdefin":2730,"uel":2731,"ĠDepartment":2732,"ctions":2733,"ĠCour":2734,"ĠNow":2735,"ze":2736,"ites":2737,"itution":2738,"Ġlate":2739,"Ġspeak":2740,"ners":2741,"Ġlegal":2742,"ari":2743,"ĠCor":2744,"Ġweeks":2745,"Ġmodel":2746,"Ġpred":2747,"Ġexact":2748,"BC":2749,"ĠBy":2750,"ING":2751,"osing":2752,"Ġtakes":2753,"Ġregard":2754,"Ġopportun":2755,"Ġprice":2756,"Ġ198":2757,"ĠApr":2758,"fully":2759,"Ġord":2760,"Ġproblems":2761,"ruction":2762,"ham":2763,"ĠCount":2764,"lege":2765,"Ġleaders":2766,"ET":2767,"lev":2768,"Ġdeep":2769,"ological":2770,"ese":2771,"haps":2772,"ĠSome":2773,"Ġpers":2774,"Ġcontract":2775,"Ġrelationship":2776,"sp":2777,"oud":2778,"Ġbase":2779,"48":2780,"mit":2781,"Ad":2782,"ancial":2783,"Ġconsum":2784,"Ġpotential":2785,"Ġlangu":2786,"rem":2787,"eth":2788,"Ġrelig":2789,"ressed":2790,"66":2791,"Ġlink":2792,"Ġlower":2793,"ayer":2794,"ĠJune":2795,"Ġfem":2796,"unt":2797,"erc":2798,"urd":2799,"Ġcontact":2800,"Ġill":2801,"Ġmother":2802,"Ġestab":2803,"htt":2804,"ĠMarch":2805,"ĠBro":2806,"ĠChina":2807,"Ġ29":2808,"Ġsqu":2809,"Ġprovided":2810,"Ġaverage":2811,"asons":2812,"Ġ2011":2813,"Ġexam":2814,"lin":2815,"55":2816,"ned":2817,"Ġperfect":2818,"Ġtou":2819,"alse":2820,"ux":2821,"Ġbuy":2822,"Ġshot":2823,"Ġcollect":2824,"Ġphot":2825,"Ġplayed":2826,"Ġsurpr":2827,"Ġofficials":2828,"Ġsimple":2829,"avy":2830,"Ġindustry":2831,"Ġhands":2832,"ground":2833,"Ġpull":2834,"Ġround":2835,"Ġuser":2836,"Ġrange":2837,"uary":2838,"Ġprivate":2839,"ops":2840,"ees":2841,"Ġways":2842,"ĠMich":2843,"Ġveh":2844,"Ġexcept":2845,"Ġterms":2846,"imum":2847,"pper":2848,"ION":2849,"ores":2850,"ĠDragon":2851,"oul":2852,"Ġden":2853,"Ġperformance":2854,"Ġbill":2855,"cil":2856,"47":2857,"Ġenvironment":2858,"Ġexc":2859,"add":2860,"Ġworth":2861,"Ġpict":2862,"Ġchance":2863,"Ġ2018":2864,"bor":2865,"Ġspeed":2866,"iction":2867,"Ġalleg":2868,"ĠJapan":2869,"atory":2870,"reet":2871,"Ġmatch":2872,"ĠII":2873,"Ġstru":2874,"order":2875,"Ġste":2876,"Ġliving":2877,"Ġstruct":2878,"ino":2879,"Ġsepar":2880,"hern":2881,"Ġresponse":2882,"Ġenjoy":2883,"Ġvia":2884,"AD":2885,"uments":2886,"acebook":2887,"Ġmember":2888,"ibr":2889,"izing":2890,"Ġtool":2891,"ĠMon":2892,"ĠWhile":2893,"hood":2894,"ĠAng":2895,"ĠDef":2896,"Ġoffer":2897,"Tr":2898,"aur":2899,"Ġturned":2900,"ĠJuly":2901,"down":2902,"anced":2903,"Ġrecently":2904,"ĠEar":2905,"Ġce":2906,"ĠStar":2907,"ĠCong":2908,"rought":2909,"Ġblood":2910,"Ġhope":2911,"Ġcomment":2912,"aint":2913,"Ġarri":2914,"iles":2915,"Ġparticip":2916,"ought":2917,"ription":2918,"08":2919,"49":2920,"Ġgave":2921,"Ġselect":2922,"Ġkilled":2923,"sych":2924,"Ġgoes":2925,"ij":2926,"Ġcoll":2927,"Ġimpact":2928,"atives":2929,"ĠSer":2930,"09":2931,"ĠAugust":2932,"Ġboy":2933,"de":2934,"ĠDes":2935,"Ġfelt":2936,"US":2937,"Ġexpected":2938,"Ġimage":2939,"ĠMark":2940,"ccording":2941,"oice":2942,"EC":2943,"ĠMag":2944,"ened":2945,"hold":2946,"ĠPost":2947,"Ġprevent":2948,"No":2949,"Ġinvolved":2950,"Ġeyes":2951,"Ġquickly":2952,"At":2953,"unk":2954,"Ġbehav":2955,"Ġur":2956,"Ġled":2957,"come":2958,"ey":2959,"Ġcandid":2960,"Ġearlier":2961,"Ġfocus":2962,"ety":2963,"Pro":2964,"ledge":2965,"ixed":2966,"illed":2967,"Ġpopular":2968,"AP":2969,"Ġsett":2970,"light":2971,"Ġvarious":2972,"inks":2973,"Ġlevels":2974,"Ġroad":2975,"ellig":2976,"ables":2977,"hel":2978,"ittee":2979,"ĠGener":2980,"ype":2981,"Ġheard":2982,"icles":2983,"Ġmis":2984,"Ġusers":2985,"ĠSan":2986,"Ġimprove":2987,"Ġfather":2988,"Ġsearch":2989,"They":2990,"vil":2991,"Ġprofess":2992,"Ġknew":2993,"Ġloss":2994,"Ġevents":2995,"65":2996,"Ġbillion":2997,"07":2998,"02":2999,"ĠNews":3000,"ĠAM":3001,"Ġcover":3002,"where":3003,"ension":3004,"Ġbott":3005,"Ġareas":3006,"ences":3007,"ope":3008,"ĠTwitter":3009,"ael":3010,"Ġgets":3011,"ĠGoogle":3012,"Ġsn":3013,"iant":3014,"Ġvote":3015,"Ġnearly":3016,"Ġincluded":3017,"Ġrecogn":3018,"zz":3019,"mm":3020,"aled":3021,"Ġhappened":3022,"04":3023,"Ġhot":3024,"Ġwhose":3025,"Ġcivil":3026,"Ġsuff":3027,"oes":3028,"itiz":3029,"ĠSyri":3030,"Ġrespond":3031,"Ġhon":3032,"Ġfeatures":3033,"Ġeconomic":3034,"ĠApril":3035,"rim":3036,"Ġtechnology":3037,"Ġoption":3038,"aging":3039,"Ġpurch":3040,"Re":3041,"Ġlat":3042,"chie":3043,"isl":3044,"Ġrecomm":3045,"uf":3046,"Ġtraining":3047,"Ġeffects":3048,"Ġfast":3049,"Ġ2010":3050,"Ġoccur":3051,"Ġwebsite":3052,"Ġemail":3053,"Ġsens":3054,"ech":3055,"Ġoil":3056,"Ġinflu":3057,"Ġcurrently":3058,"ĠSch":3059,"ĠAdd":3060,"Ġgoal":3061,"Ġscient":3062,"Ġconv":3063,"100":3064,"emy":3065,"Ġdecided":3066,"Ġtravel":3067,"Ġmention":3068,"LL":3069,"03":3070,"Ġelection":3071,"Ġphone":3072,"Ġlooks":3073,"Ġsituation":3074,"Ġcy":3075,"Ġhor":3076,"bed":3077,"ĠCourt":3078,"aily":3079,"aves":3080,"Ġquality":3081,"ĠComp":3082,"wise":3083,"Ġtable":3084,"Ġstaff":3085,"ĠWind":3086,"ett":3087,"Ġtried":3088,"idered":3089,"Ġaddition":3090,"Ġbox":3091,"Ġlack":3092,"arily":3093,"Ġwide":3094,"Ġmid":3095,"Ġboard":3096,"ysis":3097,"Ġanti":3098,"ha":3099,"Ġdig":3100,"ening":3101,"Ġdro":3102,"Con":3103,"68":3104,"Ġslow":3105,"based":3106,"sequ":3107,"Ġpath":3108,"Ex":3109,"aker":3110,"Ġworked":3111,"Ġpen":3112,"Ġengine":3113,"Ġlooked":3114,"ĠSuper":3115,"ĠServ":3116,"Ġvictim":3117,"Un":3118,"Ġproperty":3119,"Ġintrodu":3120,"Ġexecut":3121,"ĠPM":3122,"Le":3123,"Ġcolor":3124,"ĠMore":3125,"Ġ60":3126,"Ġnetwork":3127,"Ġdate":3128,"cul":3129,"idge":3130,"Ġextra":3131,"31":3132,"Ġsle":3133,"67":3134,"Ġwond":3135,"Ġreports":3136,"just":3137,"ĠAustral":3138,"Ġcapital":3139,"Ġens":3140,"Ġcommand":3141,"Ġallowed":3142,"Ġprep":3143,"Ġcapt":3144,"hib":3145,"Ġnumbers":3146,"chan":3147,"Ġfair":3148,"mp":3149,"oms":3150,"Ġreach":3151,"With":3152,"tain":3153,"Ġbroad":3154,"Ġcouple":3155,"ecause":3156,"lying":3157,"ĠFeb":3158,"Ġscreen":3159,"Ġlives":3160,"Ġprior":3161,"ĠCongress":3162,"Ar":3163,"Ġapproach":3164,"Ġemer":3165,"aries":3166,"ĠDis":3167,"serv":3168,"ĠNe":3169,"Ġbuilt":3170,"cies":3171,"Ġrepe":3172,"Ġrules":3173,"force":3174,"ĠPal":3175,"Ġfinancial":3176,"Ġconsidered":3177,"ĠChar":3178,"nces":3179,"ĠIS":3180,"Ġbrought":3181,"Ġbi":3182,"iers":3183,"ĠSim":3184,"OP":3185,"Ġproducts":3186,"Ġvisit":3187,"Ġdocument":3188,"Ġconduct":3189,"Ġcompletely":3190,"ining":3191,"ĠCalif":3192,"ibly":3193,"Ġwritten":3194,"ĠTV":3195,"ements":3196,"Ġdraw":3197,"One":3198,"Ġpublished":3199,"Ġsecret":3200,"rain":3201,"het":3202,"ĠFacebook":3203,"onday":3204,"ĠUp":3205,"Ġsexual":3206,"Ġthous":3207,"ĠPat":3208,"Ġess":3209,"Ġstandard":3210,"Ġarm":3211,"ges":3212,"ection":3213,"Ġfell":3214,"Ġforeign":3215,"ani":3216,"ĠFriday":3217,"Ġregular":3218,"inary":3219,"Ġincreased":3220,"Ġusually":3221,"Ġdemon":3222,"Ġdark":3223,"Ġadditional":3224,"rol":3225,"ĠOf":3226,"Ġproduction":3227,"!!":3228,"undred":3229,"Ġinternational":3230,"idents":3231,"ĠFree":3232,"roup":3233,"Ġrace":3234,"Ġmach":3235,"Ġhuge":3236,"All":3237,"lear":3238,"ovember":3239,"Ġtown":3240,"Ġattention":3241,"ĠOff":3242,"yond":3243,"ĠThen":3244,"field":3245,"Ġterror":3246,"raz":3247,"ĠBo":3248,"Ġmeeting":3249,"ĠPark":3250,"Ġarrest":3251,"Ġfear":3252,"Ġaw":3253,"ĠVal":3254,"oring":3255,"',":3256,"Ġextreme":3257,"arr":3258,"Ġworkers":3259,"After":3260,"Ġ31":3261,"net":3262,"ament":3263,"Ġdirectly":3264,"Ġpopulation":3265,"ube":3266,"ĠOctober":3267,"ĠIN":3268,"ĠJanuary":3269,"59":3270,"ĠDavid":3271,"Ġcross":3272,"cember":3273,"ĠFirst":3274,"Ġmessage":3275,"irit":3276,"Ġnation":3277,"Ġpoll":3278,"isions":3279,"Ġanswer":3280,"ny":3281,"isode":3282,"Ġcarry":3283,"ĠRussia":3284,"Ġhear":3285,"ength":3286,"roy":3287,"Ġnatural":3288,"inally":3289,"Ġdog":3290,"mitted":3291,"Ġtrade":3292,"Ġsubst":3293,"Ġmultiple":3294,"ĠAfric":3295,"Ġfans":3296,"Ġsort":3297,"Ġglobal":3298,"ication":3299,"ĠWed":3300,"ara":3301,"Ġachie":3302,"Ġlanguage":3303,"vey":3304,"Ġtal":3305,"Ġnecessary":3306,"Ġdetails":3307,"Ġsen":3308,"ĠSund":3309,"ĠReg":3310,"ĠRec":3311,"06":3312,"Ġsil":3313,"ressive":3314,"Ġmedical":3315,"unch":3316,"ornia":3317,"Ġund":3318,"fort":3319,"ocks":3320,"ĠMonday":3321,"uesday":3322,"craft":3323,"77":3324,"urt":3325,"Ġver":3326,"ĠHill":3327,"Ġreceive":3328,"Ġmorning":3329,"estern":3330,"Ġbank":3331,"Ġsat":3332,"irth":3333,"ĠHigh":3334,"Ġdevice":3335,"ĠTHE":3336,"ĠCenter":3337,"Ġsafe":3338,"Ġple":3339,"ĠCanada":3340,"Ġsystems":3341,"Ġassist":3342,"Ġsurv":3343,"Ġbattle":3344,"ĠSoc":3345,"vertis":3346,"She":3347,"Ġpaper":3348,"Ġgrowth":3349,"Ġcast":3350,"Sc":3351,"Ġplans":3352,"lled":3353,"Ġparts":3354,"Ġwall":3355,"Ġmovement":3356,"Ġpractice":3357,"imately":3358,"Ġdisplay":3359,"Ġsometimes":3360,"omp":3361,"ĠPaul":3362,"ĠYes":3363,"king":3364,"58":3365,"oly":3366,"Ġson":3367,"Ġavoid":3368,"okes":3369,"ĠJew":3370,"Ġtowards":3371,"asc":3372,"Ġ//":3373,"ĠKore":3374,"Ġtalking":3375,"Ġcorrect":3376,"Ġspent":3377,"icks":3378,"iable":3379,"eared":3380,"Ġterm":3381,"Ġwants":3382,"oming":3383,"Ġut":3384,"Ġdoub":3385,"Ġforces":3386,"Ġplease":3387,"69":3388,"ĠNovember":3389,"atform":3390,"ondon":3391,"Ġones":3392,"Ġimmediately":3393,"ĠRussian":3394,"ĠMet":3395,"Ġdeg":3396,"Ġparents":3397,"CH":3398,"ĠAmericans":3399,"aly":3400,"ĠMod":3401,"Ġshown":3402,"Ġconditions":3403,"Ġstuff":3404,"Ġreb":3405,"ĠYour":3406,"Ġincludes":3407,"nown":3408,"ĠSam":3409,"Ġexperien":3410,"mission":3411,"ĠEven":3412,"aught":3413,"Ġannounced":3414,"ĠRepublican":3415,"Ġdetermin":3416,"Ġdescribed":3417,"ĠCounty":3418,"()":3419,"Ġdoor":3420,"Ġchanged":3421,"Ġneigh":3422,"ĠHere":3423,"Ġclean":3424,"Ġpan":3425,"ĠDecember":3426,"ĠEuropean":3427,"iring":3428,"apter":3429,"Ġclub":3430,"ĠTuesday":3431,"Ġpaid":3432,"ĠNet":3433,"Ġattacks":3434,"Ġcharacters":3435,"Ġalone":3436,"Ġdirector":3437,"dom":3438,"Ġ35":3439,"Ġload":3440,"Ġrout":3441,"ĠCalifornia":3442,"Ġfinally":3443,"Ġrac":3444,"Ġcontr":3445,"Ġexactly":3446,"resh":3447,"pri":3448,"ĠIslam":3449,"Ġnature":3450,"Ġcareer":3451,"Ġlatest":3452,"Ġconvers":3453,"ĠSl":3454,"pose":3455,"cient":3456,"ĠInc":3457,"ivity":3458,"88":3459,"ĠAtt":3460,"ĠMor":3461,"nesday":3462,"Ġweight":3463,"ken":3464,"Ġnote":3465,"Ġteams":3466,"Ġ\\":3467,"airs":3468,"ĠGreen":3469,"Ġhundred":3470,"onent":3471,"Ġstreng":3472,"Ġconsist":3473,"icated":3474,"Ġregul":3475,"Ġlic":3476,"astic":3477,"Ġten":3478,"ursday":3479,"elligence":3480,"ously":3481,"ĠUK":3482,"BI":3483,"Ġcosts":3484,"Ġindepend":3485,"ĠAP":3486,"Ġnormal":3487,"Ġhom":3488,"Ġobvious":3489,"Ġswe":3490,"Ġstar":3491,"Ġready":3492,"acher":3493,"Ġimplement":3494,"gest":3495,"Ġsong":3496,"ĠGet":3497,"ĠLab":3498,"Ġinteresting":3499,"using":3500,"Ġgiving":3501,"ĠSunday":3502,"Ġetc":3503,"Ġmiddle":3504,"Ġremember":3505,"right":3506,"osition":3507,"utions":3508,"Ġmax":3509,"46":3510,"Ġyourself":3511,"Ġdemand":3512,"Ġtreatment":3513,"Ġdanger":3514,"ĠCons":3515,"Ġguy":3516,"ĠBritish":3517,"Ġphysical":3518,"Ġrelated":3519,"Ġremain":3520,"Ġcouldn":3521,"Ġrefer":3522,"Ġcitiz":3523,"box":3524,"ENT":3525,"board":3526,"Ġinn":3527,"IG":3528,"ero":3529,"ĠStreet":3530,"ospital":3531,"rench":3532,"chers":3533,"Ġstra":3534,"OL":3535,"ager":3536,"ĠAN":3537,"Ġeasily":3538,"IA":3539,"enge":3540,"iny":3541,"Ġclos":3542,"ocked":3543,"Ġuses":3544,"ĠCoun":3545,"Im":3546,"uild":3547,"??":3548,"more":3549,"Ġang":3550,"Ġwrite":3551,"olute":3552,"57":3553,"Ġleader":3554,"Ġreading":3555,"</":3556,"Ġautom":3557,"ests":3558,"43":3559,"Ġlegisl":3560,"ĠGold":3561,"Ġdesigned":3562,"ĠST":3563,"ĠLeg":3564,"ares":3565,"Ġbeaut":3566,"ĠTex":3567,"Ġappears":3568,"Ġstrugg":3569,"ĠRom":3570,"Ġ00":3571,"Ġchoice":3572,"Ġparticularly":3573,"ĠFrom":3574,"oper":3575,"ĠLondon":3576,"anned":3577,"Ġallows":3578,"obile":3579,"Ġdifference":3580,"âĢ¢":3581,"ĠView":3582,"ĠWednesday":3583,"Ġalthough":3584,"Ġrelative":3585,"Ġapplication":3586,"atever":3587,"Ġaren":3588,"Ġmyself":3589,"Ġimag":3590,"Ġdise":3591,"Ġsociety":3592,"Ġfrequ":3593,"ĠEnglish":3594,"Ġpoor":3595,"ĠDay":3596,"Ġwriting":3597,"Ġseven":3598,"Ġstarting":3599,"Ġbud":3600,"Ġprint":3601,"ĠTrans":3602,"ufact":3603,"ĠStud":3604,"new":3605,"Ġcrim":3606,"Ġgives":3607,"Ġcool":3608,"ae":3609,"iance":3610,"ĠGeneral":3611,"Ġthinking":3612,"Ġsave":3613,"Ġlimited":3614,"ĠParty":3615,"Ġmeaning":3616,"pen":3617,"owers":3618,"ĠJack":3619,"EM":3620,"Ġnice":3621,"rupt":3622,"Ġgas":3623,"Ġeight":3624,"Ġfeet":3625,"Ġeffort":3626,"Ġign":3627,"icit":3628,"Bl":3629,"coin":3630,"Ġopin":3631,"Ġbrain":3632,"While":3633,"hest":3634,"ĠThursday":3635,"Ġwouldn":3636,"aughter":3637,"Ġtouch":3638,"lements":3639,"Ġstudies":3640,"Ġcenter":3641,"cont":3642,"orge":3643,"Ġcomputer":3644,"Ġinvestigation":3645,"Pl":3646,"orks":3647,"Ġ2008":3648,"Ġincreasing":3649,"Ġstore":3650,"Ġcomments":3651,"Ġbal":3652,"men":3653,"Ġdoll":3654,"Ġliber":3655,"Ġwife":3656,"Ġlaws":3657,"aturday":3658,"itness":3659,"Ġmodern":3660,"ĠSk":3661,"Ġadministration":3662,"Ġopportunity":3663,"Ġsal":3664,"Ġpowerful":3665,"My":3666,"Ġclaims":3667,"ĠEarth":3668,"ords":3669,"Ġtitle":3670,"Ġesc":3671,"name":3672,"Not":3673,"omen":3674,"Ġbeyond":3675,"Ġcamer":3676,"Ġsell":3677,"itute":3678,"earch":3679,"Ġappl":3680,"iment":3681,"42":3682,"ĠArt":3683,"Ġunf":3684,"Ġviolence":3685,"urg":3686,"ĠEast":3687,"Ġcompared":3688,"Ġoptions":3689,"Ġthroughout":3690,"Ġvs":3691,"igr":3692,".[":3693,"aches":3694,"78":3695,"Ġfiles":3696,"FL":3697,"EL":3698,"arian":3699,"ĠJames":3700,"ĠAir":3701,"anch":3702,"Ġdetail":3703,"Ġpiece":3704,"PS":3705,"Ġnamed":3706,"Ġeducation":3707,"Ġdrive":3708,"Ġitems":3709,"Ġstudent":3710,"iced":3711,"::":3712,"ico":3713,"Ġthrow":3714,"Ġscene":3715,"Ġcomplex":3716,"Ġ2009":3717,"Ġprec":3718,"ĠBre":3719,"79":3720,"Ġconcept":3721,"Ġstatus":3722,"aming":3723,"Ġdied":3724,"Ġknowledge":3725,"Ġbeginning":3726,"OD":3727,"ruary":3728,"Ġcertainly":3729,"Ġguys":3730,"Ġslight":3731,"inn":3732,"ounds":3733,"Ġfine":3734,"Ġfat":3735,"ications":3736,"Ġperhaps":3737,"ĠAnt":3738,"Ġincome":3739,"Ġhttps":3740,"Ġmajority":3741,"ports":3742,"ston":3743,"Ġgreater":3744,"Ġfeed":3745,"entially":3746,"Ġsafety":3747,"Ġunique":3748,"andom":3749,"Ġgone":3750,"Ġshowed":3751,"Ġhistor":3752,"Ġcounter":3753,"ius":3754,"ida":3755,"Ġleading":3756,"ipe":3757,"Ġsend":3758,"ĠDonald":3759,"erve":3760,"Ġdefense":3761,"inese":3762,"Ġyes":3763,"ĠFire":3764,"ĠMuslim":3765,"raq":3766,"Ġcontinued":3767,"osh":3768,"Ġprovides":3769,"Ġprison":3770,"ĠPre":3771,"Ġhappy":3772,"Ġeconomy":3773,"Ġtrust":3774,"ags":3775,"ĠGame":3776,"Ġweapons":3777,"uman":3778,"ĠCle":3779,"itation":3780,"Ġanalysis":3781,"ĠTimes":3782,"Ġscience":3783,"->":3784,"Ġfigure":3785,"Ġdisapp":3786,"enty":3787,"Ġsoftware":3788,"Ġult":3789,"Ġofficers":3790,"New":3791,"Is":3792,"Ġremains":3793,"ĠIndia":3794,"Ġpsych":3795,"rief":3796,"Ġcat":3797,"esc":3798,"Ġobserv":3799,"Ġstage":3800,"ĠDark":3801,"Ġenter":3802,"change":3803,"Ġpassed":3804,"Ġdespite":3805,"ĠOut":3806,"Ġmovie":3807,"rs":3808,"Ġvoice":3809,"mine":3810,"ĠPlay":3811,"Ġtoward":3812,"ĠTer":3813,"Ġregion":3814,"Ġvalues":3815,"orters":3816,"Ġmount":3817,"Ġofficer":3818,"ĠOther":3819,"ban":3820,"Ġhous":3821,"wood":3822,"room":3823,"IV":3824,"ĠSun":3825,"see":3826,"ĠOver":3827,"rog":3828,"90":3829,"Ġlay":3830,"ĠTur":3831,"awn":3832,"Ġpressure":3833,"ĠSub":3834,"Ġbooks":3835,"edom":3836,"ĠSand":3837,"AA":3838,"ago":3839,"Ġreasons":3840,"ford":3841,"Ġactivity":3842,"UT":3843,"Now":3844,"ĠSenate":3845,"cell":3846,"night":3847,"Ġcalls":3848,"inter":3849,"Ġletter":3850,"ĠRob":3851,"ĠJe":3852,"Ġchoose":3853,"ĠLaw":3854,"Get":3855,"Be":3856,"Ġrob":3857,"Ġtypes":3858,"Ġplatform":3859,"Ġquarter":3860,"RA":3861,"ĠTime":3862,"Ġmaybe":3863,"ĠCr":3864,"95":3865,"pre":3866,"Ġmoving":3867,"Ġlif":3868,"Ġgold":3869,"Ġsom":3870,"Ġpatients":3871,"Ġtruth":3872,"ĠKe":3873,"urance":3874,"antly":3875,"mar":3876,"Ġcharge":3877,"ĠGreat":3878,"Ġcele":3879,"--------------------------------":3880,"Ġrock":3881,"roid":3882,"ancy":3883,"Ġcredit":3884,"aud":3885,"By":3886,"ĠEvery":3887,"Ġmoved":3888,"inger":3889,"ribution":3890,"Ġnames":3891,"Ġstraight":3892,"ĠHealth":3893,"ĠWell":3894,"Ġfeature":3895,"Ġrule":3896,"Ġsche":3897,"inated":3898,"ĠMichael":3899,"berg":3900,"41":3901,"iled":3902,"band":3903,"Ġclick":3904,"ĠAngel":3905,"onents":3906,"ÂŃ":3907,"ĠIraq":3908,"ĠSaturday":3909,"Ġaware":3910,"part":3911,"Ġpattern":3912,"OW":3913,"ĠLet":3914,"Ġgrad":3915,"igned":3916,"Ġassociated":3917,"Ġstyle":3918,"no":3919,"iation":3920,"aith":3921,"ilies":3922,"Ġstories":3923,"uration":3924,"Ġindividuals":3925,"ĠâĢ¦":3926,"miss":3927,"ĠAssoci":3928,"ishing":3929,"aby":3930,"Ġsummer":3931,"ĠBen":3932,"Ġ32":3933,"Ġarch":3934,"uty":3935,"ĠTexas":3936,"hol":3937,"Ġfully":3938,"Ġmill":3939,"Ġfollowed":3940,"ĠBill":3941,"ĠIndian":3942,"ĠSecret":3943,"ĠBel":3944,"ĠFebruary":3945,"Ġjobs":3946,"Ġseemed":3947,"ĠGovern":3948,"ipped":3949,"Ġreality":3950,"Ġlines":3951,"Ġpark":3952,"Ġmeasure":3953,"ĠOur":3954,"IM":3955,"Ġbrother":3956,"Ġgrowing":3957,"Ġban":3958,"Ġestim":3959,"Ġcry":3960,"ĠSchool":3961,"Ġmechan":3962,"ĠOF":3963,"ĠWindows":3964,"Ġrates":3965,"ĠOh":3966,"Ġpositive":3967,"Ġculture":3968,"istics":3969,"ica":3970,"Ġhar":3971,"ya":3972,"itely":3973,"ipp":3974,"Ġmap":3975,"encies":3976,"ĠWilliam":3977,"II":3978,"akers":3979,"56":3980,"ĠMart":3981,"ĠRem":3982,"Ġaltern":3983,"itude":3984,"Ġcoach":3985,"rowd":3986,"Don":3987,"Ġkids":3988,"Ġjournal":3989,"Ġcorpor":3990,"Ġfalse":3991,"Ġweb":3992,"Ġsleep":3993,"Ġcontain":3994,"Ġsto":3995,"Ġbed":3996,"iverse":3997,"ĠRich":3998,"ĠChinese":3999,"Ġpun":4000,"Ġmeant":4001,"known":4002,"Ġnotice":4003,"Ġfavorite":4004,"aven":4005,"Ġcondition":4006,"Ġpurpose":4007,"))":4008,"Ġorganization":4009,"Ġchalleng":4010,"Ġmanufact":4011,"Ġsusp":4012,"ĠAc":4013,"Ġcritic":4014,"unes":4015,"uclear":4016,"Ġmer":4017,"vention":4018,"Ġ80":4019,"Ġmist":4020,"ĠUs":4021,"ĠTor":4022,"http":4023,"olf":4024,"Ġlarger":4025,"Ġadvant":4026,"Ġresear":4027,"Ġactions":4028,"ml":4029,"Ġkept":4030,"Ġaim":4031,",'":4032,"col":4033,"Ġbenefits":4034,"ifying":4035,"Ġactual":4036,"ĠInternational":4037,"Ġvehicle":4038,"Ġchief":4039,"Ġefforts":4040,"ĠLeague":4041,"ĠMost":4042,"Ġwait":4043,"Ġadult":4044,"Ġoverall":4045,"Ġspeech":4046,"Ġhighly":4047,"Ġfemale":4048,"Ġerror":4049,"Ġeffective":4050,"54":4051,"Ġencour":4052,"well":4053,"Ġfailed":4054,"Ġconserv":4055,"Ġprograms":4056,"Ġtrou":4057,"Ġahead":4058,"500":4059,"vertisement":4060,"IP":4061,"ĠFound":4062,"pir":4063,"Ġ%":4064,"Ġcrime":4065,"ander":4066,"Ġlocation":4067,"ĠIran":4068,"Ġbehavior":4069,"azing":4070,"Ġrare":4071,"Ġemb":4072,"Ġcaused":4073,"Ġship":4074,"Ġactive":4075,"Ġcontribut":4076,"Ġgreen":4077,"Ġacqu":4078,"Ġreflect":4079,"venue":4080,"Ġfirm":4081,"Ġbirth":4082,"].":4083,"Ġclearly":4084,"Ġemot":4085,"Ġagency":4086,"riage":4087,"Ġmemory":4088,"98":4089,"SA":4090,"ĠSee":4091,"acing":4092,"CC":4093,"Ġbiggest":4094,"Ġrap":4095,"Ġbasic":4096,"Ġband":4097,"eat":4098,"Ġsuspect":4099,"ĠMac":4100,"Ġ90":4101,"mark":4102,"istan":4103,"Ġspread":4104,"ams":4105,"ki":4106,"asy":4107,"rav":4108,"ĠRober":4109,"Ġdemonstr":4110,"rated":4111,"Ġabsolute":4112,"Ġplaces":4113,"Ġimpl":4114,"ibrary":4115,"Ġcards":4116,"Ġdestroy":4117,"Ġvirt":4118,"vere":4119,"Ġappeared":4120,"yan":4121,"point":4122,"Ġbeg":4123,"Ġtemper":4124,"spe":4125,"anted":4126,"ears":4127,"ĠDirect":4128,"Ġlength":4129,"Ġblog":4130,"amb":4131,"Ġinteg":4132,"Ġresources":4133,"acc":4134,"iful":4135,"Ġspot":4136,"Ġforced":4137,"Ġthousands":4138,"ĠMinister":4139,"Ġqual":4140,"ĠFrench":4141,"atically":4142,"Ġgenerally":4143,"Ġdrink":4144,"Ġthus":4145,"IL":4146,"odes":4147,"Ġappropri":4148,"ĠRead":4149,"Ġwhom":4150,"Ġeye":4151,"Ġcollege":4152,"Ġ45":4153,"irection":4154,"Ġensure":4155,"Ġapparent":4156,"iders":4157,"Ġreligious":4158,"Ġminor":4159,"olic":4160,"Ġtro":4161,"ĠWhy":4162,"ribute":4163,"met":4164,"Ġprimary":4165,"Ġdeveloped":4166,"Ġpeace":4167,"Ġskin":4168,"ste":4169,"ava":4170,"Ġblue":4171,"Ġfamilies":4172,"Ġir":4173,"Ġapply":4174,"Ġinform":4175,"ĠSmith":4176,"CT":4177,"ii":4178,"Ġlimit":4179,"Ġresist":4180,"................":4181,"umn":4182,"Ġconflic":4183,"Ġtwe":4184,"udd":4185,"ĠTom":4186,"Ġliter":4187,"que":4188,"bon":4189,"Ġhair":4190,"Ġeventually":4191,"Ġpus":4192,"Ġhelped":4193,"Ġagg":4194,"orney":4195,"ĠApple":4196,"Ġfit":4197,"ĠSur":4198,"Ġprem":4199,"Ġsales":4200,"Ġseconds":4201,"Ġstrength":4202,"Ġfeeling":4203,"¿½":4204,"Ġtour":4205,"Ġknows":4206,"oom":4207,"Ġexerc":4208,"Ġsomew":4209,"�":4210,">>":4211,"Ġspokes":4212,"Ġideas":4213,"Ġregist":4214,"soft":4215,"ĠDel":4216,"ĠPC":4217,"Ġpropos":4218,"Ġlaunch":4219,"Ġbottom":4220,"TH":4221,"ĠPlease":4222,"vest":4223,"itz":4224,"ĠInter":4225,"Ġscript":4226,"Ġrat":4227,"arning":4228,"Ġil":4229,"ĠJer":4230,"ĠAre":4231,"Ġwhatever":4232,"oken":4233,"cience":4234,"Ġmode":4235,"Ġagree":4236,"Ġsources":4237,"Ġinitial":4238,"Ġrestrict":4239,"Ġwonder":4240,"usion":4241,"####":4242,"ĠSil":4243,"ville":4244,"Ġburn":4245,"tw":4246,"asion":4247,"Ġ£":4248,"Ġnor":4249,"uing":4250,"Ġreached":4251,"Ġsun":4252,"Ġcateg":4253,"igration":4254,"Ġcook":4255,"Ġpromot":4256,"Ġmale":4257,"Ġclimate":4258,"Ġfix":4259,"Ġalleged":4260,"UR":4261,"alled":4262,"Ġimages":4263,"Cont":4264,"ota":4265,"Ġschools":4266,"ios":4267,"Ġdrop":4268,"Ġstream":4269,"ĠMo":4270,"Ġpreviously":4271,"aling":4272,"Ġpet":4273,"Ġdouble":4274,"Ġ(@":4275,"annel":4276,"Ġdefault":4277,"ties":4278,"Ġrank":4279,"ĠDec":4280,"ĠCouncil":4281,"Ġweapon":4282,"Ġstock":4283,"Ġanaly":4284,"ĠStr":4285,"Ġpicture":4286,"ĠPolice":4287,"ference":4288,"Ġcentury":4289,"Ġcitizens":4290,"Ġonto":4291,"Ġexpand":4292,"Ġhero":4293,"ĠSol":4294,"Ġwild":4295,"Ġupdate":4296,"Ġcustomers":4297,"ront":4298,"def":4299,"Ġlik":4300,"Ġcriminal":4301,"ĠChristian":4302,"SP":4303,"76":4304,"Ġleaving":4305,"Ġotherwise":4306,"ĠDist":4307,"Ġbasis":4308,"52":4309,"53":4310,"icip":4311,"ĠBer":4312,"Ġrecommend":4313,"Ġfloor":4314,"Ġcrowd":4315,"oles":4316,"Ġ70":4317,"Ġcentral":4318,"ĠEv":4319,"Ġdream":4320,"Ġdownload":4321,"Ġconfir":4322,"ĠThom":4323,"Ġwindow":4324,"Ġhappens":4325,"Ġunit":4326,"Ġtend":4327,"Ġspl":4328,"Ġbecomes":4329,"Ġfighting":4330,"Ġpredict":4331,"ĠPress":4332,"ĠPower":4333,"Ġheavy":4334,"aked":4335,"Ġfan":4336,"orter":4337,"ategy":4338,"BA":4339,"izes":4340,"Ġspend":4341,"Here":4342,"Ġ2007":4343,"Ġadop":4344,"ĠHam":4345,"Ġfootball":4346,"ĠPort":4347,"oday":4348,"51":4349,"ampions":4350,"Ġtransfer":4351,"ht":4352,"Ġ38":4353,"term":4354,"acity":4355,"Ġbur":4356,"],":4357,"ternal":4358,"rig":4359,"but":4360,"Ġtherefore":4361,"ĠBecause":4362,"resp":4363,"rey":4364,"Ġmission":4365,"Some":4366,"Ġnoted":4367,"Ġassum":4368,"Ġdisease":4369,"Ġedit":4370,"Ġprogress":4371,"rd":4372,"ĠBrown":4373,"ocal":4374,"Ġadding":4375,"Ġraised":4376,"ĠAny":4377,"Ġtick":4378,"Ġseeing":4379,"ĠPeople":4380,"Ġagreement":4381,"Ġserver":4382,"Ġwat":4383,"Ġdebate":4384,"Ġsupposed":4385,"iling":4386,"Ġlargest":4387,"Ġsuccessful":4388,"ĠPri":4389,"ĠDemocratic":4390,"Ġjump":4391,"ĠSyria":4392,"Ġowners":4393,"Ġoffers":4394,"Ġshooting":4395,"Ġeffic":4396,"sey":4397,"Ġhaven":4398,"verse":4399,"tered":4400,"ĠLight":4401,"imal":4402,"ĠBig":4403,"Ġdefend":4404,"Ġbeat":4405,"Ġrecords":4406,"%)":4407,"Ġscen":4408,"Ġemployees":4409,"Ġdevices":4410,"hem":4411,"Ġcommer":4412,"ĠMex":4413,"Ġbenefit":4414,"ĠProf":4415,"Ġilleg":4416,"Ġsurface":4417,"ĠAlso":4418,"Ġharm":4419,"ingly":4420,"wide":4421,"ĠAlex":4422,"Ġshut":4423,"ĠCur":4424,"Ġlose":4425,"pm":4426,"Ġchallenge":4427,"semb":4428,"Ġstation":4429,"Ġintelligence":4430,"Ġaccur":4431,"ĠFlor":4432,"Ġrequires":4433,"ĠMal":4434,"bum":4435,"Ġhospital":4436,"Ġspirit":4437,"Ġoffered":4438,"Ġproduce":4439,"ĠCommun":4440,"Ġcreating":4441,"Ġcris":4442,"spect":4443,"Ġended":4444,"Ġdaily":4445,"Ġvoters":4446,"lands":4447,"ias":4448,"ih":4449,"ona":4450,"Ġsmart":4451,"ĠOffice":4452,"ĠLord":4453,"rial":4454,"ĠInternet":4455,"Ġcircum":4456,"Ġextremely":4457,"'.":4458,"Ġopinion":4459,"ĠMil":4460,"Ġgain":4461,"BS":4462,"ĠFin":4463,"yp":4464,"Ġuseful":4465,"Ġbudget":4466,"Ġcomfort":4467,"isf":4468,"Ġbackground":4469,"eline":4470,"Ġepisode":4471,"Ġenemy":4472,"Ġtrial":4473,"Ġestablish":4474,"date":4475,"ĠCap":4476,"Ġcontinues":4477,"Ġshowing":4478,"ĠUnion":4479,"with":4480,"Ġposted":4481,"ĠSystem":4482,"Ġeat":4483,"rian":4484,"Ġrise":4485,"ĠGermany":4486,"ils":4487,"Ġsigned":4488,"Ġvill":4489,"Ġgrand":4490,"mor":4491,"ĠEngland":4492,"Ġprojects":4493,"umber":4494,"Ġconference":4495,"za":4496,"Ġresponsible":4497,"ĠArab":4498,"Ġlearned":4499,"âĢĶâĢĶ":4500,"ipping":4501,"ĠGeorge":4502,"OC":4503,"Ġreturned":4504,"ĠAustralia":4505,"Ġbrief":4506,"Qu":4507,"Ġbrand":4508,"illing":4509,"abled":4510,"Ġhighest":4511,"Ġtrain":4512,"ĠCommission":4513,"while":4514,"Ġnom":4515,"ception":4516,"Ġmut":4517,"ĠBlue":4518,"Ġincident":4519,"vant":4520,"86":4521,"ĠID":4522,"Ġnuclear":4523,"74":4524,"ĠLike":4525,"ĠRE":4526,"ĠMicro":4527,"li":4528,"mail":4529,"Ġcharges":4530,"89":4531,"Ġadjust":4532,"ado":4533,"Ġearth":4534,"NA":4535,"Ġprices":4536,"PA":4537,"Ġdraft":4538,"Ġruns":4539,"Ġcandidate":4540,"enses":4541,"Ġmanagement":4542,"ĠPhil":4543,"ĠMiss":4544,"Ġteach":4545,"gram":4546,"Ġunderstanding":4547,"ait":4548,"icago":4549,"Add":4550,"ĠEp":4551,"secut":4552,"Ġseparate":4553,"Ġinstance":4554,"Ġeth":4555,"Ġunless":4556,"********":4557,"ĠFore":4558,"inate":4559,"Ġoperations":4560,"Sp":4561,"Ġfaith":4562,"gar":4563,"ĠChurch":4564,"ronic":4565,"Ġconfig":4566,"osure":4567,"Ġactivities":4568,"Ġtraditional":4569,"Ġ36":4570,"Ġdirection":4571,"Ġmachine":4572,"Ġsurround":4573,"Ġpush":4574,"unction":4575,"ĠEU":4576,"Ġeasier":4577,"Ġargument":4578,"GB":4579,"Ġmicro":4580,"Ġspending":4581,"izations":4582,"Ġtheory":4583,"adow":4584,"Ġcalling":4585,"ĠLast":4586,"Ġder":4587,"Ġinfluence":4588,"Ġcommit":4589,"Ġphoto":4590,"Ġunc":4591,"istry":4592,"gn":4593,"aste":4594,"acks":4595,"Ġdisp":4596,"ady":4597,"do":4598,"ĠGood":4599,"Ġ`":4600,"Ġwish":4601,"Ġrevealed":4602,"³³":4603,"lig":4604,"Ġenforce":4605,"ĠCommittee":4606,"Ġchem":4607,"Ġmiles":4608,"Ġinterested":4609,"Ġsolution":4610,"icy":4611,"inct":4612,"Ġ->":4613,"ĠDet":4614,"Ġremoved":4615,"Ġcompar":4616,"eah":4617,"Ġplant":4618,"ĠSince":4619,"Ġachieve":4620,"Ġadvantage":4621,"Ġslightly":4622,"bing":4623,"Ġplaced":4624,"under":4625,"2015":4626,"ĠMad":4627,"Ġtim":4628,"oses":4629,"Ġcru":4630,"ĠRock":4631,"Ġmostly":4632,"Ġnegative":4633,"Ġsetting":4634,"Ġproduced":4635,"Ġmur":4636,"Ġconnection":4637,"ĠMer":4638,"Ġdriver":4639,"Ġexecutive":4640,"Ġassault":4641,"Ġborn":4642,"ĠVer":4643,"tained":4644,"Ġstructure":4645,"Ġreduce":4646,"Ġdecades":4647,"Ġded":4648,"uke":4649,"ĠMany":4650,"idden":4651,"Ġleague":4652,"Se":4653,"Ġjoin":4654,"Ġdisco":4655,"Ġdie":4656,"cks":4657,"actions":4658,"Ġassess":4659,"agn":4660,"Ġgoals":4661,"ours":4662,"IR":4663,"Ġsenior":4664,"iller":4665,"mod":4666,"ipment":4667,"ocol":4668,"uy":4669,"ĠQue":4670,"Ġparties":4671,"irgin":4672,"Ġlearning":4673,"itable":4674,"Ġstreet":4675,"Ġcamera":4676,"App":4677,"Ġskills":4678,"bre":4679,"cious":4680,"Ġcelebr":4681,"ĠFranc":4682,"Ġexisting":4683,"Ġwilling":4684,"lor":4685,"Ġid":4686,"ĠSpace":4687,"Ġcritical":4688,"ĠLa":4689,"ortunately":4690,"Ġserve":4691,"Ġcold":4692,"Ġspecies":4693,"TS":4694,"Ġanimals":4695,"ĠBay":4696,"Ġolder":4697,"ĠUnder":4698,"estic":4699,"ĠTre":4700,"Ġteacher":4701,"Ġprefer":4702,"vis":4703,"Ġthread":4704,"ĠMatt":4705,"Ġmanager":4706,"ãĥ»":4707,"Ġprofessional":4708,"ĠVol":4709,"Ġnotes":4710,"These":4711,"ula":4712,"Ġfresh":4713,"ented":4714,"uzz":4715,"edy":4716,"clusion":4717,"ĠRel":4718,"Ġdoubt":4719,"EO":4720,"Ġopened":4721,"ĠBit":4722,"Advertisement":4723,"Ġguess":4724,"ĠUN":4725,"Ġsequ":4726,"Ġexplain":4727,"otten":4728,"Ġattract":4729,"aks":4730,"Ġstring":4731,"Ġcontext":4732,"ossible":4733,"ĠRepublicans":4734,"Ġsolid":4735,"Ġcities":4736,"Ġasking":4737,"Ġrandom":4738,"ups":4739,"uries":4740,"arant":4741,"dden":4742,"gl":4743,"ĠFlorida":4744,"Ġdepend":4745,"ĠScott":4746,"Ġ33":4747,"ĠiT":4748,"icon":4749,"Ġmentioned":4750,"Ġ2000":4751,"Ġclaimed":4752,"Ġdefinitely":4753,"ulf":4754,"Ġcore":4755,"Ġopening":4756,"ĠConst":4757,"which":4758,"ĠTra":4759,"AG":4760,"72":4761,"Ġbelieved":4762,"ada":4763,"Ġ48":4764,"ĠSecurity":4765,"yright":4766,"ĠPet":4767,"ĠLou":4768,"Ġholding":4769,"================":4770,"Ġice":4771,"Ġbrow":4772,"Ġauthorities":4773,"host":4774,"word":4775,"Ġscore":4776,"ĠDiv":4777,"Ġcells":4778,"Ġtransl":4779,"Ġneighbor":4780,"Ġremove":4781,"uct":4782,"Ġdistrict":4783,"ĠAccording":4784,"Ġworse":4785,"Ġconcerns":4786,"Ġpresidential":4787,"Ġpolicies":4788,"ĠHall":4789,"73":4790,"Ġhus":4791,"AY":4792,"Ġ2006":4793,"ĠJud":4794,"Ġindependent":4795,"ĠJustice":4796,"iliar":4797,"print":4798,"ighter":4799,"Ġprotection":4800,"zen":4801,"Ġsudden":4802,"house":4803,"ĠJes":4804,"PR":4805,"ĠInf":4806,"Ġbul":4807,"Ġ_":4808,"ĠService":4809,"ĠPR":4810,"Ġstrategy":4811,"ffect":4812,"Ġgirls":4813,"Ġmissing":4814,"oyal":4815,"ĠTeam":4816,"ulated":4817,"Ġdat":4818,"Ġpolitics":4819,"abor":4820,"According":4821,"Ġspell":4822,"Ġgraph":4823,"orthern":4824,"TC":4825,"Ab":4826,"Ġlabor":4827,"isher":4828,"Ġkick":4829,"ĠiTunes":4830,"Ġsteps":4831,"poses":4832,"Ġsmaller":4833,"En":4834,"bert":4835,"Ġroll":4836,"Ġresearchers":4837,"Ġclosed":4838,"Ġtransport":4839,"Ġlawy":4840,"________________":4841,"ĠChicago":4842,"Ġaspect":4843,"Ġnone":4844,"Ġmarriage":4845,"96":4846,"Ġelements":4847,"ĠFre":4848,"ĠSal":4849,"Ġdram":4850,"FC":4851,"top":4852,"equ":4853,"Ġhearing":4854,"Ġsupported":4855,"Ġtesting":4856,"cohol":4857,"Ġmassive":4858,"Ġstick":4859,"Ġguard":4860,"isco":4861,"phone":4862,"From":4863,"However":4864,"Ġborder":4865,"Ġcopy":4866,"ography":4867,"list":4868,"71":4869,"Ġowner":4870,"class":4871,"ruit":4872,"rate":4873,"ĠOnce":4874,"Ġdigital":4875,"Ġtask":4876,"ERS":4877,"Ġincred":4878,"tes":4879,"++":4880,"ĠFrance":4881,"Ġbreat":4882,"owl":4883,"Ġissued":4884,"ĠWestern":4885,"Ġdetect":4886,"Ġpartners":4887,"Ġshared":4888,"ĠCall":4889,"Ġcancer":4890,"ache":4891,"ribe":4892,"Ġexplained":4893,"Ġheat":4894,"{\"":4895,"Ġinvestment":4896,"ĠBook":4897,"Ġwood":4898,"Ġtools":4899,"ĠAlthough":4900,"Ġbelief":4901,"Ġcrisis":4902,"Ġge":4903,"ĠMP":4904,"Ġoperation":4905,"type":4906,"~~":4907,"ga":4908,"Ġcontains":4909,"anta":4910,"Ġexpress":4911,"ĠGroup":4912,"ĠJournal":4913,"ka":4914,"Ġamb":4915,"ĠUSA":4916,"Ġfinding":4917,"Ġfunding":4918,"how":4919,"Ġestablished":4920,"ideos":4921,"Ġdegree":4922,"Ġdangerous":4923,"anging":4924,"Ġfreedom":4925,"pport":4926,"outhern":4927,"Ġchurch":4928,"Ġcatch":4929,"ĠTwo":4930,"Ġpresence":4931,"ĠGuard":4932,"Up":4933,"Ġauthority":4934,"ĠProject":4935,"Ġbutton":4936,"Ġconsequ":4937,"Ġvalid":4938,"Ġweak":4939,"Ġstarts":4940,"Ġreference":4941,"ĠMem":4942,"\")":4943,"UN":4944,"orage":4945,"ĠOpen":4946,"Ġcollection":4947,"ym":4948,"gency":4949,"Ġbeautiful":4950,"ros":4951,"Ġtells":4952,"Ġwaiting":4953,"nel":4954,"Ġproviding":4955,"ĠDemocrats":4956,"Ġdaughter":4957,"Ġmaster":4958,"Ġpurposes":4959,"ĠJapanese":4960,"Ġequal":4961,"Ġturns":4962,"Ġdocuments":4963,"Ġwatching":4964,"Res":4965,"Ġran":4966,"2014":4967,"Ġreject":4968,"ĠKorea":4969,"Ġvictims":4970,"Level":4971,"erences":4972,"Ġwitness":4973,"Ġ34":4974,"Ġreform":4975,"coming":4976,"Ġoccup":4977,"Ġcaught":4978,"Ġtraffic":4979,"ading":4980,"Ġmodels":4981,"ario":4982,"Ġserved":4983,"Ġbatter":4984,"uate":4985,"ĠSecretary":4986,"Ġagreed":4987,"Ġtruly":4988,"ynam":4989,"ĠRet":4990,"Ġunits":4991,"ĠResearch":4992,"hand":4993,"azine":4994,"ĠMike":4995,"Ġvariety":4996,"otal":4997,"Ġamazing":4998,"<|endoftext|>":4999}
\ No newline at end of file

megatron-deepspeed_dtk22.10/tests/ds_config.json

+{
+  "train_micro_batch_size_per_gpu": 1,
+  "train_batch_size": 16,
+  "gradient_clipping": 1.0,
+  "zero_optimization": {
+    "stage": 1
+  },
+  "fp16": {
+    "enabled": true,
+    "loss_scale": 0,
+    "loss_scale_window": 500,
+    "hysteresis": 2,
+    "min_loss_scale": 1,
+    "initial_scale_power": 12
+  },
+  "zero_allow_untested_optimizer": true,
+  "steps_per_print": 2000,
+  "wall_clock_breakdown": false
+}

megatron-deepspeed_dtk22.10/tests/ds_config_bf16.json

+{
+  "train_micro_batch_size_per_gpu": 1,
+  "train_batch_size": 16,
+  "gradient_clipping": 1.0,
+  "zero_optimization": {
+    "stage": 0
+  },
+  "bf16": {
+    "enabled": true
+  },
+  "zero_allow_untested_optimizer": true,
+  "steps_per_print": 2000,
+  "wall_clock_breakdown": false
+}

megatron-deepspeed_dtk22.10/tests/ds_config_cl.json

+{
+  "train_micro_batch_size_per_gpu": 1,
+  "train_batch_size": 16,
+  "gradient_clipping": 1.0,
+  "zero_optimization": {
+    "stage": 1
+  },
+  "fp16": {
+    "enabled": true,
+    "loss_scale": 0,
+    "loss_scale_window": 500,
+    "hysteresis": 2,
+    "min_loss_scale": 1,
+    "initial_scale_power": 12
+  },
+  "curriculum_learning": {
+    "enabled": true,
+    "curriculum_type": "seqlen",
+    "min_difficulty": 8,
+    "max_difficulty": 128,
+    "schedule_type": "fixed_linear",
+    "schedule_config": {
+      "total_curriculum_step": 30,
+      "difficulty_step": 4
+    }
+  },
+  "steps_per_print": 2000,
+  "wall_clock_breakdown": false
+}

megatron-deepspeed_dtk22.10/tests/ds_config_inference.json

+{
+  "train_micro_batch_size_per_gpu": 1,
+  "train_batch_size": 16,
+  "fp16": {
+    "enabled": true,
+    "loss_scale": 0,
+    "loss_scale_window": 500,
+    "hysteresis": 2,
+    "min_loss_scale": 1,
+    "initial_scale_power": 12
+  },
+  "zero_allow_untested_optimizer": false,
+  "steps_per_print": 2000,
+  "wall_clock_breakdown": false
+}

megatron-deepspeed_dtk22.10/tests/test_activations.py

+import random
+import unittest
+
+import torch
+from torch.nn import functional as F
+
+from megatron.model.glu_activations import GLU_ACTIVATIONS, geglu, liglu, reglu, swiglu
+from megatron.testing_utils import set_seed, torch_assert_equal
+
+
+class TestActivations(unittest.TestCase):
+    def setUp(self):
+        """setup an input of reasonable size"""
+        set_seed()
+        self.batch_size = random.randint(2, 64)
+        self.seq_len = random.randint(256, 1025)
+        self.num_channels = random.randint(1, 384) * 2
+        self.x = torch.randn(self.batch_size, self.seq_len, self.num_channels)
+        self.x1, self.x2 = self.x.chunk(2, dim=-1)
+        # glu should halve the last dimension
+        self.output_shape = [self.batch_size, self.seq_len, self.num_channels // 2]
+
+    def test_shapes(self):
+        for activation_fn in GLU_ACTIVATIONS.values():
+            output = activation_fn(self.x)
+            self.assertEqual(list(output.shape), self.output_shape)
+
+    def test_liglu(self):
+        expected = self.x1 * self.x2
+        torch_assert_equal(liglu(self.x), expected)
+
+    def test_geglu(self):
+        expected = self.x1 * F.gelu(self.x2)
+        torch_assert_equal(geglu(self.x), expected)
+
+    def test_reglu(self):
+        expected = self.x1 * F.relu(self.x2)
+        torch_assert_equal(reglu(self.x), expected)
+
+    def test_swiglu(self):
+        expected = self.x1 * F.silu(self.x2)
+        torch_assert_equal(swiglu(self.x), expected)
+
+    # from megatron.testing_utils import require_torch_bf16
+    # @require_torch_bf16
+    # def test_bf16_jit(self):
+    #     x_bf16 = self.x.to(torch.bfloat16)
+    #     for activation_fn in GLU_ACTIVATIONS.values():
+    #         output = activation_fn(x_bf16)
+    #         self.assertEqual(list(output.shape), self.output_shape)

megatron-deepspeed_dtk22.10/tests/test_basic.py

+def test_import():
+    import megatron

megatron-deepspeed_dtk22.10/tests/test_checkpoints.py

+# Copyright 2020 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import io
+import os
+import pytest
+from pathlib import Path
+
+from parameterized import parameterized
+from megatron.testing_utils import (
+    CaptureStdout,
+    TestCasePlus,
+    execute_subprocess_async,
+    get_gpu_count,
+    require_deepspeed,
+    require_torch_gpu,
+    require_torch_multi_gpu,
+    set_seed
+)
+
+set_seed(42)
+
+
+def parameterized_custom_name_func(func, param_num, param):
+    # customize the test name generator function as we want both params to appear in the sub-test
+    # name, as by default it shows only the first param
+    param_based_name = parameterized.to_safe_name("_to_".join(str(x) for x in param.args))
+    return f"{func.__name__}_{param_based_name}"
+
+params = [
+    # TP_PP_DP
+    ["1_1_1", "1_1_1"],
+    ["2_1_1", "1_1_1"],
+    ["1_2_1", "1_1_1"],
+    ["1_1_2", "1_1_1"],
+
+    ["2_1_1", "2_1_1"],
+    ["1_1_1", "2_1_1"],
+    ["1_1_1", "1_2_1"],
+    ["1_1_1", "1_1_2"],
+
+    ["1_1_2", "1_1_2"],
+    ["1_1_2", "2_1_1"],
+    ["1_1_2", "1_2_1"],
+
+    ["1_2_1", "1_2_1"],
+    ["1_2_1", "2_1_1"],
+    ["1_2_1", "1_1_2"],
+
+    ["2_1_1", "2_1_1"],
+    ["2_1_1", "1_2_1"],
+    ["2_1_1", "1_1_2"],
+
+    ["2_2_2", "1_1_1"],
+    ["2_2_2", "2_2_2"],
+    ["1_1_1", "2_2_2"],
+
+    ["1_1_8", "2_2_2"],
+
+]
+
+def get_launcher(num_gpus):
+    # 1. explicitly set --num_nodes=1 just in case these tests end up run on a multi-node setup
+    # - it won't be able to handle that
+    return f"deepspeed --num_nodes 1 --num_gpus {num_gpus}".split()
+
+@require_deepspeed
+@require_torch_gpu
+class MegDSTestCheckpoints(TestCasePlus):
+    """ """
+
+    def setUp(self):
+        super().setUp()
+
+        # at times magatron fails to build kernels and doesn't remove the lock file, which makes
+        # subsequent runs hang - so make sure there is no lock when starting the testing
+        meg_lock_file_path = self.repo_root_dir_str + "/megatron/fused_kernels/build/lock"
+        if os.path.exists(meg_lock_file_path):
+            os.unlink(meg_lock_file_path)
+
+    def get_config(self, output_dir, tp_size, pp_size, dp_size):
+        data_dir = f"{self.data_dir}/gpt2"
+
+        num_gpus = pp_size * tp_size * dp_size
+        print(f"Using {num_gpus} GPUs")
+
+        n_samples = 300 # about 56 iterations
+
+        exit_interval = 20 # some samples in the first half and then some more in the 2nd half after resume
+        seq_len = 128
+
+        # XXX: for now while testing shapes make it really short and fast
+        exit_interval = 1
+        seq_len = 8
+
+
+        # common/shared configs
+
+        ds_args = f"""
+                --deepspeed
+                --deepspeed_config {self.test_file_dir_str}/ds_config_bf16.json
+                --zero-stage 0
+                --deepspeed-activation-checkpointing
+        """.split()
+
+        args = f"""
+                --tensor-model-parallel-size {tp_size}
+                --pipeline-model-parallel-size {pp_size}
+                --distributed-backend nccl
+
+                --log-interval 1
+                --save-interval 1
+                --eval-interval 10
+                --eval-iters 1
+                --checkpoint-activations
+                --partition-activations
+                --exit-interval {exit_interval}
+
+                --merge-file {data_dir}/gpt2-tiny-merges.txt
+                --vocab-file {data_dir}/gpt2-tiny-vocab.json
+                --save {output_dir}/checkpoints
+                --load {output_dir}/checkpoints
+                --data-path {data_dir}/meg-gpt2-openwebtext_text_document
+                --tensorboard-dir {output_dir}/tensorboard
+                --tensorboard-queue-size 5
+                --log-timers-to-tensorboard
+                --log-batch-size-to-tensorboard
+                --log-validation-ppl-to-tensorboard
+
+                --num-layers 2
+                --hidden-size 8
+                --num-attention-heads 2
+                --seq-length {seq_len}
+                --max-position-embeddings 8
+                --micro-batch-size 1
+                --global-batch-size 16
+                --train-samples {n_samples}
+
+                --embed-layernorm
+                --position-embedding-type alibi
+
+                --optimizer adam
+                --adam-beta1 0.9
+                --adam-beta2 0.95
+                --adam-eps 1e-8
+                --lr 1e-4
+                --lr-warmup-samples 5
+                --lr-decay-samples 6
+                --clip-grad 1.0
+                --weight-decay 1e-1
+                --bf16
+
+                --log-level debug
+                --log-level-replica info
+        """.split()
+
+
+        # XXX: fails to handle:
+        #--embed-layernorm
+        #
+# stderr: RuntimeError: Error(s) in loading state_dict for VocabParallelEmbedding:
+# stderr:         size mismatch for norm.weight: copying a param with shape torch.Size([128]) from checkpoint, the shape in current model is torch.Size([64]).
+# stderr:         size mismatch for norm.bias: copying a param with shape torch.Size([128]) from checkpoint, the shape in current model is torch.Size([64]).
+
+        return args, ds_args, num_gpus
+
+
+    def train_checkpoint(self, output_dir, tp_size=1, pp_size=1, dp_size=1):
+        src_dir = self.src_dir
+        script = [f"{src_dir}/pretrain_gpt.py"]
+
+        args, ds_args, num_gpus = self.get_config(output_dir, tp_size, pp_size, dp_size)
+        launcher = get_launcher(num_gpus)
+        cmd = launcher + script + args + ds_args
+        # keep for quick debug
+        #print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
+
+        # 1. test training from scratch (no checkpoint)
+        with CaptureStdout() as cs:
+            execute_subprocess_async(cmd, env=self.get_env())
+
+        # test deepspeed is running
+        self.assertIn("DeepSpeed info", cs.out)
+
+        # test reports
+        self.assertIn("consumed samples", cs.out)
+
+        # test there should be no checkpoint this round
+        self.assertIn(f"Unable to find latest file at {output_dir}/checkpoints/latest", cs.out)
+
+        # test checkpoint saving
+        self.assertIn("successfully saved checkpoint at iteration", cs.out)
+
+    def convert_checkpoint_to_universal(self, output_dir, step):
+        cmd = f"""
+            python tools/convert_checkpoint/ds_to_universal.py
+            --input_folder  {output_dir}/checkpoints/global_step{step}
+            --output_folder {output_dir}/checkpoints/global_step{step}_universal
+        """.split()
+        # keep for quick debug
+        # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
+
+        with CaptureStdout() as cs:
+            execute_subprocess_async(cmd, env=self.get_env())
+
+        self.assertIn("Convert DeepSpeed Checkpoint to Universal Checkpoint", cs.out)
+
+    def resume_from_checkpoint(self, output_dir, tp_size=1, pp_size=1, dp_size=1):
+        src_dir = self.src_dir
+        script = [f"{src_dir}/pretrain_gpt.py"]
+
+        args, ds_args, num_gpus = self.get_config(output_dir, tp_size, pp_size, dp_size)
+        launcher = get_launcher(num_gpus)
+        cmd = launcher + script + args + ds_args
+        # keep for quick debug
+        # print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
+
+        with CaptureStdout() as cs:
+            execute_subprocess_async(cmd, env=self.get_env())
+
+        # test checkpoint loading
+        self.assertIn(f"successfully loaded checkpoint from {output_dir}/checkpoints", cs.out)
+
+        # test reports
+        self.assertIn("consumed samples", cs.out)
+
+        # test checkpoint saving
+        self.assertIn("successfully saved checkpoint at iteration", cs.out)
+
+    def resume_from_universal_checkpoint(self, output_dir, tp_size=1, pp_size=1, dp_size=1):
+        src_dir = self.src_dir
+        script = [f"{src_dir}/pretrain_gpt.py"]
+
+        args, ds_args, num_gpus = self.get_config(output_dir, tp_size, pp_size, dp_size)
+        launcher = get_launcher(num_gpus)
+        cmd = launcher + script + args + ds_args + ["--universal-checkpoint"]
+        # keep for quick debug
+        #print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
+
+        with CaptureStdout() as cs:
+            execute_subprocess_async(cmd, env=self.get_env())
+
+        # test checkpoint loading
+        self.assertIn(f"successfully loaded checkpoint from {output_dir}/checkpoints", cs.out)
+
+        # test reports
+        self.assertIn("consumed samples", cs.out)
+
+        # test checkpoint saving
+        self.assertIn("successfully saved checkpoint at iteration", cs.out)
+
+
+    @require_torch_multi_gpu
+    @parameterized.expand(params, name_func=parameterized_custom_name_func)
+    def test_checkpoint_reshaping_main(self, src, tgt):
+        # this test needs at least 2 gpus - if there are more gpus it will do more extensive testing
+
+        tp_size_src, pp_size_src, dp_size_src = list(map(int, src.split('_')))
+        tp_size_tgt, pp_size_tgt, dp_size_tgt = list(map(int, tgt.split('_')))
+
+        n_gpus = get_gpu_count()
+        n_gpus_src = tp_size_src * pp_size_src * dp_size_src
+        n_gpus_tgt = tp_size_tgt * pp_size_tgt * dp_size_tgt
+
+        if n_gpus_src > n_gpus:
+            pytest.skip(f"the test requires {n_gpus_src} gpus for source topology but have only {n_gpus}")
+        if n_gpus_tgt > n_gpus:
+            pytest.skip(f"the test requires {n_gpus_tgt} gpus for target topology but have only {n_gpus}")
+
+        output_dir = self.get_auto_remove_tmp_dir("./xxx", after=False)
+
+        # 1. train with initial topology defined in the first arg of params
+        self.train_checkpoint(output_dir, tp_size=tp_size_src , pp_size=pp_size_src , dp_size=dp_size_src )
+
+        # 2. convert checkpoint to universal checkpoint (topology )
+        self.convert_checkpoint_to_universal(output_dir=output_dir, step=1)
+
+        # 3. check we can resume training from a reshaped checkpoint to the target topology - the last arg of params
+        self.resume_from_universal_checkpoint(output_dir, tp_size=tp_size_tgt, pp_size=pp_size_tgt, dp_size=dp_size_tgt)
+
+
+    @require_torch_multi_gpu
+    def test_checkpoint_reshaping_empty_dir(self):
+
+        output_dir = self.get_auto_remove_tmp_dir() # "./xxx", after=False)
+        with self.assertRaises(RuntimeError) as context:
+            self.convert_checkpoint_to_universal(output_dir=output_dir, step=1)

megatron-deepspeed_dtk22.10/tests/test_dataloaders.py

+import itertools
+import os
+import shutil
+from typing import Set
+from unittest.mock import patch
+
+import deepspeed
+import torch
+
+import finetune_t0_non_causal_decoder
+from megatron import global_vars, get_tokenizer, initialize_megatron, get_args
+from megatron.data import mlm_dataset, mtf_dataset, decoder_packed_mtf_dataset
+from megatron.data.data_samplers import build_pretraining_data_loader
+from megatron.testing_utils import TestCasePlus, flatten_arguments, mockenv_context, torch_assert_equal
+
+
+def get_default_args():
+    """return a dictionary with key as argument name and value as additional arguments"""
+    return {
+        # GPT_ARGS
+        "--num-layers": "2",
+        "--hidden-size": "128",
+        "--num-attention-heads": "4",
+        "--seq-length": "512",
+        "--max-position-embeddings": "512",
+        "--micro-batch-size": "4",
+        "--global-batch-size": "8",
+        "--lr-decay-iters": "320000",
+        "--lr-decay-style": "cosine",
+        "--lr": "0.00015",
+        "--min-lr": "1.0e-5",
+        "--train-iters": "5000",
+        "--tokenizer-type": "PretrainedFromHF",
+        "--tokenizer-name-or-path": "gpt2",
+        "--data-impl": "mmap",
+        "--split": "949,50,1",
+        "--distributed-backend": "nccl",
+        "--weight-decay": "1e-2",
+        "--clip-grad": "1.0",
+        "--lr-warmup-fraction": ".01",
+        "--fp16": "",
+
+        "--attention-dropout": "0",
+        "--hidden-dropout": "0",
+
+        # OUTPUT_ARGS
+        "--log-interval": "10",
+        "--save-interval": "500",
+        "--eval-interval": "100",
+        "--eval-iters": "10",
+        "--checkpoint-activations": "",
+
+        # DATA_ARGS
+    }
+
+def get_dummy_mtf_decoder_packed_data(micro_batch_size: int, seq_length: int, vocab_size: int, special_tokens_ids: Set[int]):
+    seq_length += 1
+
+    num_segments = torch.randint(1, 5, ())
+    segment_ids = torch.zeros(micro_batch_size, seq_length, dtype=torch.long)
+    is_inputs = torch.zeros(micro_batch_size, seq_length, dtype=torch.bool)
+    for batch_id in range(micro_batch_size):
+        # - `*2`: Hack in order to two start_new_segements to be seperated with two tokens at least
+        # - `+1`: Hack in order the start_mew_segments not to be 0
+        start_new_segments = torch.sort(torch.randperm((seq_length - 2) // 2, )[:num_segments]).values * 2 + 1
+        segment_ids[batch_id, start_new_segments] = 1
+
+        end_inputs = [
+            torch.randint(low=start_segment, high=end_segment, size=())
+            for start_segment, end_segment in zip([0, *start_new_segments], [*start_new_segments, seq_length])
+        ]
+        for end_input, start_segment in zip(end_inputs, [0, *start_new_segments]):
+            is_inputs[batch_id][start_segment: end_input + 1] = True
+
+    segment_ids = torch.cumsum(segment_ids, dim=-1) + 1
+
+    tokens = torch.randint(high=vocab_size, size=(micro_batch_size, seq_length), dtype=torch.long)
+    flatten_token_view = tokens.view(-1,)
+    for token_id in range(len(flatten_token_view)):
+        token = flatten_token_view[token_id]
+        # While token is a special tokens we change that token
+        while token in special_tokens_ids:
+            flatten_token_view[token_id] = (token + 1) % vocab_size
+            token = flatten_token_view[token_id]
+
+    return {
+        "decoder_token_ids": tokens,
+        "decoder_segment_ids": segment_ids,
+        "decoder_is_inputs": is_inputs
+    }
+
+class TestDataLoading(TestCasePlus):
+    def setUp(self) -> None:
+        super().setUp()
+
+        # We reset all global variables
+        global_vars._GLOBAL_ARGS = None
+        global_vars._GLOBAL_NUM_MICROBATCHES_CALCULATOR = None
+        global_vars._GLOBAL_TOKENIZER = None
+        global_vars._GLOBAL_TENSORBOARD_WRITER = None
+        global_vars._GLOBAL_ADLR_AUTORESUME = None
+        global_vars._GLOBAL_TIMERS = None
+
+        self.dist_env_1_gpu = dict(
+            MASTER_ADDR="localhost", MASTER_PORT="9994", RANK="0", LOCAL_RANK="0", WORLD_SIZE="1"
+        )
+
+    def copy_data_to_temp(self, root_dir, prefix):
+        """copy data to temp, and return paths to temp version"""
+        src_path = os.path.join(root_dir, prefix)
+        src_dirname = os.path.dirname(src_path)
+
+        tmp_dir = self.get_auto_remove_tmp_dir()
+        dest_path = os.path.join(tmp_dir, prefix)
+        dest_dirname = os.path.dirname(dest_path)
+        os.makedirs(dest_dirname, exist_ok=True)
+        for folder in os.listdir(src_dirname):
+            src_folder = os.path.join(src_dirname, folder)
+            dest_folder = os.path.join(dest_dirname, folder)
+            if src_folder.startswith(src_path):
+                if os.path.isdir(src_folder):
+                    shutil.copytree(src_folder, dest_folder)
+                else:
+                    shutil.copy2(src_folder, dest_folder)
+        return dest_path
+
+    def test_mlm_dataset(self):
+        command_args = get_default_args()
+        data_path = self.copy_data_to_temp(self.data_dir, "gpt2/meg-gpt2-openwebtext_text_document")
+        command_args["--data-path"] = data_path
+        command_args["--noise-density"] = "0.15"
+        command_args["--mean-noise-span-length"] = "3"
+        command_args["--vocab-extra-ids"] = "100"
+
+        with patch('sys.argv', flatten_arguments(command_args)):
+            with mockenv_context(**self.dist_env_1_gpu):
+                deepspeed.init_distributed()
+                initialize_megatron()
+
+                # tokenizer
+                tokenizer = get_tokenizer()
+                # SEP is required to put in MLM preprocessed.
+                tokenizer.tokenizer.add_special_tokens({"sep_token": "<s>"})
+
+                args = get_args()
+                train_val_test_num_samples = [
+                    args.train_iters * args.global_batch_size,
+                    args.eval_iters * args.global_batch_size,
+                    0
+                ]
+                train_ds, valid_ds, test_ds = mlm_dataset.build_train_valid_test_datasets(
+                    data_prefix=args.data_path,
+                    data_impl=args.data_impl,
+                    splits_string=args.split,
+                    # TODO @thomasw21 figure how that value works
+                    train_valid_test_num_samples=train_val_test_num_samples,
+                    sequence_length=args.seq_length,
+                    noise_density=args.noise_density,
+                    mean_noise_span_length=args.mean_noise_span_length,
+                    seed=args.seed,
+                    skip_warmup=(not args.mmap_warmup)
+                )
+
+                sample = train_ds[0]
+                # +1 is needed to compute labels. As inputs and targets are just concatenated.
+                self.assertEqual(len(sample["input_tokens"]) + len(sample["target_tokens"]), args.seq_length + 1)
+
+                # We make sure that inputs/targets end with <sep>
+                self.assertEqual(sample["input_tokens"][-1], tokenizer.sep)
+                self.assertEqual(sample["target_tokens"][-1], tokenizer.sep)
+
+    def test_decoder_packed_mtf_dataloader(self):
+        command_args = get_default_args()
+        data_path = self.copy_data_to_temp(self.data_dir, "gpt2/ag_news_prompt")
+        command_args["--data-path"] = data_path
+
+        with patch('sys.argv', flatten_arguments(command_args)):
+            with mockenv_context(**self.dist_env_1_gpu):
+                deepspeed.init_distributed()
+                initialize_megatron()
+
+                args = get_args()
+                tokenizer = get_tokenizer()
+                # Hack: `gpt2` doesn't have a padding token, so we override that value.
+                tokenizer.tokenizer.pad_token_id = tokenizer.tokenizer.eos_token_id
+
+                train_val_test_num_samples = [
+                    args.train_iters * args.global_batch_size,
+                    args.eval_iters * args.global_batch_size,
+                    0
+                ]
+                train_ds, valid_ds, test_ds = decoder_packed_mtf_dataset.build_train_valid_test_datasets(
+                    data_prefix=args.data_path,
+                    data_impl=args.data_impl,
+                    splits_string=args.split,
+                    # TODO @thomasw21 figure how that value works
+                    train_valid_test_num_samples=train_val_test_num_samples,
+                    seq_length=args.seq_length + 1,
+                    pad_token=tokenizer.pad,
+                    eos_token=tokenizer.eos,
+                    seed=args.seed,
+                    skip_warmup=(not args.mmap_warmup)
+                )
+
+                batch_iterator = build_pretraining_data_loader(
+                    train_ds, consumed_samples=0, num_workers=4
+                )
+
+                last_padding_size = 0
+                for i, items in enumerate(batch_iterator):
+                    micro_batch_size, seq_length = items["decoder_token_ids"].shape
+
+                    # Check dtypes
+                    self.assertEqual(items["decoder_token_ids"].dtype, torch.int64)
+                    self.assertEqual(items["decoder_segment_ids"].dtype, torch.int64)
+                    self.assertEqual(items["decoder_is_inputs"].dtype, torch.bool)
+
+                    # `micro_batch_size` correspond to the one in argument
+                    self.assertEqual(micro_batch_size, args.micro_batch_size)
+                    # `seq_length` correspond to the one in argument + 1 in order to get tokens/labels
+                    self.assertEqual(seq_length, args.seq_length + 1)
+
+                    original_samples_count = 0
+                    for batch_id in range(micro_batch_size):
+                        segment_ids = [k for k, _ in itertools.groupby(items["decoder_segment_ids"][batch_id])]
+                        # `segment_ids` is [1,2,...]
+                        self.assertEqual(segment_ids[:-1], list(range(1, len(segment_ids))))
+                        # `0` signify that the tokens are padding
+                        self.assertIn(segment_ids[-1], [0, len(segment_ids)])
+                        original_samples_count += len([segment_id for segment_id in segment_ids if segment_id != 0])
+
+                    # Test that we actually pack, ie we have more samples than the `batch_size`
+                    self.assertGreater(original_samples_count, micro_batch_size)
+
+                    # Test that the first sample of each batch couldn't fit inside the previous batch
+                    first_sample_segment_ids = next(itertools.groupby(items["decoder_segment_ids"][0]))[1]
+                    first_sample_size = len(list(first_sample_segment_ids))
+                    self.assertGreater(first_sample_size, last_padding_size)
+
+                    # update `last_padding_size`
+                    last_padding_size = len([None for segment_id in items["decoder_segment_ids"][micro_batch_size - 1] if segment_id == 0])
+
+
+    def test_finetune_t0_non_causal_decoder_get_batch_pipe(self):
+        command_args = get_default_args()
+        command_args["--position-embedding-type"] = "alibi"
+
+        with patch('sys.argv', flatten_arguments(command_args)):
+            with mockenv_context(**self.dist_env_1_gpu):
+                deepspeed.init_distributed()
+                initialize_megatron()
+
+                args = get_args()
+                tokenizer = get_tokenizer()
+                # Hack: `gpt2` doesn't have a padding token, so we override that value.
+                tokenizer.tokenizer.pad_token_id = tokenizer.tokenizer.eos_token_id
+
+                # Dummy data
+                data = get_dummy_mtf_decoder_packed_data(
+                    micro_batch_size=args.micro_batch_size,
+                    seq_length=args.seq_length,
+                    vocab_size=args.padded_vocab_size,
+                    special_tokens_ids={tokenizer.pad}
+                )
+
+                (tokens, position_ids, attention_mask), (labels, loss_mask) = finetune_t0_non_causal_decoder.get_batch_pipe(data)
+
+                tokens = tokens.cpu()
+                position_ids = position_ids.cpu()
+                attention_mask = attention_mask.cpu()
+                labels = labels.cpu()
+                loss_mask = loss_mask.cpu()
+
+                self.assertEqual(loss_mask.dtype, torch.float)
+                torch_assert_equal(loss_mask.bool(), ~data["decoder_is_inputs"][:, 1:] * (data["decoder_token_ids"][:, :-1] != tokenizer.pad))
+                torch_assert_equal(tokens, data["decoder_token_ids"][:, :-1])
+                torch_assert_equal(labels, data["decoder_token_ids"][:, 1:])
+
+                for batch_id in range(args.micro_batch_size):
+                    segment_cuts = torch.nonzero(data["decoder_segment_ids"][batch_id, 1:] - data["decoder_segment_ids"][batch_id, :-1]) + 1
+                    for segment_start, segment_end in zip([0, *segment_cuts], [*segment_cuts, args.seq_length]):
+                        self.assertTrue(torch.all(attention_mask[batch_id, 0, segment_start: segment_end, :segment_start]))
+                        self.assertTrue(torch.all(attention_mask[batch_id, 0, segment_start: segment_end, segment_end:]))
+
+                # TODO @thomasw21 make sure that we reset `position_ids`